Livt packages
Eccelerators.Tokenization
Version 0.2.0
Bounded BPE and Unigram tokenization, text normalization, and decoding components for Livt hardware.
Published files (27)
src/Storage/ArrayStorage.lvt
namespace Eccelerators.Tokenization.Storage
/**
* Dependency-free array provider for small IStorage consumers.
* Owns CAPACITY elements; no BRAM inference or transaction-latency guarantee is made.
* Callers retain the storage interface's exclusive-access and initialization-before-read rules.
*/
component ArrayStorage<T, CAPACITY: int> : IStorage<T>
{
static assert CAPACITY > 0, "invalid storage capacity"
// Owned fallback elements; consumers control which locations are live.
data: T[CAPACITY]
/**
* Returns true for element addresses 0..CAPACITY-1.
*/
override fn IsValidAddress(address: int) bool
{
return address >= 0 && address < CAPACITY
}
/**
* Returns the element at address; asserts that address is in 0..CAPACITY-1.
* Consumers must still obey IStorage initialization-before-read rules.
*/
override fn Read(address: int) T
{
assert IsValidAddress(address)
return data[address]
}
/**
* Stores value at address; asserts that address is in 0..CAPACITY-1.
*/
override fn Write(address: int, value: T)
{
assert IsValidAddress(address)
data[address] = value
}
}