Eccelerators.Tokenization

src/Unigram/AsciiMetaspace.lvt

namespace Eccelerators.Tokenization.Unigram

using Eccelerators.Tokenization.Storage

/**
 * Array-backed compatibility facade for StoredAsciiMetaspace.
 * Owns local byte storage and the shared algorithm; borrows the normalization map.
 * Preserves the original bounds, error and Reset contract without imposing a RAM dependency.
 */
component AsciiMetaspace<MAX_INPUT: int, MAX_BYTES: int, MAP: IAsciiNormalization>
{
	// Owned local storage and shared normalization core; map is borrowed through the core.
	storage: ArrayStorage<byte, MAX_BYTES>
	core: StoredAsciiMetaspace<MAX_INPUT, MAX_BYTES, MAP, ArrayStorage<byte, MAX_BYTES>>

	/**
	 * Borrows the normalization map and owns local byte storage plus the shared normalization core.
	 * Call Reset before a request; no model tables or platform RAM are allocated here.
	 */
	new(map: MAP)
	{
		storage = new ArrayStorage<byte, MAX_BYTES>()
		core = new StoredAsciiMetaspace<MAX_INPUT, MAX_BYTES, MAP, ArrayStorage<byte, MAX_BYTES>>(map, storage)
	}

	/**
	 * Invalidates normalized bytes and clears input count, whitespace state and sticky failure.
	 * Does not clear storage or reset the borrowed mapping provider; serialize this with Accept.
	 */
	public fn Reset()
	{
		core.Reset()
	}

	/**
	 * Maps and appends one printable ASCII byte (0x20..0x7e), counting it toward MAX_INPUT.
	 * Waits for all output writes; true means the normalized prefix is available through Read.
	 * Invalid input/mapping, inadequate storage or overflow returns false and invalidates all bytes
	 * until Reset. Runs of spaces become one UTF-8 U+2581 marker.
	 */
	public fn Accept(value: byte) bool
	{
		return core.Accept(value)
	}

	/**
	 * Returns the live normalized byte count, or zero after Reset or failure.
	 */
	public fn Length() int
	{
		return core.Length()
	}

	/**
	 * Returns a live normalized byte at index 0..Length()-1, waiting for its storage read.
	 * Returns zero for invalid indices or a failed request; this does not reserve the byte.
	 */
	public fn Read(index: int) byte
	{
		return core.Read(index)
	}

	/**
	 * Reports a sticky input, mapping, storage-capacity or overflow error since Reset.
	 */
	public fn Failed() bool
	{
		return core.Failed()
	}
}