Eccelerators.Tokenization

src/Unigram/UnigramSegmenter.lvt

namespace Eccelerators.Tokenization.Unigram

/**
 * Array-backed compatibility facade for StoredUnigramSegmenter.
 * Owns its local workspace and shared algorithm; borrows the vocabulary.
 * Public calls are serialized; larger hardware designs may inject RAM through the stored core.
 */
component UnigramSegmenter<MAX_BYTES: int, MAX_TOKENS: int, VOCAB: IUnigramVocabulary>
{
	// Owned workspace and shared segmentation core; vocabulary is borrowed.
	storage: ArrayUnigramStorage<5 * MAX_BYTES + 3 + 2 * MAX_TOKENS>
	core: StoredUnigramSegmenter<MAX_BYTES, MAX_TOKENS, VOCAB, ArrayUnigramStorage<5 * MAX_BYTES + 3 + 2 * MAX_TOKENS>>

	/**
	 * Borrows the vocabulary and owns the array workspace and shared segmentation core.
	 * Reset begins a request; callers serialize all operations in the same context.
	 */
	new(vocab: VOCAB)
	{
		storage = new ArrayUnigramStorage<5 * MAX_BYTES + 3 + 2 * MAX_TOKENS>()
		core = new StoredUnigramSegmenter<MAX_BYTES, MAX_TOKENS, VOCAB,
			ArrayUnigramStorage<5 * MAX_BYTES + 3 + 2 * MAX_TOKENS>>(vocab, storage)
	}

	/**
	 * Discards both the current segment and all accumulated token IDs, and clears sticky failure.
	 * Does not clear workspace or reset vocabulary/provider state. Calls must be serialized;
	 * hardware cancellation requires shared context reset with the storage provider.
	 */
	public fn Reset()
	{
		core.Reset()
	}

	/**
	 * Discards current input bytes while retaining accumulated tokens and any sticky failure.
	 * Use at pre-tokenizer boundaries; this does not recover a failed request.
	 */
	public fn BeginSegment()
	{
		core.BeginSegment()
	}

	/**
	 * Waits for one byte to be stored in the current segment, up to MAX_BYTES.
	 * Checks required contiguous workspace capacity before access. Overflow or short storage
	 * invalidates all accumulated tokens and returns false; UTF-8 validation occurs in EncodeSegment.
	 */
	public fn Accept(value: byte) bool
	{
		return core.Accept(value)
	}

	/**
	 * Validates a nonempty UTF-8 segment and waits for exact-cost Viterbi/backtracking to finish.
	 * Appends IDs to the request output, fusing adjacent unknowns only within this segment.
	 * True clears segment bytes for reuse; false invalidates every output and requires Reset.
	 * The caller owns pre-tokenizer boundaries and must not run concurrent provider operations.
	 */
	public fn EncodeSegment() bool
	{
		return core.EncodeSegment()
	}

	/**
	 * Returns the number of accumulated output IDs, or zero after Reset or failure.
	 */
	public fn Count() int
	{
		return core.Count()
	}

	/**
	 * Returns the accumulated token ID at index 0..Count()-1, waiting for workspace access.
	 * Returns -1 for an invalid index or failed request.
	 */
	public fn Token(index: int) int
	{
		return core.Token(index)
	}

	/**
	 * Reports sticky capacity, UTF-8, vocabulary or path-validation failure since Reset.
	 */
	public fn Failed() bool
	{
		return core.Failed()
	}
}