Class LocalTokenizerProcessor

java.lang.Object

uno.anahata.asi.gemini.tokenizer.LocalTokenizerProcessor

final class LocalTokenizerProcessor
extends Object

Field Summary

Fields

Modifier and Type	Field	Description
private Map<Byte,Token>	byte2Token
private Map<Integer,Byte>	idToByte
private static final Logger	log
private com.google.genai.proto.SentencepieceModel.ModelProto	model
private Map<String,Integer>	pieces
private Map<String,Integer>	reserved
private int	unkID
private Set<String>	userDefined
private Trie	uTrie

Constructor Summary

Constructors

Constructor	Description
LocalTokenizerProcessor(com.google.genai.proto.SentencepieceModel.ModelProto model)

Method Summary

Modifier and Type	Method	Description
private void	addNewCandidate(List<Symbol> symbols, PriorityQueue<MergeCandidate> pq, int left, int right)
private int	convertHexValue(String bv)
String	decodeIds(List<Integer> ids)
List<Token>	encode(String text)
private boolean	isByteId(int id)
private boolean	isMergeCandidateValid(List<Symbol> symbols, MergeCandidate symbol)
private String	maybeMerge(String a, String b)
private String	normalize(String text)
private String	replaceSentencePieceSeparator(String pieceText)
private int	symbolToID(Symbol symbol)

Methods inherited from class Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

log

private static final Logger log

userDefined

private Set<String> userDefined

uTrie

private Trie uTrie

pieces

private Map<String,Integer> pieces

reserved

private Map<String,Integer> reserved

byte2Token

private Map<Byte,Token> byte2Token

idToByte

private Map<Integer,Byte> idToByte

model

private com.google.genai.proto.SentencepieceModel.ModelProto model

unkID

private int unkID

Constructor Details

LocalTokenizerProcessor

public LocalTokenizerProcessor(com.google.genai.proto.SentencepieceModel.ModelProto model)

Method Details

encode

public List<Token> encode(String text)
                   throws IllegalStateException

Throws:

IllegalStateException

decodeIds

public String decodeIds(List<Integer> ids)

normalize

private String normalize(String text)

addNewCandidate

private void addNewCandidate(List<Symbol> symbols,
 PriorityQueue<MergeCandidate> pq,
 int left,
 int right)

maybeMerge

private String maybeMerge(String a,
 String b)

isMergeCandidateValid

private boolean isMergeCandidateValid(List<Symbol> symbols,
 MergeCandidate symbol)

symbolToID

private int symbolToID(Symbol symbol)

isByteId

private boolean isByteId(int id)

replaceSentencePieceSeparator

private String replaceSentencePieceSeparator(String pieceText)

convertHexValue

private int convertHexValue(String bv)