node: expose more methods in base tokenizer
@@ -8,6 +8,30 @@ export { Encoding, TruncationOptions };
export class BaseTokenizer {
constructor(protected tokenizer: Tokenizer) {}
+ /**
+ * Add the given tokens to the vocabulary
+ *
+ * @param tokens A list of tokens to add to the vocabulary.
+ * Each token can either be a string, or a tuple with a string representing the token,
+ * and a boolean option representing whether to match on single words only.
+ * If the boolean is not included, it defaults to False
+ * @returns The number of tokens that were added to the vocabulary
+ */
+ addTokens(tokens: (string | [string, boolean])[]): number {
+ return this.tokenizer.addTokens(tokens);
+ }
+
+ /**
+ * Add the given special tokens to the vocabulary, and treat them as special tokens.
+ * The special tokens will never be processed by the model, and will be removed while decoding.
+ *
+ * @param tokens The list of special tokens to add
+ * @returns The number of tokens that were added to the vocabulary
+ */
+ addSpecialTokens(tokens: string[]): number {
+ return this.tokenizer.addSpecialTokens(tokens);
+ }
+
/**
* Encode the given sequence
*
@@ -61,4 +85,24 @@ export class BaseTokenizer {
disablePadding(): void {
return this.tokenizer.disablePadding();
}
+
+ /**
+ * Convert the given token id to its corresponding string
+ *
+ * @param id The token id to convert
+ * @returns The corresponding string if it exists
+ */
+ idToToken(id: number): string | undefined {
+ return this.tokenizer.idToToken(id);
+ }
+
+ /**
+ * Convert the given token to its corresponding id
+ *
+ * @param token The token to convert
+ * @returns The corresponding id if it exists
+ */
+ tokenToId(token: string): number | undefined {
+ return this.tokenizer.tokenToId(token);
+ }
}
