Node - Remove `addSpecialTokens` from `BertWordPieceTokenizer`
@@ -10,10 +10,6 @@ import { wordPieceTrainer } from "../../bindings/trainers";
import { BaseTokenizer, getTokenContent } from "./base.tokenizer";
export interface BertWordPieceOptions {
- /**
- * @default true
- */
- addSpecialTokens?: boolean;
/**
* @default true
*/
@@ -99,7 +95,6 @@ type BertTokenizerConfig = Required<Omit<BertWordPieceOptions, "vocabFile">> & {
*/
export class BertWordPieceTokenizer extends BaseTokenizer<BertTokenizerConfig> {
private static readonly defaultBertOptions: BertTokenizerConfig = {
- addSpecialTokens: true,
cleanText: true,
clsToken: "[CLS]",
handleChineseChars: true,
@@ -164,7 +159,7 @@ export class BertWordPieceTokenizer extends BaseTokenizer<BertTokenizerConfig> {
tokenizer.setNormalizer(normalizer);
tokenizer.setPreTokenizer(bertPreTokenizer());
- if (opts.vocabFile && opts.addSpecialTokens) {
+ if (opts.vocabFile) {
const sepTokenId = tokenizer.tokenToId(getTokenContent(opts.sepToken));
if (sepTokenId === undefined) {
throw new Error("sepToken not found in the vocabulary");
