Lolalb commited on
Commit
99b99a1
·
verified ·
1 Parent(s): b8defd9

Upload tokenizer

Browse files
Files changed (2) hide show
  1. tokenizer.py +0 -1
  2. tokenizer_config.json +1 -2
tokenizer.py CHANGED
@@ -73,7 +73,6 @@ class ProteinTokenizer(PreTrainedTokenizerFast):
73
  ambiguous_token_ids=ambiguous_token_ids,
74
  model_input_names=["input_ids", "attention_mask", "special_tokens_mask"],
75
  tokenizer_object=tokenizer_object,
76
- **kwargs,
77
  )
78
 
79
  if other_special_token_ids is not None:
 
73
  ambiguous_token_ids=ambiguous_token_ids,
74
  model_input_names=["input_ids", "attention_mask", "special_tokens_mask"],
75
  tokenizer_object=tokenizer_object,
 
76
  )
77
 
78
  if other_special_token_ids is not None:
tokenizer_config.json CHANGED
@@ -69,6 +69,5 @@
69
  "truncation_side": "right",
70
  "unk_token": "<unk>",
71
  "unk_token_id": 1,
72
- "vocab_path": "/home/mila/l/lola.lebreton/AMPLIFY/conf/tokenizer/amplify_vocab.txt",
73
- "vocab_size": 27
74
  }
 
69
  "truncation_side": "right",
70
  "unk_token": "<unk>",
71
  "unk_token_id": 1,
72
+ "vocab_path": "/home/mila/l/lola.lebreton/AMPLIFY/conf/tokenizer/amplify_vocab.txt"
 
73
  }