get_vocab raise NotImplementedError
-- Process 1 terminated with the following error:
Traceback (most recent call last):
File "/home/test/anaconda3/envs/zy/lib/python3.12/site-packages/torch/multiprocessing/spawn.py", line 90, in _wrap
fn(i, *args)
File "/home/test/zy/joy-caption-pre-alpha/app-multi-alpha.py", line 175, in run
models = load_models(rank)
^^^^^^^^^^^^^^^^^
File "/home/test/zy/joy-caption-pre-alpha/app-multi-alpha.py", line 52, in load_models
tokenizer = AutoTokenizer.from_pretrained(MODEL_PATH, use_fast=False)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/test/anaconda3/envs/zy/lib/python3.12/site-packages/transformers/models/auto/tokenization_auto.py", line 896, in from_pretrained
return tokenizer_class.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/test/anaconda3/envs/zy/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2291, in from_pretrained
return cls._from_pretrained(
^^^^^^^^^^^^^^^^^^^^^
File "/home/test/anaconda3/envs/zy/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 2525, in _from_pretrained
tokenizer = cls(*init_inputs, **init_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/test/anaconda3/envs/zy/lib/python3.12/site-packages/transformers/tokenization_utils.py", line 436, in init
self._add_tokens(
File "/home/test/anaconda3/envs/zy/lib/python3.12/site-packages/transformers/tokenization_utils.py", line 536, in _add_tokens
current_vocab = self.get_vocab().copy()
^^^^^^^^^^^^^^^^
File "/home/test/anaconda3/envs/zy/lib/python3.12/site-packages/transformers/tokenization_utils_base.py", line 1685, in get_vocab
raise NotImplementedError()
NotImplementedError