"""simple docstring"""




import os
import re
import shutil
import sys
import tempfile
import unittest

import black


__lowerCamelCase											=  os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, "utils"))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
__lowerCamelCase											=  "    def __init__(self, config):\n        super().__init__()\n        self.transform = BertPredictionHeadTransform(config)\n\n        # The output weights are the same as the input embeddings, but there is\n        # an output-only bias for each token.\n        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)\n\n        self.bias = nn.Parameter(torch.zeros(config.vocab_size))\n\n        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`\n        self.decoder.bias = self.bias\n\n    def forward(self, hidden_states):\n        hidden_states = self.transform(hidden_states)\n        hidden_states = self.decoder(hidden_states)\n        return hidden_states\n"
class     UpperCamelCase__(			unittest.TestCase ):





						def 				snake_case__	(       self     )					->				List[str]:
													A__	     =							tempfile.mkdtemp()
													os.makedirs(os.path.join(self.transformer_dir		,'models/bert/'     )     )
													A__	     =							self.transformer_dir
													shutil.copy(
													    os.path.join(__UpperCAmelCase		,'src/transformers/models/bert/modeling_bert.py'     )		,os.path.join(self.transformer_dir		,'models/bert/modeling_bert.py'     )		,)





						def 				snake_case__	(       self     )					->				Tuple:
													A__	     =							'src/transformers'
													shutil.rmtree(self.transformer_dir     )





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase		,__UpperCAmelCase=None     )					->				str:
													A__	     =							comment + f'''\nclass {class_name}(nn.Module):\n''' + class_code
													if overwrite_result is not None:
																				A__	     =							comment + f'''\nclass {class_name}(nn.Module):\n''' + overwrite_result
													A__	     =							black.Mode(target_versions={black.TargetVersion.PYaa}		,line_length=1_19     )
													A__	     =							black.format_str(__UpperCAmelCase		,mode=__UpperCAmelCase     )
													A__	     =							os.path.join(self.transformer_dir		,'new_code.py'     )
													with open(__UpperCAmelCase		,'w'		,newline='\n'     ) as f:
																				f.write(__UpperCAmelCase     )
													if overwrite_result is None:
																				self.assertTrue(len(check_copies.is_copy_consistent(__UpperCAmelCase     )     ) == 0     )
													else:
																				check_copies.is_copy_consistent(f.name		,overwrite=__UpperCAmelCase     )
																				with open(__UpperCAmelCase		,'r'     ) as f:
																											self.assertTrue(f.read()		,__UpperCAmelCase     )





						def 				snake_case__	(       self     )					->				str:
													A__	     =							check_copies.find_code_in_transformers('models.bert.modeling_bert.BertLMPredictionHead'     )
													self.assertEqual(__UpperCAmelCase		,__UpperCAmelCase     )





						def 				snake_case__	(       self     )					->				Optional[Any]:
													# Base copy consistency
													self.check_copy_consistency(
													    '# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead'		,'BertLMPredictionHead'		,REFERENCE_CODE + '\n'		,)

													# With no empty line at the end
													self.check_copy_consistency(
													    '# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead'		,'BertLMPredictionHead'		,__UpperCAmelCase		,)

													# Copy consistency with rename
													self.check_copy_consistency(
													    '# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel'		,'TestModelLMPredictionHead'		,re.sub('Bert'		,'TestModel'		,__UpperCAmelCase     )		,)

													# Copy consistency with a really long name
													A__	     =							'TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason'
													self.check_copy_consistency(
													    f'''# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}'''		,f'''{long_class_name}LMPredictionHead'''		,re.sub('Bert'		,__UpperCAmelCase		,__UpperCAmelCase     )		,)

													# Copy consistency with overwrite
													self.check_copy_consistency(
													    '# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel'		,'TestModelLMPredictionHead'		,__UpperCAmelCase		,overwrite_result=re.sub('Bert'		,'TestModel'		,__UpperCAmelCase     )		,)





						def 				snake_case__	(       self     )					->				Tuple:
													A__	     =							check_copies.LOCALIZED_READMES['README_zh-hans.md']

													A__	     =							(
													    '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the'
													    ' Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for'
													    ' Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong'
													    ' Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1.'
													    ' **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),'
													    ' released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and'
													    ' lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same'
													    ' method has been applied to compress GPT2 into'
													    ' [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into'
													    ' [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),'
													    ' Multilingual BERT into'
													    ' [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German'
													    ' version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**'
													    ' (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders'
													    ' as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang'
													    ' Luong, Quoc V. Le, Christopher D. Manning.'
													)
													A__	     =							(
													    '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'
													    ' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
													    ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
													    ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'
													)
													A__	     =							(
													    '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'
													    ' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
													    ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
													    ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1.'
													    ' **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文'
													    ' [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and'
													    ' lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same'
													    ' method has been applied to compress GPT2 into'
													    ' [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into'
													    ' [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),'
													    ' Multilingual BERT into'
													    ' [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German'
													    ' version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自'
													    ' Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather'
													    ' than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,'
													    ' Christopher D. Manning 发布。\n'
													)

													A__     ,  A__	     =							check_copies.convert_to_localized_md(
													    __UpperCAmelCase		,__UpperCAmelCase		,localized_readme['format_model_list']     )

													self.assertFalse(__UpperCAmelCase     )
													self.assertEqual(__UpperCAmelCase		,__UpperCAmelCase     )

													A__     ,  A__	     =							check_copies.convert_to_localized_md(
													    __UpperCAmelCase		,__UpperCAmelCase		,localized_readme['format_model_list']     )

													# Check whether the number of models is equal to README.md after conversion.
													self.assertTrue(__UpperCAmelCase     )

													A__	     =							(
													    '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the'
													    ' Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for'
													    ' Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong'
													    ' Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.'
													)
													A__	     =							(
													    '1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and'
													    ' the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
													    ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
													    ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'
													)
													A__	     =							(
													    '1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the'
													    ' Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of'
													    ' Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian'
													    ' Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n'
													)

													A__     ,  A__	     =							check_copies.convert_to_localized_md(
													    __UpperCAmelCase		,__UpperCAmelCase		,localized_readme['format_model_list']     )

													# Check if the model link is synchronized.
													self.assertEqual(__UpperCAmelCase		,__UpperCAmelCase     )






"""simple docstring"""




import os
from typing import List, Optional, Union

from ...tokenization_utils import PreTrainedTokenizer
from ...tokenization_utils_base import AddedToken
from ...utils import logging


__lowerCamelCase											=  logging.get_logger(__name__)

__lowerCamelCase											=  {"vocab_file": "vocab.txt"}

__lowerCamelCase											=  {
    "vocab_file": {
        "facebook/esm2_t6_8M_UR50D": "https://huggingface.co/facebook/esm2_t6_8M_UR50D/resolve/main/vocab.txt",
        "facebook/esm2_t12_35M_UR50D": "https://huggingface.co/facebook/esm2_t12_35M_UR50D/resolve/main/vocab.txt",
    },
}

__lowerCamelCase											=  {
    "facebook/esm2_t6_8M_UR50D": 10_24,
    "facebook/esm2_t12_35M_UR50D": 10_24,
}






def   UpperCAmelCase					(	UpperCamelCase__				):






							"""simple docstring"""





							with open(UpperCamelCase__       ,    'r'				) as f:
														A__	     =							f.read().splitlines()
														return [l.strip() for l in lines]


class     UpperCamelCase__(			__A ):
						lowerCAmelCase__  :						Any													=		VOCAB_FILES_NAMES
						lowerCAmelCase__  :						List[str]													=		PRETRAINED_VOCAB_FILES_MAP
						lowerCAmelCase__  :						List[Any]													=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowerCAmelCase__  :						Dict													=		['input_ids', 'attention_mask']





						def __init__(       self		,__UpperCAmelCase		,__UpperCAmelCase="<unk>"		,__UpperCAmelCase="<cls>"		,__UpperCAmelCase="<pad>"		,__UpperCAmelCase="<mask>"		,__UpperCAmelCase="<eos>"		,**__UpperCAmelCase		,)					->				Optional[int]:
													super().__init__(**__UpperCAmelCase     )
													A__	     =							load_vocab_file(__UpperCAmelCase     )
													A__	     =							dict(enumerate(self.all_tokens     )     )
													A__	     =							{tok: ind for ind, tok in enumerate(self.all_tokens     )}
													A__	     =							unk_token
													A__	     =							cls_token
													A__	     =							pad_token
													A__	     =							mask_token
													A__	     =							eos_token
													A__	     =							self.all_tokens
													self._create_trie(self.unique_no_split_tokens     )





						def 				snake_case__	(       self		,__UpperCAmelCase     )					->				str:
													return self._id_to_token.get(__UpperCAmelCase		,self.unk_token     )





						def 				snake_case__	(       self		,__UpperCAmelCase     )					->				int:
													return self._token_to_id.get(__UpperCAmelCase		,self._token_to_id.get(self.unk_token     )     )





						def 				snake_case__	(       self		,__UpperCAmelCase		,**__UpperCAmelCase     )					->				Optional[int]:
													return text.split()





						def 				snake_case__	(       self		,__UpperCAmelCase=False     )					->				int:
													return len(self._id_to_token     )





						def 				snake_case__	(       self     )					->				List[str]:
													return {token: i for i, token in enumerate(self.all_tokens     )}





						def 				snake_case__	(       self		,__UpperCAmelCase     )					->				int:
													return self._token_to_id.get(__UpperCAmelCase		,self._token_to_id.get(self.unk_token     )     )





						def 				snake_case__	(       self		,__UpperCAmelCase     )					->				str:
													return self._id_to_token.get(__UpperCAmelCase		,self.unk_token     )





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase = None     )					->				List[int]:
													A__	     =							[self.cls_token_id]
													A__	     =							[self.eos_token_id]  # No sep token in ESM vocabulary
													if token_ids_a is None:
																				if self.eos_token_id is None:
																											return cls + token_ids_a
																				else:
																											return cls + token_ids_a + sep
													elif self.eos_token_id is None:
																				raise ValueError('Cannot tokenize multiple sequences when EOS token is not set!'     )
													return cls + token_ids_a + sep + token_ids_a + sep  # Multiple inputs always have an EOS token





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase = None		,__UpperCAmelCase = False     )					->				List[int]:
													if already_has_special_tokens:
																				if token_ids_a is not None:
																											raise ValueError(
																											    'You should not supply a second sequence if the provided sequence of '
																											    'ids is already formatted with special tokens for the model.'     )

																				return [1 if token in self.all_special_ids else 0 for token in token_ids_a]
													A__	     =							[1] + ([0] * len(__UpperCAmelCase     )) + [1]
													if token_ids_a is not None:
																				mask += [0] * len(__UpperCAmelCase     ) + [1]
													return mask





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase     )					->				List[str]:
													A__	     =							os.path.join(__UpperCAmelCase		,(filename_prefix + '-' if filename_prefix else '') + 'vocab.txt'     )
													with open(__UpperCAmelCase		,'w'     ) as f:
																				f.write('\n'.join(self.all_tokens     )     )
													return (vocab_file,)





						@property
						def 				snake_case__	(       self     )					->				int:
													return self.get_vocab_size(with_added_tokens=__UpperCAmelCase     )





						def 				snake_case__	(       self		,__UpperCAmelCase		,__UpperCAmelCase = False     )					->				int:
													return super()._add_tokens(__UpperCAmelCase		,special_tokens=__UpperCAmelCase     )





