import os
import re
import shutil
import sys
import tempfile
import unittest

import black


__lowerCamelCase    :				List[Any]							     =	os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, '''utils'''))

import check_copies  # noqa: E402


# This is the reference code that will be used in the tests.
# If BertLMPredictionHead is changed in modeling_bert.py, this code needs to be manually updated.
__lowerCamelCase    :				Optional[Any]							     =	'''    def __init__(self, config):
        super().__init__()
        self.transform = BertPredictionHeadTransform(config)

        # The output weights are the same as the input embeddings, but there is
        # an output-only bias for each token.
        self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)

        self.bias = nn.Parameter(torch.zeros(config.vocab_size))

        # Need a link between the two variables so that the bias is correctly resized with `resize_token_embeddings`
        self.decoder.bias = self.bias

    def forward(self, hidden_states):
        hidden_states = self.transform(hidden_states)
        hidden_states = self.decoder(hidden_states)
        return hidden_states
'''







class __snake_case       (		unittest.TestCase	):




   def 			__a		(					self    :       List[str]   ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__    							=   tempfile.mkdtemp()
     os.makedirs(os.path.join(self.transformer_dir ,			"""models/bert/"""   )   )
     SCREAMING_SNAKE_CASE__    							=   self.transformer_dir
     shutil.copy(
         os.path.join(_lowercase ,			"""src/transformers/models/bert/modeling_bert.py"""   ) ,			os.path.join(self.transformer_dir ,			"""models/bert/modeling_bert.py"""   ) ,			)




   def 			__a		(					self    :       List[Any]   ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__    							=   """src/transformers"""
     shutil.rmtree(self.transformer_dir   )




   def 			__a		(					self    :       str ,			_lowercase    :       Union[str, Any] ,			_lowercase    :       int ,			_lowercase    :       str ,			_lowercase    :       Any=None   ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__    							=   comment + f"""\nclass {class_name}(nn.Module):\n""" + class_code
     if overwrite_result is not None:
       SCREAMING_SNAKE_CASE__    							=   comment + f"""\nclass {class_name}(nn.Module):\n""" + overwrite_result
     SCREAMING_SNAKE_CASE__    							=   black.Mode(target_versions={black.TargetVersion.PYaa} ,			line_length=1_19   )
     SCREAMING_SNAKE_CASE__    							=   black.format_str(_lowercase ,			mode=_lowercase   )
     SCREAMING_SNAKE_CASE__    							=   os.path.join(self.transformer_dir ,			"""new_code.py"""   )
     with open(_lowercase ,			"""w""" ,			newline="""\n"""   ) as f:
       f.write(_lowercase   )
     if overwrite_result is None:
       self.assertTrue(len(check_copies.is_copy_consistent(_lowercase   )   ) == 0   )
     else:
       check_copies.is_copy_consistent(f.name ,			overwrite=_lowercase   )
       with open(_lowercase ,			"""r"""   ) as f:
         self.assertTrue(f.read() ,			_lowercase   )




   def 			__a		(					self    :       List[str]   ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__    							=   check_copies.find_code_in_transformers("""models.bert.modeling_bert.BertLMPredictionHead"""   )
     self.assertEqual(_lowercase ,			_lowercase   )




   def 			__a		(					self    :       Optional[Any]   ):



     """simple docstring"""






     self.check_copy_consistency(
         """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead""" ,			"""BertLMPredictionHead""" ,			REFERENCE_CODE + """\n""" ,			)

     # With no empty line at the end
     self.check_copy_consistency(
         """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead""" ,			"""BertLMPredictionHead""" ,			_lowercase ,			)

     # Copy consistency with rename
     self.check_copy_consistency(
         """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel""" ,			"""TestModelLMPredictionHead""" ,			re.sub("""Bert""" ,			"""TestModel""" ,			_lowercase   ) ,			)

     # Copy consistency with a really long name
     SCREAMING_SNAKE_CASE__    							=   """TestModelWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason"""
     self.check_copy_consistency(
         f"""# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->{long_class_name}""" ,			f"""{long_class_name}LMPredictionHead""" ,			re.sub("""Bert""" ,			_lowercase ,			_lowercase   ) ,			)

     # Copy consistency with overwrite
     self.check_copy_consistency(
         """# Copied from transformers.models.bert.modeling_bert.BertLMPredictionHead with Bert->TestModel""" ,			"""TestModelLMPredictionHead""" ,			_lowercase ,			overwrite_result=re.sub("""Bert""" ,			"""TestModel""" ,			_lowercase   ) ,			)




   def 			__a		(					self    :       List[str]   ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__    							=   check_copies.LOCALIZED_READMES["""README_zh-hans.md"""]

     SCREAMING_SNAKE_CASE__    							=   (
         """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"""
         """ Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"""
         """ Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"""
         """ Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut.\n1."""
         """ **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (from HuggingFace),"""
         """ released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"""
         """ lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same"""
         """ method has been applied to compress GPT2 into"""
         """ [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"""
         """ [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"""
         """ Multilingual BERT into"""
         """ [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"""
         """ version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)**"""
         """ (from Google Research/Stanford University) released with the paper [ELECTRA: Pre-training text encoders"""
         """ as discriminators rather than generators](https://arxiv.org/abs/2003.10555) by Kevin Clark, Minh-Thang"""
         """ Luong, Quoc V. Le, Christopher D. Manning."""
     )
     SCREAMING_SNAKE_CASE__    							=   (
         """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
         """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
         """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
         """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
     )
     SCREAMING_SNAKE_CASE__    							=   (
         """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
         """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
         """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
         """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n1."""
         """ **[DistilBERT](https://huggingface.co/transformers/model_doc/distilbert.html)** (来自 HuggingFace) 伴随论文"""
         """ [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and"""
         """ lighter](https://arxiv.org/abs/1910.01108) 由 Victor Sanh, Lysandre Debut and Thomas Wolf 发布。 The same"""
         """ method has been applied to compress GPT2 into"""
         """ [DistilGPT2](https://github.com/huggingface/transformers/tree/main/examples/distillation), RoBERTa into"""
         """ [DistilRoBERTa](https://github.com/huggingface/transformers/tree/main/examples/distillation),"""
         """ Multilingual BERT into"""
         """ [DistilmBERT](https://github.com/huggingface/transformers/tree/main/examples/distillation) and a German"""
         """ version of DistilBERT.\n1. **[ELECTRA](https://huggingface.co/transformers/model_doc/electra.html)** (来自"""
         """ Google Research/Stanford University) 伴随论文 [ELECTRA: Pre-training text encoders as discriminators rather"""
         """ than generators](https://arxiv.org/abs/2003.10555) 由 Kevin Clark, Minh-Thang Luong, Quoc V. Le,"""
         """ Christopher D. Manning 发布。\n"""
     )

     SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__    							=   check_copies.convert_to_localized_md(
         _lowercase ,			_lowercase ,			localized_readme["""format_model_list"""]   )

     self.assertFalse(_lowercase   )
     self.assertEqual(_lowercase ,			_lowercase   )

     SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__    							=   check_copies.convert_to_localized_md(
         _lowercase ,			_lowercase ,			localized_readme["""format_model_list"""]   )

     # Check whether the number of models is equal to README.md after conversion.
     self.assertTrue(_lowercase   )

     SCREAMING_SNAKE_CASE__    							=   (
         """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (from Google Research and the"""
         """ Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for"""
         """ Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong"""
         """ Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut."""
     )
     SCREAMING_SNAKE_CASE__    							=   (
         """1. **[ALBERT](https://huggingface.co/transformers/main/model_doc/albert.html)** (来自 Google Research and"""
         """ the Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
         """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
         """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
     )
     SCREAMING_SNAKE_CASE__    							=   (
         """1. **[ALBERT](https://huggingface.co/transformers/model_doc/albert.html)** (来自 Google Research and the"""
         """ Toyota Technological Institute at Chicago) 伴随论文 [ALBERT: A Lite BERT for Self-supervised Learning of"""
         """ Language Representations](https://arxiv.org/abs/1909.11942), 由 Zhenzhong Lan, Mingda Chen, Sebastian"""
         """ Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut 发布。\n"""
     )

     SCREAMING_SNAKE_CASE__							,  SCREAMING_SNAKE_CASE__    							=   check_copies.convert_to_localized_md(
         _lowercase ,			_lowercase ,			localized_readme["""format_model_list"""]   )

     # Check if the model link is synchronized.
     self.assertEqual(_lowercase ,			_lowercase   )

import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
   import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase    :				Any							     =	logging.get_logger(__name__)

__lowerCamelCase    :				Optional[Any]							     =	{'''vocab_file''': '''vocab.txt'''}

__lowerCamelCase    :				Union[str, Any]							     =	{
    '''vocab_file''': {
        '''openbmb/cpm-ant-10b''': '''https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt''',
    },
}

__lowerCamelCase    :				Optional[Any]							     =	{
    '''openbmb/cpm-ant-10b''': 1024,
}



def 			__SCREAMING_SNAKE_CASE						(      __UpperCamelCase					: str			)							->      Optional[Any]:







  """simple docstring"""





  SCREAMING_SNAKE_CASE__    							=   collections.OrderedDict()
  with open(__UpperCamelCase				,  """r"""				,  encoding="""utf-8"""			) as reader:
    SCREAMING_SNAKE_CASE__    							=   reader.readlines()
  for index, token in enumerate(__UpperCamelCase			):
    SCREAMING_SNAKE_CASE__    							=   token.rstrip("""\n"""			)
    SCREAMING_SNAKE_CASE__    							=   index
  return vocab







class __snake_case       (		lowerCamelCase_	):




   def __init__(					self    :       Optional[int] ,			_lowercase    :       Optional[Any] ,			_lowercase    :       int="<unk>" ,			_lowercase    :       int=2_00   ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__    							=   vocab
     SCREAMING_SNAKE_CASE__    							=   unk_token
     SCREAMING_SNAKE_CASE__    							=   max_input_chars_per_word




   def 			__a		(					self    :       Optional[int] ,			_lowercase    :       List[Any]   ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__    							=   list(_lowercase   )
     if len(_lowercase   ) > self.max_input_chars_per_word:
       return [self.unk_token]

     SCREAMING_SNAKE_CASE__    							=   0
     SCREAMING_SNAKE_CASE__    							=   []
     while start < len(_lowercase   ):
       SCREAMING_SNAKE_CASE__    							=   len(_lowercase   )
       SCREAMING_SNAKE_CASE__    							=   None
       while start < end:
         SCREAMING_SNAKE_CASE__    							=   """""".join(chars[start:end]   )
         if substr in self.vocab:
           SCREAMING_SNAKE_CASE__    							=   substr
           break
         end -= 1
       if cur_substr is None:
         sub_tokens.append(self.unk_token   )
         start += 1
       else:
         sub_tokens.append(_lowercase   )
         SCREAMING_SNAKE_CASE__    							=   end

     return sub_tokens







class __snake_case       (		lowerCamelCase_	):
   lowerCAmelCase_														=							VOCAB_FILES_NAMES
   lowerCAmelCase_														=							PRETRAINED_VOCAB_FILES_MAP
   lowerCAmelCase_														=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCAmelCase_														=							["input_ids", "attention_mask"]
   lowerCAmelCase_														=							False




   def __init__(					self    :       int ,			_lowercase    :       str ,			_lowercase    :       List[Any]="<d>" ,			_lowercase    :       List[Any]="</d>" ,			_lowercase    :       Union[str, Any]="<s>" ,			_lowercase    :       List[str]="</s>" ,			_lowercase    :       str="<pad>" ,			_lowercase    :       int="<unk>" ,			_lowercase    :       List[str]="</n>" ,			_lowercase    :       Tuple="</_>" ,			_lowercase    :       Any="left" ,			**_lowercase    :       Any ,			):



     """simple docstring"""






     requires_backends(self ,			["""jieba"""]   )
     super().__init__(
         bod_token=_lowercase ,			eod_token=_lowercase ,			bos_token=_lowercase ,			eos_token=_lowercase ,			pad_token=_lowercase ,			unk_token=_lowercase ,			line_token=_lowercase ,			space_token=_lowercase ,			padding_side=_lowercase ,			**_lowercase ,			)
     SCREAMING_SNAKE_CASE__    							=   bod_token
     SCREAMING_SNAKE_CASE__    							=   eod_token
     SCREAMING_SNAKE_CASE__    							=   load_vocab(_lowercase   )
     SCREAMING_SNAKE_CASE__    							=   self.encoder[space_token]
     SCREAMING_SNAKE_CASE__    							=   self.encoder[line_token]

     del self.encoder[space_token]
     del self.encoder[line_token]

     SCREAMING_SNAKE_CASE__    							=   collections.OrderedDict(sorted(self.encoder.items() ,			key=lambda _lowercase   : x[1]   )   )
     SCREAMING_SNAKE_CASE__    							=   {v: k for k, v in self.encoder.items()}

     SCREAMING_SNAKE_CASE__    							=   WordpieceTokenizer(vocab=self.encoder ,			unk_token=self.unk_token   )




   @property
   def 			__a		(					self    :       Optional[Any]   ):



     """simple docstring"""






     return self.encoder[self.bod_token]




   @property
   def 			__a		(					self    :       List[Any]   ):



     """simple docstring"""






     return self.encoder[self.eod_token]




   @property
   def 			__a		(					self    :       Any   ):



     """simple docstring"""






     return self.encoder["\n"]




   @property
   def 			__a		(					self    :       Union[str, Any]   ):



     """simple docstring"""






     return len(self.encoder   )




   def 			__a		(					self    :       int   ):



     """simple docstring"""






     return dict(self.encoder ,			**self.added_tokens_encoder   )




   def 			__a		(					self    :       Union[str, Any] ,			_lowercase    :       str   ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__    							=   []
     for x in jieba.cut(_lowercase ,			cut_all=_lowercase   ):
       output_tokens.extend(self.wordpiece_tokenizer.tokenize(_lowercase   )   )
     return output_tokens




   def 			__a		(					self    :       int ,			_lowercase    :       Any ,			**_lowercase    :       Any   ):



     """simple docstring"""






     SCREAMING_SNAKE_CASE__    							=   [i for i in token_ids if i >= 0]
     SCREAMING_SNAKE_CASE__    							=   [
         x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
     ]
     return super()._decode(_lowercase ,			**_lowercase   )




   def 			__a		(					self    :       Optional[int] ,			_lowercase    :       List[Any]   ):



     """simple docstring"""






     return token in self.encoder




   def 			__a		(					self    :       List[str] ,			_lowercase    :       List[str]   ):



     """simple docstring"""






     return "".join(_lowercase   )




   def 			__a		(					self    :       Optional[int] ,			_lowercase    :       Any   ):



     """simple docstring"""






     return self.encoder.get(_lowercase ,			self.encoder.get(self.unk_token   )   )




   def 			__a		(					self    :       Tuple ,			_lowercase    :       List[Any]   ):



     """simple docstring"""






     return self.decoder.get(_lowercase ,			self.unk_token   )




   def 			__a		(					self    :       Optional[Any] ,			_lowercase    :       str ,			_lowercase    :       Optional[str] = None   ):



     """simple docstring"""






     if os.path.isdir(_lowercase   ):
       SCREAMING_SNAKE_CASE__    							=   os.path.join(
           _lowercase ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
     else:
       SCREAMING_SNAKE_CASE__    							=   (filename_prefix + """-""" if filename_prefix else """""") + save_directory
     SCREAMING_SNAKE_CASE__    							=   0
     if " " in self.encoder:
       SCREAMING_SNAKE_CASE__    							=   self.encoder[""" """]
       del self.encoder[" "]
     if "\n" in self.encoder:
       SCREAMING_SNAKE_CASE__    							=   self.encoder["""\n"""]
       del self.encoder["\n"]
     SCREAMING_SNAKE_CASE__    							=   collections.OrderedDict(sorted(self.encoder.items() ,			key=lambda _lowercase   : x[1]   )   )
     with open(_lowercase ,			"""w""" ,			encoding="""utf-8"""   ) as writer:
       for token, token_index in self.encoder.items():
         if index != token_index:
           logger.warning(
               f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
               """ Please check that the vocabulary is not corrupted!"""   )
           SCREAMING_SNAKE_CASE__    							=   token_index
         writer.write(token + """\n"""   )
         index += 1
     return (vocab_file,)




   def 			__a		(					self    :       int ,			_lowercase    :       List[int] ,			_lowercase    :       List[int] = None   ):



     """simple docstring"""






     if token_ids_a is None:
       return [self.bos_token_id] + token_ids_a
     return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a




   def 			__a		(					self    :       Union[str, Any] ,			_lowercase    :       List[int] ,			_lowercase    :       Optional[List[int]] = None ,			_lowercase    :       bool = False   ):



     """simple docstring"""






     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=_lowercase ,			token_ids_a=_lowercase ,			already_has_special_tokens=_lowercase   )

     if token_ids_a is not None:
       return [1] + ([0] * len(_lowercase   )) + [1] + ([0] * len(_lowercase   ))
     return [1] + ([0] * len(_lowercase   ))
