"""simple docstring"""



import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 				snake_case__     (   snake_case_,		unittest.TestCase   ):
 _snake_case   :			List[Any]           =		BertTokenizer
 _snake_case   :			Optional[Any]           =		BertTokenizerFast
 _snake_case   :			int           =		True
 _snake_case   :			Optional[Any]           =		True
 _snake_case   :			List[str]           =		filter_non_english


 def 		a__ (				self		):
        super().setUp()

        __a     				=  [
            "[UNK]",
            "[CLS]",
            "[SEP]",
            "[PAD]",
            "[MASK]",
            "want",
            "##want",
            "##ed",
            "wa",
            "un",
            "runn",
            "##ing",
            ",",
            "low",
            "lowest",
        ]
        __a     				=  os.path.join(self.tmpdirname  ,						VOCAB_FILES_NAMES["vocab_file"]		)
        with open(self.vocab_file  ,						"w"  ,						encoding="utf-8"		) as vocab_writer:
               vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)


 def 		a__ (				self  ,						lowerCamelCase		):
        __a     				=  "UNwant\u00E9d,running"
        __a     				=  "unwanted, running"
        return input_text, output_text


 def 		a__ (				self		):
        __a     				=  self.tokenizer_class(self.vocab_file		)

        __a     				=  tokenizer.tokenize("UNwant\u00E9d,running"		)
        self.assertListEqual(lowerCamelCase  ,						["un", "##want", "##ed", ",", "runn", "##ing"]		)
        self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase		)  ,						[9, 6, 7, 12, 10, 11]		)


 def 		a__ (				self		):
        if not self.test_rust_tokenizer:
               return

        __a     				=  self.get_tokenizer()
        __a     				=  self.get_rust_tokenizer()

        __a     				=  "UNwant\u00E9d,running"

        __a     				=  tokenizer.tokenize(lowerCamelCase		)
        __a     				=  rust_tokenizer.tokenize(lowerCamelCase		)
        self.assertListEqual(lowerCamelCase  ,						lowerCamelCase		)

        __a     				=  tokenizer.encode(lowerCamelCase  ,						add_special_tokens=lowerCamelCase		)
        __a     				=  rust_tokenizer.encode(lowerCamelCase  ,						add_special_tokens=lowerCamelCase		)
        self.assertListEqual(lowerCamelCase  ,						lowerCamelCase		)

        __a     				=  self.get_rust_tokenizer()
        __a     				=  tokenizer.encode(lowerCamelCase		)
        __a     				=  rust_tokenizer.encode(lowerCamelCase		)
        self.assertListEqual(lowerCamelCase  ,						lowerCamelCase		)

        # With lower casing
        __a     				=  self.get_tokenizer(do_lower_case=lowerCamelCase		)
        __a     				=  self.get_rust_tokenizer(do_lower_case=lowerCamelCase		)

        __a     				=  "UNwant\u00E9d,running"

        __a     				=  tokenizer.tokenize(lowerCamelCase		)
        __a     				=  rust_tokenizer.tokenize(lowerCamelCase		)
        self.assertListEqual(lowerCamelCase  ,						lowerCamelCase		)

        __a     				=  tokenizer.encode(lowerCamelCase  ,						add_special_tokens=lowerCamelCase		)
        __a     				=  rust_tokenizer.encode(lowerCamelCase  ,						add_special_tokens=lowerCamelCase		)
        self.assertListEqual(lowerCamelCase  ,						lowerCamelCase		)

        __a     				=  self.get_rust_tokenizer()
        __a     				=  tokenizer.encode(lowerCamelCase		)
        __a     				=  rust_tokenizer.encode(lowerCamelCase		)
        self.assertListEqual(lowerCamelCase  ,						lowerCamelCase		)


 def 		a__ (				self		):
        __a     				=  BasicTokenizer()

        self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"		)  ,						["ah", "\u535A", "\u63A8", "zz"]		)


 def 		a__ (				self		):
        __a     				=  BasicTokenizer(do_lower_case=lowerCamelCase		)

        self.assertListEqual(
            tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)  ,						["hello", "!", "how", "are", "you", "?"]		)
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)  ,						["hello"]		)


 def 		a__ (				self		):
        __a     				=  BasicTokenizer(do_lower_case=lowerCamelCase  ,						strip_accents=lowerCamelCase		)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)  ,						["hällo", "!", "how", "are", "you", "?"]		)
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)  ,						["h\u00E9llo"]		)


 def 		a__ (				self		):
        __a     				=  BasicTokenizer(do_lower_case=lowerCamelCase  ,						strip_accents=lowerCamelCase		)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)  ,						["hallo", "!", "how", "are", "you", "?"]		)
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)  ,						["hello"]		)


 def 		a__ (				self		):
        __a     				=  BasicTokenizer(do_lower_case=lowerCamelCase		)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)  ,						["hallo", "!", "how", "are", "you", "?"]		)
        self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)  ,						["hello"]		)


 def 		a__ (				self		):
        __a     				=  BasicTokenizer(do_lower_case=lowerCamelCase		)

        self.assertListEqual(
            tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)  ,						["HeLLo", "!", "how", "Are", "yoU", "?"]		)


 def 		a__ (				self		):
        __a     				=  BasicTokenizer(do_lower_case=lowerCamelCase  ,						strip_accents=lowerCamelCase		)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)  ,						["HäLLo", "!", "how", "Are", "yoU", "?"]		)


 def 		a__ (				self		):
        __a     				=  BasicTokenizer(do_lower_case=lowerCamelCase  ,						strip_accents=lowerCamelCase		)

        self.assertListEqual(
            tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)  ,						["HaLLo", "!", "how", "Are", "yoU", "?"]		)


 def 		a__ (				self		):
        __a     				=  BasicTokenizer(do_lower_case=lowerCamelCase  ,						never_split=["[UNK]"]		)

        self.assertListEqual(
            tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"		)  ,						["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]		)


 def 		a__ (				self		):
        __a     				=  BasicTokenizer()
        __a     				=  "a\n'll !!to?'d of, can't."
        __a     				=  ["a", "'", "ll", "!", "!", "to", "?", "'", "d", "of", ",", "can", "'", "t", "."]
        self.assertListEqual(tokenizer.tokenize(lowerCamelCase		)  ,						lowerCamelCase		)


 def 		a__ (				self		):
        __a     				=  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

        __a     				=  {}
        for i, token in enumerate(lowerCamelCase		):
               __a     				=  i
        __a     				=  WordpieceTokenizer(vocab=lowerCamelCase  ,						unk_token="[UNK]"		)

        self.assertListEqual(tokenizer.tokenize(""		)  ,						[]		)

        self.assertListEqual(tokenizer.tokenize("unwanted running"		)  ,						["un", "##want", "##ed", "runn", "##ing"]		)

        self.assertListEqual(tokenizer.tokenize("unwantedX running"		)  ,						["[UNK]", "runn", "##ing"]		)


 def 		a__ (				self		):
        self.assertTrue(_is_whitespace(" "		)		)
        self.assertTrue(_is_whitespace("\t"		)		)
        self.assertTrue(_is_whitespace("\r"		)		)
        self.assertTrue(_is_whitespace("\n"		)		)
        self.assertTrue(_is_whitespace("\u00A0"		)		)

        self.assertFalse(_is_whitespace("A"		)		)
        self.assertFalse(_is_whitespace("-"		)		)


 def 		a__ (				self		):
        self.assertTrue(_is_control("\u0005"		)		)

        self.assertFalse(_is_control("A"		)		)
        self.assertFalse(_is_control(" "		)		)
        self.assertFalse(_is_control("\t"		)		)
        self.assertFalse(_is_control("\r"		)		)


 def 		a__ (				self		):
        self.assertTrue(_is_punctuation("-"		)		)
        self.assertTrue(_is_punctuation("$"		)		)
        self.assertTrue(_is_punctuation("`"		)		)
        self.assertTrue(_is_punctuation("."		)		)

        self.assertFalse(_is_punctuation("A"		)		)
        self.assertFalse(_is_punctuation(" "		)		)


 def 		a__ (				self		):
        __a     				=  self.get_tokenizer()
        __a     				=  self.get_rust_tokenizer()

        # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
        self.assertListEqual([tokenizer.tokenize(lowerCamelCase		) for t in ["Test", "\xad", "test"]]  ,						[["[UNK]"], [], ["[UNK]"]]		)

        self.assertListEqual(
            [rust_tokenizer.tokenize(lowerCamelCase		) for t in ["Test", "\xad", "test"]]  ,						[["[UNK]"], [], ["[UNK]"]]		)


 @slow
 def 		a__ (				self		):
        __a     				=  self.tokenizer_class.from_pretrained("bert-base-uncased"		)

        __a     				=  tokenizer.encode("sequence builders"  ,						add_special_tokens=lowerCamelCase		)
        __a     				=  tokenizer.encode("multi-sequence build"  ,						add_special_tokens=lowerCamelCase		)

        __a     				=  tokenizer.build_inputs_with_special_tokens(lowerCamelCase		)
        __a     				=  tokenizer.build_inputs_with_special_tokens(lowerCamelCase  ,						lowerCamelCase		)

        assert encoded_sentence == [101] + text + [102]
        assert encoded_pair == [101] + text + [102] + text_a + [102]


 def 		a__ (				self		):
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                      __a     				=  self.rust_tokenizer_class.from_pretrained(lowerCamelCase  ,						**lowerCamelCase		)

                      __a     				=  F"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                      __a     				=  tokenizer_r.encode_plus(
                          lowerCamelCase  ,						return_attention_mask=lowerCamelCase  ,						return_token_type_ids=lowerCamelCase  ,						return_offsets_mapping=lowerCamelCase  ,						add_special_tokens=lowerCamelCase  ,						)

                      __a     				=  tokenizer_r.do_lower_case if hasattr(lowerCamelCase  ,						"do_lower_case"		) else False
                      __a     				=  (
                          [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), "A"),
                              ((1, 2), ","),
                              ((3, 5), "na"),
                              ((5, 6), "##ï"),
                              ((6, 8), "##ve"),
                              ((9, 15), tokenizer_r.mask_token),
                              ((16, 21), "Allen"),
                              ((21, 23), "##NL"),
                              ((23, 24), "##P"),
                              ((25, 33), "sentence"),
                              ((33, 34), "."),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                          if not do_lower_case
                          else [
                              ((0, 0), tokenizer_r.cls_token),
                              ((0, 1), "a"),
                              ((1, 2), ","),
                              ((3, 8), "naive"),
                              ((9, 15), tokenizer_r.mask_token),
                              ((16, 21), "allen"),
                              ((21, 23), "##nl"),
                              ((23, 24), "##p"),
                              ((25, 33), "sentence"),
                              ((33, 34), "."),
                              ((0, 0), tokenizer_r.sep_token),
                          ]
                      )

                      self.assertEqual(
                          [e[1] for e in expected_results]  ,						tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]		)		)
                      self.assertEqual([e[0] for e in expected_results]  ,						tokens["offset_mapping"]		)






 def 		a__ (				self		):
        __a     				=  ["的", "人", "有"]
        __a     				=  "".join(lowerCamelCase		)
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                      __a     				=  True
                      __a     				=  self.tokenizer_class.from_pretrained(lowerCamelCase  ,						**lowerCamelCase		)
                      __a     				=  self.rust_tokenizer_class.from_pretrained(lowerCamelCase  ,						**lowerCamelCase		)

                      __a     				=  tokenizer_p.encode(lowerCamelCase  ,						add_special_tokens=lowerCamelCase		)
                      __a     				=  tokenizer_r.encode(lowerCamelCase  ,						add_special_tokens=lowerCamelCase		)

                      __a     				=  tokenizer_r.convert_ids_to_tokens(lowerCamelCase		)
                      __a     				=  tokenizer_p.convert_ids_to_tokens(lowerCamelCase		)

                      # it is expected that each Chinese character is not preceded by "##"
                      self.assertListEqual(lowerCamelCase  ,						lowerCamelCase		)
                      self.assertListEqual(lowerCamelCase  ,						lowerCamelCase		)

                      __a     				=  False
                      __a     				=  self.rust_tokenizer_class.from_pretrained(lowerCamelCase  ,						**lowerCamelCase		)
                      __a     				=  self.tokenizer_class.from_pretrained(lowerCamelCase  ,						**lowerCamelCase		)

                      __a     				=  tokenizer_r.encode(lowerCamelCase  ,						add_special_tokens=lowerCamelCase		)
                      __a     				=  tokenizer_p.encode(lowerCamelCase  ,						add_special_tokens=lowerCamelCase		)

                      __a     				=  tokenizer_r.convert_ids_to_tokens(lowerCamelCase		)
                      __a     				=  tokenizer_p.convert_ids_to_tokens(lowerCamelCase		)

                      # it is expected that only the first Chinese character is not preceded by "##".
                      __a     				=  [
                          F"##{token}" if idx != 0 else token for idx, token in enumerate(lowerCamelCase		)
                      ]
                      self.assertListEqual(lowerCamelCase  ,						lowerCamelCase		)
                      self.assertListEqual(lowerCamelCase  ,						lowerCamelCase		)

"""simple docstring"""



import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
   import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__:Tuple			=	logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__:List[Any]			=	{"""vocab_file""": """vocab.txt"""}

SCREAMING_SNAKE_CASE__:Optional[int]			=	{
    """vocab_file""": {
        """openbmb/cpm-ant-10b""": """https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt""",
    },
}

SCREAMING_SNAKE_CASE__:Tuple			=	{
    """openbmb/cpm-ant-10b""": 1024,
}



def 					_lowerCamelCase(					a							):
       __a     				=  collections.OrderedDict()
       with open(a ,  "r" ,  encoding="utf-8"							) as reader:
              __a     				=  reader.readlines()
       for index, token in enumerate(a							):
              __a     				=  token.rstrip("\n"							)
              __a     				=  index
       return vocab






class 				snake_case__     (   snake_case_   ):


 def __init__(				self  ,						lowerCamelCase  ,						lowerCamelCase="<unk>"  ,						lowerCamelCase=200		):
        __a     				=  vocab
        __a     				=  unk_token
        __a     				=  max_input_chars_per_word






 def 		a__ (				self  ,						lowerCamelCase		):
        __a     				=  list(lowerCamelCase		)
        if len(lowerCamelCase		) > self.max_input_chars_per_word:
               return [self.unk_token]

        __a     				=  0
        __a     				=  []
        while start < len(lowerCamelCase		):
               __a     				=  len(lowerCamelCase		)
               __a     				=  None
               while start < end:
                      __a     				=  "".join(chars[start:end]		)
                      if substr in self.vocab:
                             __a     				=  substr
                             break
                      end -= 1
               if cur_substr is None:
                      sub_tokens.append(self.unk_token		)
                      start += 1
               else:
                      sub_tokens.append(lowerCamelCase		)
                      __a     				=  end

        return sub_tokens






class 				snake_case__     (   snake_case_   ):
 _snake_case   :			Optional[int]           =		VOCAB_FILES_NAMES
 _snake_case   :			Optional[int]           =		PRETRAINED_VOCAB_FILES_MAP
 _snake_case   :			Optional[Any]           =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _snake_case   :			int           =		["""input_ids""", """attention_mask"""]
 _snake_case   :			int           =		False


 def __init__(				self  ,						lowerCamelCase  ,						lowerCamelCase="<d>"  ,						lowerCamelCase="</d>"  ,						lowerCamelCase="<s>"  ,						lowerCamelCase="</s>"  ,						lowerCamelCase="<pad>"  ,						lowerCamelCase="<unk>"  ,						lowerCamelCase="</n>"  ,						lowerCamelCase="</_>"  ,						lowerCamelCase="left"  ,						**lowerCamelCase  ,						):
        requires_backends(self  ,						["jieba"]		)
        super().__init__(
            bod_token=lowerCamelCase  ,						eod_token=lowerCamelCase  ,						bos_token=lowerCamelCase  ,						eos_token=lowerCamelCase  ,						pad_token=lowerCamelCase  ,						unk_token=lowerCamelCase  ,						line_token=lowerCamelCase  ,						space_token=lowerCamelCase  ,						padding_side=lowerCamelCase  ,						**lowerCamelCase  ,						)
        __a     				=  bod_token
        __a     				=  eod_token
        __a     				=  load_vocab(lowerCamelCase		)
        __a     				=  self.encoder[space_token]
        __a     				=  self.encoder[line_token]

        del self.encoder[space_token]
        del self.encoder[line_token]

        __a     				=  collections.OrderedDict(sorted(self.encoder.items()  ,						key=lambda lowerCamelCase		: x[1]		)		)
        __a     				=  {v: k for k, v in self.encoder.items()}

        __a     				=  WordpieceTokenizer(vocab=self.encoder  ,						unk_token=self.unk_token		)


 @property
 def 		a__ (				self		):
        return self.encoder[self.bod_token]


 @property
 def 		a__ (				self		):
        return self.encoder[self.eod_token]


 @property
 def 		a__ (				self		):
        return self.encoder["\n"]


 @property
 def 		a__ (				self		):
        return len(self.encoder		)


 def 		a__ (				self		):
        return dict(self.encoder  ,						**self.added_tokens_encoder		)


 def 		a__ (				self  ,						lowerCamelCase		):
        __a     				=  []
        for x in jieba.cut(lowerCamelCase  ,						cut_all=lowerCamelCase		):
               output_tokens.extend(self.wordpiece_tokenizer.tokenize(lowerCamelCase		)		)
        return output_tokens


 def 		a__ (				self  ,						lowerCamelCase  ,						**lowerCamelCase		):
        __a     				=  [i for i in token_ids if i >= 0]
        __a     				=  [
            x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
        ]
        return super()._decode(lowerCamelCase  ,						**lowerCamelCase		)


 def 		a__ (				self  ,						lowerCamelCase		):
        return token in self.encoder


 def 		a__ (				self  ,						lowerCamelCase		):
        return "".join(lowerCamelCase		)


 def 		a__ (				self  ,						lowerCamelCase		):
        return self.encoder.get(lowerCamelCase  ,						self.encoder.get(self.unk_token		)		)


 def 		a__ (				self  ,						lowerCamelCase		):
        return self.decoder.get(lowerCamelCase  ,						self.unk_token		)


 def 		a__ (				self  ,						lowerCamelCase  ,						lowerCamelCase = None		):
        if os.path.isdir(lowerCamelCase		):
               __a     				=  os.path.join(
                   lowerCamelCase  ,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
        else:
               __a     				=  (filename_prefix + "-" if filename_prefix else "") + save_directory
        __a     				=  0
        if " " in self.encoder:
               __a     				=  self.encoder[" "]
               del self.encoder[" "]
        if "\n" in self.encoder:
               __a     				=  self.encoder["\n"]
               del self.encoder["\n"]
        __a     				=  collections.OrderedDict(sorted(self.encoder.items()  ,						key=lambda lowerCamelCase		: x[1]		)		)
        with open(lowerCamelCase  ,						"w"  ,						encoding="utf-8"		) as writer:
               for token, token_index in self.encoder.items():
                      if index != token_index:
                             logger.warning(
                                 F"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
                                 " Please check that the vocabulary is not corrupted!"		)
                             __a     				=  token_index
                      writer.write(token + "\n"		)
                      index += 1
        return (vocab_file,)


 def 		a__ (				self  ,						lowerCamelCase  ,						lowerCamelCase = None		):
        if token_ids_a is None:
               return [self.bos_token_id] + token_ids_a
        return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a






 def 		a__ (				self  ,						lowerCamelCase  ,						lowerCamelCase = None  ,						lowerCamelCase = False		):

        if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=lowerCamelCase  ,						token_ids_a=lowerCamelCase  ,						already_has_special_tokens=lowerCamelCase		)

        if token_ids_a is not None:
               return [1] + ([0] * len(lowerCamelCase		)) + [1] + ([0] * len(lowerCamelCase		))
        return [1] + ([0] * len(lowerCamelCase		))
