



"""simple docstring"""




import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class       UpperCAmelCase_    (							_UpperCamelCase			,     unittest.TestCase		):
      __SCREAMING_SNAKE_CASE		:  Dict						=					RoCBertTokenizer
      __SCREAMING_SNAKE_CASE		:  Union[str, Any]						=					None
      __SCREAMING_SNAKE_CASE		:  Tuple						=					False
      __SCREAMING_SNAKE_CASE		:  List[str]						=					True
      __SCREAMING_SNAKE_CASE		:  str						=					filter_non_english



      def 		snake_case_   (	self	:      str		):
           super().setUp()

           _UpperCAmelCase     :      Any														=       ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
           _UpperCAmelCase     :      Any														=       {}
           _UpperCAmelCase     :      Tuple														=       {}
           for i, value in enumerate(A		):
                _UpperCAmelCase     :      Any														=       i
                _UpperCAmelCase     :      List[Any]														=       i
           _UpperCAmelCase     :      str														=       os.path.join(self.tmpdirname	,      VOCAB_FILES_NAMES["vocab_file"]		)
           _UpperCAmelCase     :      Union[str, Any]														=       os.path.join(self.tmpdirname	,      VOCAB_FILES_NAMES["word_shape_file"]		)
           _UpperCAmelCase     :      str														=       os.path.join(self.tmpdirname	,      VOCAB_FILES_NAMES["word_pronunciation_file"]		)
           with open(self.vocab_file	,      "w"	,      encoding="utf-8"		) as vocab_writer:
                vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)
           with open(self.word_shape_file	,      "w"	,      encoding="utf-8"		) as word_shape_writer:
                json.dump(A	,      A	,      ensure_ascii=A		)
           with open(self.word_pronunciation_file	,      "w"	,      encoding="utf-8"		) as word_pronunciation_writer:
                json.dump(A	,      A	,      ensure_ascii=A		)



      def 		snake_case_   (	self	:      List[str]		):
           _UpperCAmelCase     :      Dict														=       self.tokenizer_class(self.vocab_file	,      self.word_shape_file	,      self.word_pronunciation_file		)

           _UpperCAmelCase     :      Any														=       tokenizer.tokenize("你好[SEP]你是谁"		)
           self.assertListEqual(A	,      ["你", "好", "[SEP]", "你", "是", "谁"]		)
           self.assertListEqual(tokenizer.convert_tokens_to_ids(A		)	,      [5, 6, 2, 5, 7, 8]		)
           self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(A		)	,      [5, 6, 2, 5, 7, 8]		)
           self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(A		)	,      [5, 6, 2, 5, 7, 8]		)



      def 		snake_case_   (	self	:      int		):
           _UpperCAmelCase     :      Optional[Any]														=       RoCBertBasicTokenizer()

           self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"		)	,      ["ah", "\u535A", "\u63A8", "zz"]		)



      def 		snake_case_   (	self	:      str		):
           _UpperCAmelCase     :      Optional[int]														=       RoCBertBasicTokenizer(do_lower_case=A		)

           self.assertListEqual(
               tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)	,      ["hello", "!", "how", "are", "you", "?"]		)
           self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)	,      ["hello"]		)



      def 		snake_case_   (	self	:      str		):
           _UpperCAmelCase     :      Dict														=       RoCBertBasicTokenizer(do_lower_case=A	,      strip_accents=A		)

           self.assertListEqual(
               tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)	,      ["hällo", "!", "how", "are", "you", "?"]		)
           self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)	,      ["h\u00E9llo"]		)



      def 		snake_case_   (	self	:      Union[str, Any]		):
           _UpperCAmelCase     :      List[Any]														=       RoCBertBasicTokenizer(do_lower_case=A	,      strip_accents=A		)

           self.assertListEqual(
               tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)	,      ["hallo", "!", "how", "are", "you", "?"]		)
           self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)	,      ["hello"]		)



      def 		snake_case_   (	self	:      int		):
           _UpperCAmelCase     :      Dict														=       RoCBertBasicTokenizer(do_lower_case=A		)

           self.assertListEqual(
               tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)	,      ["hallo", "!", "how", "are", "you", "?"]		)
           self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)	,      ["hello"]		)



      def 		snake_case_   (	self	:      Optional[int]		):
           _UpperCAmelCase     :      List[str]														=       RoCBertBasicTokenizer(do_lower_case=A		)

           self.assertListEqual(
               tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)	,      ["HeLLo", "!", "how", "Are", "yoU", "?"]		)



      def 		snake_case_   (	self	:      Tuple		):
           _UpperCAmelCase     :      Dict														=       RoCBertBasicTokenizer(do_lower_case=A	,      strip_accents=A		)

           self.assertListEqual(
               tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)	,      ["HäLLo", "!", "how", "Are", "yoU", "?"]		)



      def 		snake_case_   (	self	:      List[Any]		):
           _UpperCAmelCase     :      Optional[Any]														=       RoCBertBasicTokenizer(do_lower_case=A	,      strip_accents=A		)

           self.assertListEqual(
               tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)	,      ["HaLLo", "!", "how", "Are", "yoU", "?"]		)



      def 		snake_case_   (	self	:      Dict		):
           _UpperCAmelCase     :      int														=       RoCBertBasicTokenizer(do_lower_case=A	,      never_split=["[UNK]"]		)

           self.assertListEqual(
               tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"		)	,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]		)



      def 		snake_case_   (	self	:      Union[str, Any]		):
           _UpperCAmelCase     :      List[str]														=       ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

           _UpperCAmelCase     :      List[str]														=       {}
           for i, token in enumerate(A		):
                _UpperCAmelCase     :      Dict														=       i
           _UpperCAmelCase     :      int														=       RoCBertWordpieceTokenizer(vocab=A	,      unk_token="[UNK]"		)

           self.assertListEqual(tokenizer.tokenize(""		)	,      []		)

           self.assertListEqual(tokenizer.tokenize("unwanted running"		)	,      ["un", "##want", "##ed", "runn", "##ing"]		)

           self.assertListEqual(tokenizer.tokenize("unwantedX running"		)	,      ["[UNK]", "runn", "##ing"]		)



      def 		snake_case_   (	self	:      Optional[Any]		):
           self.assertTrue(_is_whitespace(" "		)		)
           self.assertTrue(_is_whitespace("\t"		)		)
           self.assertTrue(_is_whitespace("\r"		)		)
           self.assertTrue(_is_whitespace("\n"		)		)
           self.assertTrue(_is_whitespace("\u00A0"		)		)

           self.assertFalse(_is_whitespace("A"		)		)
           self.assertFalse(_is_whitespace("-"		)		)



      def 		snake_case_   (	self	:      str		):
           self.assertTrue(_is_control("\u0005"		)		)

           self.assertFalse(_is_control("A"		)		)
           self.assertFalse(_is_control(" "		)		)
           self.assertFalse(_is_control("\t"		)		)
           self.assertFalse(_is_control("\r"		)		)



      def 		snake_case_   (	self	:      Tuple		):
           self.assertTrue(_is_punctuation("-"		)		)
           self.assertTrue(_is_punctuation("$"		)		)
           self.assertTrue(_is_punctuation("`"		)		)
           self.assertTrue(_is_punctuation("."		)		)

           self.assertFalse(_is_punctuation("A"		)		)
           self.assertFalse(_is_punctuation(" "		)		)



      def 		snake_case_   (	self	:      int		):
           _UpperCAmelCase     :      Optional[Any]														=       self.get_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(A		) for t in ["Test", "\xad", "test"]]	,      [["[UNK]"], [], ["[UNK]"]]		)

           if self.test_rust_tokenizer:
                _UpperCAmelCase     :      int														=       self.get_rust_tokenizer()
                self.assertListEqual(
                    [rust_tokenizer.tokenize(A		) for t in ["Test", "\xad", "test"]]	,      [["[UNK]"], [], ["[UNK]"]]		)



      def 		snake_case_   (	self	:      Optional[int]		):
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'		):
                     _UpperCAmelCase     :      Union[str, Any]														=       self.rust_tokenizer_class.from_pretrained(A	,      **A		)

                     _UpperCAmelCase     :      Any														=       f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
                     _UpperCAmelCase     :      Any														=       tokenizer_r.encode_plus(
                         A	,      return_attention_mask=A	,      return_token_type_ids=A	,      return_offsets_mapping=A	,      add_special_tokens=A	,      )

                     _UpperCAmelCase     :      Optional[int]														=       tokenizer_r.do_lower_case if hasattr(A	,      "do_lower_case"		) else False
                     _UpperCAmelCase     :      Optional[int]														=       (
                         [
                             ((0, 0), tokenizer_r.cls_token),
                             ((0, 1), "A"),
                             ((1, 2), ","),
                             ((3, 5), "na"),
                             ((5, 6), "##ï"),
                             ((6, 8), "##ve"),
                             ((9, 1_5), tokenizer_r.mask_token),
                             ((1_6, 2_1), "Allen"),
                             ((2_1, 2_3), "##NL"),
                             ((2_3, 2_4), "##P"),
                             ((2_5, 3_3), "sentence"),
                             ((3_3, 3_4), "."),
                             ((0, 0), tokenizer_r.sep_token),
                         ]
                         if not do_lower_case
                         else [
                             ((0, 0), tokenizer_r.cls_token),
                             ((0, 1), "a"),
                             ((1, 2), ","),
                             ((3, 8), "naive"),
                             ((9, 1_5), tokenizer_r.mask_token),
                             ((1_6, 2_1), "allen"),
                             ((2_1, 2_3), "##nl"),
                             ((2_3, 2_4), "##p"),
                             ((2_5, 3_3), "sentence"),
                             ((3_3, 3_4), "."),
                             ((0, 0), tokenizer_r.sep_token),
                         ]
                     )

                     self.assertEqual(
                         [e[1] for e in expected_results]	,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]		)		)
                     self.assertEqual([e[0] for e in expected_results]	,      tokens["offset_mapping"]		)



      def 		snake_case_   (	self	:      List[str]		):
           _UpperCAmelCase     :      Tuple														=       ["的", "人", "有"]
           _UpperCAmelCase     :      List[Any]														=       "".join(A		)
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'		):
                     _UpperCAmelCase     :      Union[str, Any]														=       True
                     _UpperCAmelCase     :      Dict														=       self.tokenizer_class.from_pretrained(A	,      **A		)
                     _UpperCAmelCase     :      int														=       self.rust_tokenizer_class.from_pretrained(A	,      **A		)

                     _UpperCAmelCase     :      Any														=       tokenizer_p.encode(A	,      add_special_tokens=A		)
                     _UpperCAmelCase     :      List[str]														=       tokenizer_r.encode(A	,      add_special_tokens=A		)

                     _UpperCAmelCase     :      Optional[Any]														=       tokenizer_r.convert_ids_to_tokens(A		)
                     _UpperCAmelCase     :      Union[str, Any]														=       tokenizer_p.convert_ids_to_tokens(A		)

                     # it is expected that each Chinese character is not preceded by "##"
                     self.assertListEqual(A	,      A		)
                     self.assertListEqual(A	,      A		)

                     _UpperCAmelCase     :      int														=       False
                     _UpperCAmelCase     :      List[Any]														=       self.rust_tokenizer_class.from_pretrained(A	,      **A		)
                     _UpperCAmelCase     :      Any														=       self.tokenizer_class.from_pretrained(A	,      **A		)

                     _UpperCAmelCase     :      str														=       tokenizer_r.encode(A	,      add_special_tokens=A		)
                     _UpperCAmelCase     :      Any														=       tokenizer_p.encode(A	,      add_special_tokens=A		)

                     _UpperCAmelCase     :      int														=       tokenizer_r.convert_ids_to_tokens(A		)
                     _UpperCAmelCase     :      Optional[int]														=       tokenizer_p.convert_ids_to_tokens(A		)

                     # it is expected that only the first Chinese character is not preceded by "##".
                     _UpperCAmelCase     :      int														=       [
                         f'##{token}' if idx != 0 else token for idx, token in enumerate(A		)
                     ]
                     self.assertListEqual(A	,      A		)
                     self.assertListEqual(A	,      A		)



      @slow
      def 		snake_case_   (	self	:      List[str]		):
           _UpperCAmelCase     :      int														=       self.tokenizer_class(self.vocab_file	,      self.word_shape_file	,      self.word_pronunciation_file		)

           _UpperCAmelCase     :      Tuple														=       tokenizer.encode("你好"	,      add_special_tokens=A		)
           _UpperCAmelCase     :      int														=       tokenizer.encode("你是谁"	,      add_special_tokens=A		)

           _UpperCAmelCase     :      Optional[int]														=       tokenizer.build_inputs_with_special_tokens(A		)
           _UpperCAmelCase     :      Union[str, Any]														=       tokenizer.build_inputs_with_special_tokens(A	,      A		)

           assert encoded_sentence == [1] + text + [2]
           assert encoded_pair == [1] + text + [2] + text_a + [2]



      def 		snake_case_   (	self	:      List[Any]		):
           _UpperCAmelCase     :      Any														=       self.get_tokenizers(do_lower_case=A		)
           for tokenizer in tokenizers:
                with self.subTest(f'{tokenizer.__class__.__name__}'		):
                     _UpperCAmelCase     :      str														=       "你好，你是谁"
                     _UpperCAmelCase     :      Optional[int]														=       tokenizer.tokenize(A		)
                     _UpperCAmelCase     :      Tuple														=       tokenizer.convert_tokens_to_ids(A		)
                     _UpperCAmelCase     :      Optional[Any]														=       tokenizer.convert_tokens_to_shape_ids(A		)
                     _UpperCAmelCase     :      int														=       tokenizer.convert_tokens_to_pronunciation_ids(A		)
                     _UpperCAmelCase     :      List[str]														=       tokenizer.prepare_for_model(
                         A	,      A	,      A	,      add_special_tokens=A		)

                     _UpperCAmelCase     :      str														=       tokenizer.encode_plus(A	,      add_special_tokens=A		)

                     self.assertEqual(A	,      A		)





"""simple docstring"""




import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


_lowerCAmelCase							:   Any				=				logging.get_logger(__name__)

_lowerCAmelCase							:   Any				=				{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_lowerCAmelCase							:   List[str]				=				{
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/tokenizer.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/tokenizer.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/tokenizer.json",
    },
}

_lowerCAmelCase							:   List[str]				=				{
    "gpt2": 10_24,
    "gpt2-medium": 10_24,
    "gpt2-large": 10_24,
    "gpt2-xl": 10_24,
    "distilgpt2": 10_24,
}






class       UpperCAmelCase_    (							_UpperCamelCase		):
      __SCREAMING_SNAKE_CASE		:  Optional[Any]						=					VOCAB_FILES_NAMES
      __SCREAMING_SNAKE_CASE		:  int						=					PRETRAINED_VOCAB_FILES_MAP
      __SCREAMING_SNAKE_CASE		:  Optional[int]						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __SCREAMING_SNAKE_CASE		:  Optional[int]						=					['input_ids', 'attention_mask']
      __SCREAMING_SNAKE_CASE		:  Dict						=					GPTaTokenizer



      def __init__(	self	:      Optional[Any]	,      A	:      Optional[int]=None	,      A	:      List[Any]=None	,      A	:      Optional[int]=None	,      A	:      Optional[Any]="<|endoftext|>"	,      A	:      Union[str, Any]="<|endoftext|>"	,      A	:      Optional[Any]="<|endoftext|>"	,      A	:      Optional[Any]=False	,      **A	:      str	,      ):
           super().__init__(
               A	,      A	,      tokenizer_file=A	,      unk_token=A	,      bos_token=A	,      eos_token=A	,      add_prefix_space=A	,      **A	,      )

           _UpperCAmelCase     :      List[Any]														=       kwargs.pop("add_bos_token"	,      A		)

           _UpperCAmelCase     :      List[str]														=       json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
           if pre_tok_state.get("add_prefix_space"	,      A		) != add_prefix_space:
                _UpperCAmelCase     :      Dict														=       getattr(A	,      pre_tok_state.pop("type"		)		)
                _UpperCAmelCase     :      Optional[Any]														=       add_prefix_space
                _UpperCAmelCase     :      Optional[Any]														=       pre_tok_class(**A		)

           _UpperCAmelCase     :      List[Any]														=       add_prefix_space



      def 		snake_case_   (	self	:      List[Any]	,      *A	:      List[Any]	,      **A	:      List[str]		):
           _UpperCAmelCase     :      Union[str, Any]														=       kwargs.get("is_split_into_words"	,      A		)
           assert self.add_prefix_space or not is_split_into_words, (
               f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
               "to use it with pretokenized inputs."
           )

           return super()._batch_encode_plus(*A	,      **A		)



      def 		snake_case_   (	self	:      Optional[Any]	,      *A	:      List[Any]	,      **A	:      int		):
           _UpperCAmelCase     :      Union[str, Any]														=       kwargs.get("is_split_into_words"	,      A		)

           assert self.add_prefix_space or not is_split_into_words, (
               f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
               "to use it with pretokenized inputs."
           )

           return super()._encode_plus(*A	,      **A		)



      def 		snake_case_   (	self	:      Optional[int]	,      A	:      str	,      A	:      Optional[str] = None		):
           _UpperCAmelCase     :      List[Any]														=       self._tokenizer.model.save(A	,      name=A		)
           return tuple(A		)



      def 		snake_case_   (	self	:      int	,      A	:      "Conversation"		):
           _UpperCAmelCase     :      List[str]														=       []
           for is_user, text in conversation.iter_texts():
                input_ids.extend(self.encode(A	,      add_special_tokens=A		) + [self.eos_token_id]		)

           if len(A		) > self.model_max_length:
                _UpperCAmelCase     :      Optional[int]														=       input_ids[-self.model_max_length :]
           return input_ids
