


import json
import os
import unittest

from transformers import MgpstrTokenizer
from transformers.models.mgp_str.tokenization_mgp_str import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 							lowercase  (						_UpperCAmelCase    ,       unittest.TestCase			):
     lowerCamelCase			:			Union[str, Any]      = MgpstrTokenizer
     lowerCamelCase			:			Tuple      = False
     lowerCamelCase			:			str      = {}
     lowerCamelCase			:			Dict      = False







     def 							lowercase__       (						self :						Dict      ):
           super().setUp()

           # fmt: off
           SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		['''[GO]''', '''[s]''', '''0''', '''1''', '''2''', '''3''', '''4''', '''5''', '''6''', '''7''', '''8''', '''9''', '''a''', '''b''', '''c''', '''d''', '''e''', '''f''', '''g''', '''h''', '''i''', '''j''', '''k''', '''l''', '''m''', '''n''', '''o''', '''p''', '''q''', '''r''', '''s''', '''t''', '''u''', '''v''', '''w''', '''x''', '''y''', '''z''']
           # fmt: on
           SCREAMING_SNAKE_CASE__       :  str						     =		dict(zip(_lowercase   ,   range(len(_lowercase      )      )      )      )

           SCREAMING_SNAKE_CASE__       :  List[str]						     =		os.path.join(self.tmpdirname   ,   VOCAB_FILES_NAMES['''vocab_file''']      )
           with open(self.vocab_file   ,   '''w'''   ,   encoding='''utf-8'''      ) as fp:
                 fp.write(json.dumps(_lowercase      ) + '''\n'''      )







     def 							lowercase__       (						self :						Any   ,   **_lowercase :						List[Any]      ):
           return MgpstrTokenizer.from_pretrained(self.tmpdirname   ,   **_lowercase      )







     def 							lowercase__       (						self :						int   ,   _lowercase :						List[str]      ):
           SCREAMING_SNAKE_CASE__       :  Tuple						     =		'''tester'''
           SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		'''tester'''
           return input_text, output_text







     @unittest.skip('''MGP-STR always lower cases letters.'''      )
     def 							lowercase__       (						self :						Dict      ):
           pass







     def 							lowercase__       (						self :						str      ):
           SCREAMING_SNAKE_CASE__       :  Dict						     =		self.get_tokenizers(do_lower_case=_lowercase      )
           for tokenizer in tokenizers:
                 with self.subTest(f"""{tokenizer.__class__.__name__}"""      ):
                       SCREAMING_SNAKE_CASE__       :  List[str]						     =		'''[SPECIAL_TOKEN]'''

                       tokenizer.add_special_tokens({'''cls_token''': special_token}      )
                       SCREAMING_SNAKE_CASE__       :  List[Any]						     =		tokenizer.encode([special_token]   ,   add_special_tokens=_lowercase      )
                       self.assertEqual(len(_lowercase      )   ,   1      )

                       SCREAMING_SNAKE_CASE__       :  Optional[int]						     =		tokenizer.decode(_lowercase   ,   skip_special_tokens=_lowercase      )
                       self.assertTrue(special_token not in decoded      )







     def 							lowercase__       (						self :						Dict      ):
           SCREAMING_SNAKE_CASE__       :  List[Any]						     =		self.get_tokenizers()
           for tokenizer in tokenizers:
                 with self.subTest(f"""{tokenizer.__class__.__name__}"""      ):
                       SCREAMING_SNAKE_CASE__     ,       SCREAMING_SNAKE_CASE__       :  Union[str, Any]						     =		self.get_input_output_texts(_lowercase      )

                       SCREAMING_SNAKE_CASE__       :  Any						     =		tokenizer.tokenize(_lowercase      )
                       SCREAMING_SNAKE_CASE__       :  List[Any]						     =		tokenizer.convert_tokens_to_ids(_lowercase      )
                       SCREAMING_SNAKE_CASE__       :  str						     =		tokenizer.encode(_lowercase   ,   add_special_tokens=_lowercase      )
                       self.assertListEqual(_lowercase   ,   _lowercase      )

                       SCREAMING_SNAKE_CASE__       :  int						     =		tokenizer.convert_ids_to_tokens(_lowercase      )
                       self.assertNotEqual(len(_lowercase      )   ,   0      )
                       SCREAMING_SNAKE_CASE__       :  str						     =		tokenizer.decode(_lowercase      )
                       self.assertIsInstance(_lowercase   ,   _lowercase      )

                       self.assertEqual(text_a.replace(''' '''   ,   ''''''      )   ,   _lowercase      )







     @unittest.skip('''MGP-STR tokenizer only handles one sequence.'''      )
     def 							lowercase__       (						self :						Optional[int]      ):
           pass







     @unittest.skip('''inputs cannot be pretokenized in MgpstrTokenizer'''      )
     def 							lowercase__       (						self :						int      ):
           pass












import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


SCREAMING_SNAKE_CASE__     :     Dict		    =       logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__     :     int		    =       {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE__     :     Dict		    =       {
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

SCREAMING_SNAKE_CASE__     :     List[Any]		    =       {
    "bert-base-uncased": 5_12,
    "bert-large-uncased": 5_12,
    "bert-base-cased": 5_12,
    "bert-large-cased": 5_12,
    "bert-base-multilingual-uncased": 5_12,
    "bert-base-multilingual-cased": 5_12,
    "bert-base-chinese": 5_12,
    "bert-base-german-cased": 5_12,
    "bert-large-uncased-whole-word-masking": 5_12,
    "bert-large-cased-whole-word-masking": 5_12,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 5_12,
    "bert-large-cased-whole-word-masking-finetuned-squad": 5_12,
    "bert-base-cased-finetuned-mrpc": 5_12,
    "bert-base-german-dbmdz-cased": 5_12,
    "bert-base-german-dbmdz-uncased": 5_12,
    "TurkuNLP/bert-base-finnish-cased-v1": 5_12,
    "TurkuNLP/bert-base-finnish-uncased-v1": 5_12,
    "wietsedv/bert-base-dutch-cased": 5_12,
}

SCREAMING_SNAKE_CASE__     :     Union[str, Any]		    =       {
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}



class       a_       (     SCREAMING_SNAKE_CASE__     ):
	A    							= VOCAB_FILES_NAMES
	A    							= PRETRAINED_VOCAB_FILES_MAP
	A    							= PRETRAINED_INIT_CONFIGURATION
	A    							= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A    							= BertTokenizer





	def __init__(							self      ,     SCREAMING_SNAKE_CASE=None      ,     SCREAMING_SNAKE_CASE=None      ,     SCREAMING_SNAKE_CASE=True      ,     SCREAMING_SNAKE_CASE="[UNK]"      ,     SCREAMING_SNAKE_CASE="[SEP]"      ,     SCREAMING_SNAKE_CASE="[PAD]"      ,     SCREAMING_SNAKE_CASE="[CLS]"      ,     SCREAMING_SNAKE_CASE="[MASK]"      ,     SCREAMING_SNAKE_CASE=True      ,     SCREAMING_SNAKE_CASE=None      ,     **SCREAMING_SNAKE_CASE      ,     )  ->					List[str]:




				"""simple docstring"""



				super().__init__(
				    SCREAMING_SNAKE_CASE      ,     tokenizer_file=SCREAMING_SNAKE_CASE      ,     do_lower_case=SCREAMING_SNAKE_CASE      ,     unk_token=SCREAMING_SNAKE_CASE      ,     sep_token=SCREAMING_SNAKE_CASE      ,     pad_token=SCREAMING_SNAKE_CASE      ,     cls_token=SCREAMING_SNAKE_CASE      ,     mask_token=SCREAMING_SNAKE_CASE      ,     tokenize_chinese_chars=SCREAMING_SNAKE_CASE      ,     strip_accents=SCREAMING_SNAKE_CASE      ,     **SCREAMING_SNAKE_CASE      ,     )

				SCREAMING_SNAKE_CASE_								=						json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
				if (
				    normalizer_state.get('lowercase'      ,     SCREAMING_SNAKE_CASE      ) != do_lower_case
				    or normalizer_state.get('strip_accents'      ,     SCREAMING_SNAKE_CASE      ) != strip_accents
				    or normalizer_state.get('handle_chinese_chars'      ,     SCREAMING_SNAKE_CASE      ) != tokenize_chinese_chars
				):
							SCREAMING_SNAKE_CASE_								=						getattr(SCREAMING_SNAKE_CASE      ,     normalizer_state.pop('type'      )      )
							SCREAMING_SNAKE_CASE_								=						do_lower_case
							SCREAMING_SNAKE_CASE_								=						strip_accents
							SCREAMING_SNAKE_CASE_								=						tokenize_chinese_chars
							SCREAMING_SNAKE_CASE_								=						normalizer_class(**SCREAMING_SNAKE_CASE      )

				SCREAMING_SNAKE_CASE_								=						do_lower_case





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE=None      )  ->					Union[str, Any]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE = None      )  ->					List[int]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						[self.sep_token_id]
				SCREAMING_SNAKE_CASE_								=						[self.cls_token_id]
				if token_ids_a is None:
							return len(cls + token_ids_a + sep      ) * [0]
				return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE = None      )  ->					Tuple[str]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						self._tokenizer.model.save(SCREAMING_SNAKE_CASE      ,     name=SCREAMING_SNAKE_CASE      )
				return tuple(SCREAMING_SNAKE_CASE      )
