

"""simple docstring"""






import unittest

from transformers import GPTSwaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCamelCase										=		get_tests_dir("fixtures/test_sentencepiece_with_bytefallback.model")
@require_sentencepiece
@require_tokenizers
class        _lowercase					(     A_   , unittest.TestCase     ):
     _lowerCamelCase     =       GPTSwaTokenizer
     _lowerCamelCase     =       False
     _lowerCamelCase     =       True
     _lowerCamelCase     =       False







     def 	lowerCAmelCase__    (      self       ):
      super().setUp()

      # We have a SentencePiece fixture for testing
      __magic_name__     =      GPTSwaTokenizer(UpperCamelCase_ ,						eos_token='''<unk>''' ,						bos_token='''<unk>''' ,						pad_token='''<unk>'''       )

      tokenizer.save_pretrained(self.tmpdirname       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_       ):
      __magic_name__     =      """This is a test"""
      __magic_name__     =      """This is a test"""
      return input_text, output_text







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      """<s>"""
      __magic_name__     =      1

      self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCamelCase_       ) ,						UpperCamelCase_       )
      self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCamelCase_       ) ,						UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      list(self.get_tokenizer().get_vocab().keys()       )

      self.assertEqual(vocab_keys[0] ,						'''<unk>'''       )
      self.assertEqual(vocab_keys[1] ,						'''<s>'''       )
      self.assertEqual(vocab_keys[-1] ,						'''j'''       )
      self.assertEqual(len(UpperCamelCase_       ) ,						2000       )







     def 	lowerCAmelCase__    (      self       ):
      self.assertEqual(self.get_tokenizer().vocab_size ,						2000       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      GPTSwaTokenizer(UpperCamelCase_       )

      __magic_name__     =      tokenizer.tokenize('''This is a test'''       )
      self.assertListEqual(UpperCamelCase_ ,						['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']       )

      self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_       ) ,						[465, 287, 265, 631, 842]       )

      __magic_name__     =      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''       )
      # fmt: off
      self.assertListEqual(
          UpperCamelCase_ ,						['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.'''] ,						)
      # fmt: on

      __magic_name__     =      tokenizer.convert_tokens_to_ids(UpperCamelCase_       )
      self.assertListEqual(
          UpperCamelCase_ ,						[262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260] ,						)

      __magic_name__     =      tokenizer.convert_ids_to_tokens(UpperCamelCase_       )
      # fmt: off
      self.assertListEqual(
          UpperCamelCase_ ,						['''▁I''', '''▁was''', '''▁bor''', '''n''', '''▁in''', '''▁''', '''<0x39>''', '''2''', '''0''', '''0''', '''0''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁f''', '''al''', '''s''', '''<0xC3>''', '''<0xA9>''', '''.''']       )
      # fmt: on







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      GPTSwaTokenizer(UpperCamelCase_       )
      __magic_name__     =      ["""This is a test""", """I was born in 92000, and this is falsé."""]
      __magic_name__     =      [
          [465, 287, 265, 631, 842],
          [262, 272, 1525, 286, 271, 268, 60, 916, 633, 633, 633, 259, 266, 301, 287, 384, 367, 263, 198, 172, 260],
      ]

      # Test that encode_fast returns the same as tokenize + convert_tokens_to_ids
      for text, expected_ids in zip(UpperCamelCase_ ,						UpperCamelCase_       ):
       self.assertListEqual(tokenizer.encode_fast(UpperCamelCase_       ) ,						UpperCamelCase_       )

      # Test that decode_fast returns the input text
      for text, token_ids in zip(UpperCamelCase_ ,						UpperCamelCase_       ):
       self.assertEqual(tokenizer.decode_fast(UpperCamelCase_       ) ,						UpperCamelCase_       )







     @slow
     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      [
          """<|python|>def fibonacci(n)\n    if n < 0:\n        print('Incorrect input')""",
          """Hey there, how are you doing this fine day?""",
          """This is a text with a trailing spaces followed by a dot     .""",
          """Häj sväjs lillebrör! =)""",
          """Det är inget fel på Mr. Cool""",
      ]

      # fmt: off
      __magic_name__     =      {"""input_ids""": [[6_3423, 5, 6811, 1_4954, 282, 816, 3821, 6_3466, 6_3425, 6_3462, 18, 6_3978, 678, 301, 1320, 6_3423, 6_3455, 6_3458, 18, 6_3982, 4246, 3940, 1901, 4_7789, 5547, 1_8994], [1_9630, 1100, 6_3446, 1342, 633, 544, 4488, 593, 5102, 2416, 6_3495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1652, 428, 268, 1936, 515, 268, 5_8593, 2_2413, 9106, 546, 268, 3_3213, 6_3979, 698, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [5_5130, 6_3450, 924, 6_3449, 2249, 4062, 1558, 318, 6_3504, 2_1498, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [509, 377, 2827, 2559, 332, 6575, 6_3443, 2_6801, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """token_type_ids""": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}
      # fmt: on
      self.tokenizer_integration_test_util(
          expected_encoding=UpperCamelCase_ ,						model_name='''AI-Sweden/gpt-sw3-126m''' ,						sequences=UpperCamelCase_ ,						)




"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


__lowerCamelCase										=		logging.get_logger(__name__)

__lowerCamelCase										=		{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__lowerCamelCase										=		{
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

__lowerCamelCase										=		{
    "bert-base-uncased": 5_12,
    "bert-large-uncased": 5_12,
    "bert-base-cased": 5_12,
    "bert-large-cased": 5_12,
    "bert-base-multilingual-uncased": 5_12,
    "bert-base-multilingual-cased": 5_12,
    "bert-base-chinese": 5_12,
    "bert-base-german-cased": 5_12,
    "bert-large-uncased-whole-word-masking": 5_12,
    "bert-large-cased-whole-word-masking": 5_12,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 5_12,
    "bert-large-cased-whole-word-masking-finetuned-squad": 5_12,
    "bert-base-cased-finetuned-mrpc": 5_12,
    "bert-base-german-dbmdz-cased": 5_12,
    "bert-base-german-dbmdz-uncased": 5_12,
    "TurkuNLP/bert-base-finnish-cased-v1": 5_12,
    "TurkuNLP/bert-base-finnish-uncased-v1": 5_12,
    "wietsedv/bert-base-dutch-cased": 5_12,
}

__lowerCamelCase										=		{
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}
class        _lowercase					(     __UpperCAmelCase     ):
     _lowerCamelCase     =       VOCAB_FILES_NAMES
     _lowerCamelCase     =       PRETRAINED_VOCAB_FILES_MAP
     _lowerCamelCase     =       PRETRAINED_INIT_CONFIGURATION
     _lowerCamelCase     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _lowerCamelCase     =       BertTokenizer







     def __init__(      self ,						UpperCamelCase_=None ,						UpperCamelCase_=None ,						UpperCamelCase_=True ,						UpperCamelCase_="[UNK]" ,						UpperCamelCase_="[SEP]" ,						UpperCamelCase_="[PAD]" ,						UpperCamelCase_="[CLS]" ,						UpperCamelCase_="[MASK]" ,						UpperCamelCase_=True ,						UpperCamelCase_=None ,						**UpperCamelCase_ ,						):
      super().__init__(
          UpperCamelCase_ ,						tokenizer_file=UpperCamelCase_ ,						do_lower_case=UpperCamelCase_ ,						unk_token=UpperCamelCase_ ,						sep_token=UpperCamelCase_ ,						pad_token=UpperCamelCase_ ,						cls_token=UpperCamelCase_ ,						mask_token=UpperCamelCase_ ,						tokenize_chinese_chars=UpperCamelCase_ ,						strip_accents=UpperCamelCase_ ,						**UpperCamelCase_ ,						)

      __magic_name__     =      json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
      if (
          normalizer_state.get('''lowercase''' ,						UpperCamelCase_       ) != do_lower_case
          or normalizer_state.get('''strip_accents''' ,						UpperCamelCase_       ) != strip_accents
          or normalizer_state.get('''handle_chinese_chars''' ,						UpperCamelCase_       ) != tokenize_chinese_chars
      ):
       __magic_name__     =      getattr(UpperCamelCase_ ,						normalizer_state.pop('''type'''       )       )
       __magic_name__     =      do_lower_case
       __magic_name__     =      strip_accents
       __magic_name__     =      tokenize_chinese_chars
       __magic_name__     =      normalizer_class(**UpperCamelCase_       )

      __magic_name__     =      do_lower_case







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_=None       ):
      __magic_name__     =      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

      if token_ids_a:
       output += token_ids_a + [self.sep_token_id]

      return output







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_ = None       ):
      __magic_name__     =      [self.sep_token_id]
      __magic_name__     =      [self.cls_token_id]
      if token_ids_a is None:
       return len(cls + token_ids_a + sep       ) * [0]
      return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_ ,						UpperCamelCase_ = None       ):
      __magic_name__     =      self._tokenizer.model.save(UpperCamelCase_ ,						name=UpperCamelCase_       )
      return tuple(UpperCamelCase_       )

