



import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
  a_								=       """pt"""
elif is_tf_available():
  a_								=       """tf"""
else:
  a_								=       """jax"""



class 			__lowerCAmelCase  (						lowerCAmelCase__	,						unittest.TestCase ):
 lowerCAmelCase__							       =	ByTaTokenizer
 lowerCAmelCase__							       =	False




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    super().setUp()
    __lowerCamelCase    				=     ByTaTokenizer()
    tokenizer.save_pretrained(self.tmpdirname					)




 @cached_property
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    return ByTaTokenizer.from_pretrained('''google/byt5-small'''					)




 def 							lowerCamelCase   (    self						,					**__UpperCAmelCase					):



    '''simple docstring'''
    return self.tokenizer_class.from_pretrained(self.tmpdirname						,					**__UpperCAmelCase					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase=False						,					__UpperCAmelCase=20						,					__UpperCAmelCase=5					):



    '''simple docstring'''
    # XXX The default common tokenizer tests assume that every ID is decodable on its own.
    # This assumption is invalid for ByT5 because single bytes might not be
    # valid utf-8 (byte 128 for instance).
    # Here we're overriding the smallest possible method to provide
    # a clean sequence without making the same assumption.

    __lowerCamelCase    				=     []
    for i in range(len(__UpperCAmelCase					)					):
       try:
          __lowerCamelCase    				=     tokenizer.decode([i]						,					clean_up_tokenization_spaces=__UpperCAmelCase					)
       except UnicodeDecodeError:
          pass
       toks.append((i, tok)					)

    __lowerCamelCase    				=     list(filter(lambda __UpperCAmelCase					: re.match(r'''^[ a-zA-Z]+$'''						,					t[1]					)						,					__UpperCAmelCase					)					)
    __lowerCamelCase    				=     list(filter(lambda __UpperCAmelCase					: [t[0]] == tokenizer.encode(t[1]						,					add_special_tokens=__UpperCAmelCase					)						,					__UpperCAmelCase					)					)
    if max_length is not None and len(__UpperCAmelCase					) > max_length:
       __lowerCamelCase    				=     toks[:max_length]
    if min_length is not None and len(__UpperCAmelCase					) < min_length and len(__UpperCAmelCase					) > 0:
       while len(__UpperCAmelCase					) < min_length:
          __lowerCamelCase    				=     toks + toks
        # toks_str = [t[1] for t in toks]
    __lowerCamelCase    				=     [t[0] for t in toks]

    # Ensure consistency
    __lowerCamelCase    				=     tokenizer.decode(__UpperCAmelCase						,					clean_up_tokenization_spaces=__UpperCAmelCase					)
    if " " not in output_txt and len(__UpperCAmelCase					) > 1:
       __lowerCamelCase    				=     (
           tokenizer.decode([toks_ids[0]]						,					clean_up_tokenization_spaces=__UpperCAmelCase					)
           + ''' '''
           + tokenizer.decode(toks_ids[1:]						,					clean_up_tokenization_spaces=__UpperCAmelCase					)
       )
    if with_prefix_space:
       __lowerCamelCase    				=     ''' ''' + output_txt
    __lowerCamelCase    				=     tokenizer.encode(__UpperCAmelCase						,					add_special_tokens=__UpperCAmelCase					)
    return output_txt, output_ids




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.ta_base_tokenizer
    __lowerCamelCase    				=     tokenizer(['''hi</s>''', '''I went to the gym</s>''', '''</s>''']					)
    __lowerCamelCase    				=     tokenizer(['''hi''', '''I went to the gym''', '''''']					)
    self.assertListEqual(batch_with_eos_added['''input_ids''']						,					batch_without_eos_added['''input_ids''']					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.ta_base_tokenizer
    __lowerCamelCase    				=     '''Unicode €.'''
    __lowerCamelCase    				=     tokenizer(__UpperCAmelCase					)
    __lowerCamelCase    				=     [88, 113, 108, 102, 114, 103, 104, 35, 229, 133, 175, 49, 1]
    self.assertEqual(encoded['''input_ids''']						,					__UpperCAmelCase					)

    # decoding
    __lowerCamelCase    				=     tokenizer.decode(__UpperCAmelCase					)
    self.assertEqual(__UpperCAmelCase						,					'''Unicode €.</s>'''					)

    __lowerCamelCase    				=     tokenizer('''e è é ê ë'''					)
    __lowerCamelCase    				=     [104, 35, 198, 171, 35, 198, 172, 35, 198, 173, 35, 198, 174, 1]
    self.assertEqual(encoded['''input_ids''']						,					__UpperCAmelCase					)
    # decoding
    __lowerCamelCase    				=     tokenizer.decode(__UpperCAmelCase					)
    self.assertEqual(__UpperCAmelCase						,					'''e è é ê ë</s>'''					)

    # encode/decode, but with `encode` instead of `__call__`
    self.assertEqual(tokenizer.decode(tokenizer.encode('''e è é ê ë'''					)					)						,					'''e è é ê ë</s>'''					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.ta_base_tokenizer
    __lowerCamelCase    				=     ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
    # fmt: off
    __lowerCamelCase    				=     [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 1, 0]
    # fmt: on
    __lowerCamelCase    				=     tokenizer(__UpperCAmelCase						,					padding=__UpperCAmelCase						,					return_tensors=__UpperCAmelCase					)
    self.assertIsInstance(__UpperCAmelCase						,					__UpperCAmelCase					)

    if FRAMEWORK != "jax":
       __lowerCamelCase    				=     list(batch.input_ids.numpy()[0]					)
    else:
       __lowerCamelCase    				=     list(batch.input_ids.tolist()[0]					)

    self.assertListEqual(__UpperCAmelCase						,					__UpperCAmelCase					)

    self.assertEqual((2, 37)						,					batch.input_ids.shape					)
    self.assertEqual((2, 37)						,					batch.attention_mask.shape					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.ta_base_tokenizer
    __lowerCamelCase    				=     ['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
    __lowerCamelCase    				=     tokenizer(__UpperCAmelCase						,					padding=__UpperCAmelCase						,					return_tensors=__UpperCAmelCase					)
    # check if input_ids are returned and no decoder_input_ids
    self.assertIn('''input_ids'''						,					__UpperCAmelCase					)
    self.assertIn('''attention_mask'''						,					__UpperCAmelCase					)
    self.assertNotIn('''decoder_input_ids'''						,					__UpperCAmelCase					)
    self.assertNotIn('''decoder_attention_mask'''						,					__UpperCAmelCase					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.ta_base_tokenizer
    __lowerCamelCase    				=     [
        '''Summary of the text.''',
        '''Another summary.''',
    ]
    __lowerCamelCase    				=     tokenizer(
        text_target=__UpperCAmelCase						,					max_length=32						,					padding='''max_length'''						,					truncation=__UpperCAmelCase						,					return_tensors=__UpperCAmelCase					)
    self.assertEqual(32						,					targets['''input_ids'''].shape[1]					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.ta_base_tokenizer
    __lowerCamelCase    				=     ['''A long paragraph for summarization. </s>''']
    __lowerCamelCase    				=     ['''Summary of the text. </s>''']
    # fmt: off
    __lowerCamelCase    				=     [68, 35, 111, 114, 113, 106, 35, 115, 100, 117, 100, 106, 117, 100, 115, 107, 35, 105, 114, 117, 35, 118, 120, 112, 112, 100, 117, 108, 125, 100, 119, 108, 114, 113, 49, 35, 1]
    __lowerCamelCase    				=     [86, 120, 112, 112, 100, 117, 124, 35, 114, 105, 35, 119, 107, 104, 35, 119, 104, 123, 119, 49, 35, 1]
    # fmt: on

    __lowerCamelCase    				=     tokenizer(__UpperCAmelCase						,					text_target=__UpperCAmelCase					)

    self.assertEqual(__UpperCAmelCase						,					batch['''input_ids'''][0]					)
    self.assertEqual(__UpperCAmelCase						,					batch['''labels'''][0]					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    # safety check on max_len default value so we are sure the test works
    __lowerCamelCase    				=     self.get_tokenizers()
    for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""					):
          self.assertNotEqual(tokenizer.model_max_length						,					42					)

        # Now let's start the test
    __lowerCamelCase    				=     self.get_tokenizers()
    for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""					):
          # Isolate this from the other tests because we save additional tokens/etc
          __lowerCamelCase    				=     tempfile.mkdtemp()

          __lowerCamelCase    				=     ''' He is very happy, UNwant\u00E9d,running'''
          __lowerCamelCase    				=     tokenizer.encode(__UpperCAmelCase						,					add_special_tokens=__UpperCAmelCase					)
          tokenizer.save_pretrained(__UpperCAmelCase					)

          __lowerCamelCase    				=     tokenizer.__class__.from_pretrained(__UpperCAmelCase					)
          __lowerCamelCase    				=     after_tokenizer.encode(__UpperCAmelCase						,					add_special_tokens=__UpperCAmelCase					)
          self.assertListEqual(__UpperCAmelCase						,					__UpperCAmelCase					)

          shutil.rmtree(__UpperCAmelCase					)

    __lowerCamelCase    				=     self.get_tokenizers(model_max_length=42					)
    for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""					):
          # Isolate this from the other tests because we save additional tokens/etc
          __lowerCamelCase    				=     tempfile.mkdtemp()

          __lowerCamelCase    				=     ''' He is very happy, UNwant\u00E9d,running'''
          tokenizer.add_tokens(['''bim''', '''bambam''']					)
          __lowerCamelCase    				=     tokenizer.additional_special_tokens
          additional_special_tokens.append('''new_additional_special_token'''					)
          tokenizer.add_special_tokens({'''additional_special_tokens''': additional_special_tokens}					)
          __lowerCamelCase    				=     tokenizer.encode(__UpperCAmelCase						,					add_special_tokens=__UpperCAmelCase					)
          tokenizer.save_pretrained(__UpperCAmelCase					)

          __lowerCamelCase    				=     tokenizer.__class__.from_pretrained(__UpperCAmelCase					)
          __lowerCamelCase    				=     after_tokenizer.encode(__UpperCAmelCase						,					add_special_tokens=__UpperCAmelCase					)
          self.assertListEqual(__UpperCAmelCase						,					__UpperCAmelCase					)
          self.assertIn('''new_additional_special_token'''						,					after_tokenizer.additional_special_tokens					)
          self.assertEqual(after_tokenizer.model_max_length						,					42					)

          __lowerCamelCase    				=     tokenizer.__class__.from_pretrained(__UpperCAmelCase						,					model_max_length=43					)
          self.assertEqual(tokenizer.model_max_length						,					43					)

          shutil.rmtree(__UpperCAmelCase					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     []
    if self.test_slow_tokenizer:
       tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())					)

    if self.test_rust_tokenizer:
       tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())					)

    for tokenizer_class, tokenizer_utils in tokenizer_list:
       with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer_utils.save_pretrained(__UpperCAmelCase					)

          with open(os.path.join(__UpperCAmelCase						,					'''special_tokens_map.json'''					)						,					encoding='''utf-8'''					) as json_file:
             __lowerCamelCase    				=     json.load(__UpperCAmelCase					)

          with open(os.path.join(__UpperCAmelCase						,					'''tokenizer_config.json'''					)						,					encoding='''utf-8'''					) as json_file:
             __lowerCamelCase    				=     json.load(__UpperCAmelCase					)

          __lowerCamelCase    				=     [F"""<extra_id_{i}>""" for i in range(125					)]

          __lowerCamelCase    				=     added_tokens_extra_ids + [
              '''an_additional_special_token'''
          ]
          __lowerCamelCase    				=     added_tokens_extra_ids + [
              '''an_additional_special_token'''
          ]

          with open(os.path.join(__UpperCAmelCase						,					'''special_tokens_map.json'''					)						,					'''w'''						,					encoding='''utf-8'''					) as outfile:
             json.dump(__UpperCAmelCase						,					__UpperCAmelCase					)
          with open(os.path.join(__UpperCAmelCase						,					'''tokenizer_config.json'''					)						,					'''w'''						,					encoding='''utf-8'''					) as outfile:
             json.dump(__UpperCAmelCase						,					__UpperCAmelCase					)

          # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
          # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
          # "special_tokens_map.json" files
          __lowerCamelCase    				=     tokenizer_class.from_pretrained(
              __UpperCAmelCase						,					)
          self.assertIn(
              '''an_additional_special_token'''						,					tokenizer_without_change_in_init.additional_special_tokens					)
          # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
          self.assertEqual(
              ['''an_additional_special_token''']						,					tokenizer_without_change_in_init.convert_ids_to_tokens(
                  tokenizer_without_change_in_init.convert_tokens_to_ids(['''an_additional_special_token''']					)					)						,					)

          # Now we test that we can change the value of additional_special_tokens in the from_pretrained
          __lowerCamelCase    				=     added_tokens_extra_ids + [AddedToken('''a_new_additional_special_token'''						,					lstrip=__UpperCAmelCase					)]
          __lowerCamelCase    				=     tokenizer_class.from_pretrained(
              __UpperCAmelCase						,					additional_special_tokens=__UpperCAmelCase						,					)

          self.assertIn('''a_new_additional_special_token'''						,					tokenizer.additional_special_tokens					)
          self.assertEqual(
              ['''a_new_additional_special_token''']						,					tokenizer.convert_ids_to_tokens(
                  tokenizer.convert_tokens_to_ids(['''a_new_additional_special_token''']					)					)						,					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     []
    if self.test_slow_tokenizer:
       tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())					)

    if self.test_rust_tokenizer:
       tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())					)

    for tokenizer_class, tokenizer_utils in tokenizer_list:
       with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer_utils.save_pretrained(__UpperCAmelCase					)

          __lowerCamelCase    				=     tokenizer_class.from_pretrained(__UpperCAmelCase					)

          self.assertTrue(tokenizer.decode([255]					) == ''''''					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    pass




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    pass




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    pass




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    pass




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
    # and special added tokens as tokens
    __lowerCamelCase    				=     self.get_tokenizers(fast=__UpperCAmelCase						,					do_lower_case=__UpperCAmelCase					)
    for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""					):
          __lowerCamelCase    				=     ['''t''', '''h''', '''i''', '''s''', ''' ''', '''i''', '''s''', ''' ''', '''a''', ''' ''', '''t''', '''e''', '''x''', '''t''', '''</s>''']
          __lowerCamelCase    				=     tokenizer.convert_tokens_to_string(__UpperCAmelCase					)

          self.assertIsInstance(__UpperCAmelCase						,					__UpperCAmelCase					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.get_tokenizers()
    for tokenizer in tokenizers:
       with self.subTest(F"""{tokenizer.__class__.__name__}"""					):
          __lowerCamelCase    				=     [
              '''bos_token''',
              '''eos_token''',
              '''unk_token''',
              '''sep_token''',
              '''pad_token''',
              '''cls_token''',
              '''mask_token''',
          ]

          __lowerCamelCase    				=     0
          __lowerCamelCase    				=     tokenizer.convert_ids_to_tokens(
              __UpperCAmelCase						,					skip_special_tokens=__UpperCAmelCase					)

          for attr in attributes_list:
             setattr(__UpperCAmelCase						,					attr + '''_id'''						,					__UpperCAmelCase					)
             self.assertEqual(getattr(__UpperCAmelCase						,					__UpperCAmelCase					)						,					__UpperCAmelCase					)
             self.assertEqual(getattr(__UpperCAmelCase						,					attr + '''_id'''					)						,					__UpperCAmelCase					)

             setattr(__UpperCAmelCase						,					attr + '''_id'''						,					__UpperCAmelCase					)
             self.assertEqual(getattr(__UpperCAmelCase						,					__UpperCAmelCase					)						,					__UpperCAmelCase					)
             self.assertEqual(getattr(__UpperCAmelCase						,					attr + '''_id'''					)						,					__UpperCAmelCase					)

          setattr(__UpperCAmelCase						,					'''additional_special_tokens_ids'''						,					[]					)
          self.assertListEqual(getattr(__UpperCAmelCase						,					'''additional_special_tokens'''					)						,					[]					)
          self.assertListEqual(getattr(__UpperCAmelCase						,					'''additional_special_tokens_ids'''					)						,					[]					)

          setattr(__UpperCAmelCase						,					'''additional_special_tokens_ids'''						,					[token_id_to_test_setters]					)
          self.assertListEqual(getattr(__UpperCAmelCase						,					'''additional_special_tokens'''					)						,					[token_to_test_setters]					)
          self.assertListEqual(getattr(__UpperCAmelCase						,					'''additional_special_tokens_ids'''					)						,					[token_id_to_test_setters]					)











import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
  import torch


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


a_								=       logging.get_logger(__name__)
a_								=       {"""vocab_file""": """spiece.model"""}

a_								=       {
    """vocab_file""": {
        """AI-Sweden/gpt-sw3-126m""": """https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-350m""": """https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-1.6b""": """https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-6.7b""": """https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-20b""": """https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model""",
    }
}

a_								=       {
    """AI-Sweden/gpt-sw3-126m""": 2_048,
    """AI-Sweden/gpt-sw3-350m""": 2_048,
    """AI-Sweden/gpt-sw3-1.6b""": 2_048,
    """AI-Sweden/gpt-sw3-6.7b""": 2_048,
    """AI-Sweden/gpt-sw3-20b""": 2_048,
}



class 			__lowerCAmelCase  (						lowerCAmelCase__ ):
 lowerCAmelCase__							       =	VOCAB_FILES_NAMES
 lowerCAmelCase__							       =	PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__							       =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__							       =	["""input_ids""", """attention_mask"""]




 def __init__(    self						,					__UpperCAmelCase						,					__UpperCAmelCase=False						,					__UpperCAmelCase=False						,					__UpperCAmelCase=False						,					__UpperCAmelCase=None						,					__UpperCAmelCase=None						,					__UpperCAmelCase=None						,					__UpperCAmelCase=None						,					__UpperCAmelCase = None						,					**__UpperCAmelCase						,					):



    '''simple docstring'''
    __lowerCamelCase    				=     {} if sp_model_kwargs is None else sp_model_kwargs

    __lowerCamelCase    				=     kwargs.get('''name_or_path'''					)
    if name_or_path is None:
       logger.warning(
           '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
           ''' you are testing the model, this can safely be ignored'''					)
       __lowerCamelCase    				=     '''None'''

    # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
    __lowerCamelCase    				=     '''<|endoftext|>''' if eos_token is None else eos_token
    __lowerCamelCase    				=     '''<unk>''' if unk_token is None else unk_token
    if "gpt-sw3-7b" in name_or_path:
       __lowerCamelCase    				=     unk_token if pad_token is None else pad_token
       __lowerCamelCase    				=     eos_token if bos_token is None else bos_token
    else:
       __lowerCamelCase    				=     '''<pad>''' if pad_token is None else pad_token
       __lowerCamelCase    				=     '''<s>''' if bos_token is None else bos_token

    super().__init__(
        do_lower_case=__UpperCAmelCase						,					remove_space=__UpperCAmelCase						,					keep_accents=__UpperCAmelCase						,					bos_token=__UpperCAmelCase						,					eos_token=__UpperCAmelCase						,					unk_token=__UpperCAmelCase						,					pad_token=__UpperCAmelCase						,					sp_model_kwargs=self.sp_model_kwargs						,					**__UpperCAmelCase						,					)

    __lowerCamelCase    				=     do_lower_case
    __lowerCamelCase    				=     remove_space
    __lowerCamelCase    				=     keep_accents
    __lowerCamelCase    				=     vocab_file

    __lowerCamelCase    				=     spm.SentencePieceProcessor(**self.sp_model_kwargs					)
    self.sp_model.Load(__UpperCAmelCase					)

    # Used for whitespace normalization in input texts
    # fmt : off
    __lowerCamelCase    				=     {''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
    # fmt : on

    # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
    __lowerCamelCase    				=     re.compile(
        F"""[{"".join(map(__UpperCAmelCase						,					list(range(0						,					9					)					) + list(range(11						,					32					)					) + list(range(127						,					160					)					) + [160, 173, 8203]					)					)}]"""					)




 def __getstate__(    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.__dict__.copy()
    __lowerCamelCase    				=     None
    return state




 def __setstate__(    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    __lowerCamelCase    				=     d

    # for backward compatibility
    if not hasattr(self						,					'''sp_model_kwargs'''					):
       __lowerCamelCase    				=     {}

    __lowerCamelCase    				=     spm.SentencePieceProcessor(**self.sp_model_kwargs					)
    self.sp_model.Load(self.vocab_file					)




 @property
 # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    return len(self.sp_model					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.non_printing_characters_re.sub(''''''						,					__UpperCAmelCase					)

    # Normalize whitespaces
    __lowerCamelCase    				=     ''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]					)

    # NFC Unicode normalization
    __lowerCamelCase    				=     unicodedata.normalize('''NFC'''						,					__UpperCAmelCase					)
    return text




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					**__UpperCAmelCase					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.preprocess_text(__UpperCAmelCase					)
    return self.sp_model.encode(__UpperCAmelCase						,					out_type=__UpperCAmelCase					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    return self.sp_model.PieceToId(__UpperCAmelCase					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    return self.sp_model.IdToPiece(__UpperCAmelCase					)




 @staticmethod
 def 							lowerCamelCase   (    __UpperCAmelCase					):



    '''simple docstring'''
    return out_string




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    __lowerCamelCase    				=     []
    __lowerCamelCase    				=     ''''''
    __lowerCamelCase    				=     False
    for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
          # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
          if not prev_is_special:
             out_string += " "

          out_string += self.sp_model.decode(__UpperCAmelCase					) + token
          __lowerCamelCase    				=     True
          __lowerCamelCase    				=     []
       else:
          current_sub_tokens.append(__UpperCAmelCase					)
          __lowerCamelCase    				=     False
    out_string += self.sp_model.decode(__UpperCAmelCase					)

    return out_string




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     {self.convert_ids_to_tokens(__UpperCAmelCase					): i for i in range(self.vocab_size					)}
    vocab.update(self.added_tokens_encoder					)
    return vocab




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase = None					):



    '''simple docstring'''
    if not os.path.isdir(__UpperCAmelCase					):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""					)
       return
    __lowerCamelCase    				=     os.path.join(
        __UpperCAmelCase						,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

    if os.path.abspath(self.vocab_file					) != os.path.abspath(__UpperCAmelCase					) and os.path.isfile(self.vocab_file					):
       copyfile(self.vocab_file						,					__UpperCAmelCase					)
    elif not os.path.isfile(self.vocab_file					):
       with open(__UpperCAmelCase						,					'''wb'''					) as fi:
          __lowerCamelCase    				=     self.sp_model.serialized_model_proto()
          fi.write(__UpperCAmelCase					)

    return (out_vocab_file,)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase = False					):



    '''simple docstring'''

    if isinstance(__UpperCAmelCase						,					__UpperCAmelCase					):
       __lowerCamelCase    				=     self.preprocess_text(__UpperCAmelCase					)
       __lowerCamelCase    				=     self.sp_model.encode(__UpperCAmelCase					)
    else:
       __lowerCamelCase    				=     [self.preprocess_text(__UpperCAmelCase					) for t in text]
       __lowerCamelCase    				=     self.sp_model.encode(__UpperCAmelCase					)

    if return_tensors is True or return_tensors == "pt":
       __lowerCamelCase    				=     torch.tensor(__UpperCAmelCase					)

    return token_ids




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    return self.sp_model.decode(__UpperCAmelCase					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    __lowerCamelCase    				=     [F"""User: {text}""" if is_user else F"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
    __lowerCamelCase    				=     (
        F"""{self.eos_token}{self.bos_token}""" + F"""{self.bos_token}""".join(__UpperCAmelCase					) + F"""{self.bos_token}Bot:"""
    )
    return self.encode(text=__UpperCAmelCase					)






