


'''simple docstring'''


import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
      import torch


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation


_UpperCamelCase 							=  logging.get_logger(__name__)
_UpperCamelCase 							=  {"""vocab_file""": """spiece.model"""}

_UpperCamelCase 							=  {
    """vocab_file""": {
        """AI-Sweden/gpt-sw3-126m""": """https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-350m""": """https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-1.6b""": """https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-6.7b""": """https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-20b""": """https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model""",
    }
}

_UpperCamelCase 							=  {
    """AI-Sweden/gpt-sw3-126m""": 2048,
    """AI-Sweden/gpt-sw3-350m""": 2048,
    """AI-Sweden/gpt-sw3-1.6b""": 2048,
    """AI-Sweden/gpt-sw3-6.7b""": 2048,
    """AI-Sweden/gpt-sw3-20b""": 2048,
}







class 							__magic_name__	(       lowerCAmelCase ):

     """simple docstring"""






     lowerCamelCase__       						=				VOCAB_FILES_NAMES
     lowerCamelCase__       						=				PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase__       						=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase__       						=				['input_ids', 'attention_mask']


     def __init__(   self			,     lowerCamelCase			,     lowerCamelCase=False			,     lowerCamelCase=False			,     lowerCamelCase=False			,     lowerCamelCase=None			,     lowerCamelCase=None			,     lowerCamelCase=None			,     lowerCamelCase=None			,     lowerCamelCase = None			,     **lowerCamelCase			,     ):


          '''simple docstring'''





          __A	:       Optional[int]      =							{} if sp_model_kwargs is None else sp_model_kwargs

          __A	:       Optional[int]      =							kwargs.get("name_or_path"		)
          if name_or_path is None:
               logger.warning(
                   "name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"
                   " you are testing the model, this can safely be ignored"		)
               __A	:       Union[str, Any]      =							"None"

          # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
          __A	:       List[Any]      =							"<|endoftext|>" if eos_token is None else eos_token
          __A	:       Tuple      =							"<unk>" if unk_token is None else unk_token
          if "gpt-sw3-7b" in name_or_path:
               __A	:       Optional[int]      =							unk_token if pad_token is None else pad_token
               __A	:       List[Any]      =							eos_token if bos_token is None else bos_token
          else:
               __A	:       Tuple      =							"<pad>" if pad_token is None else pad_token
               __A	:       Optional[Any]      =							"<s>" if bos_token is None else bos_token

          super().__init__(
              do_lower_case=lowerCamelCase			,     remove_space=lowerCamelCase			,     keep_accents=lowerCamelCase			,     bos_token=lowerCamelCase			,     eos_token=lowerCamelCase			,     unk_token=lowerCamelCase			,     pad_token=lowerCamelCase			,     sp_model_kwargs=self.sp_model_kwargs			,     **lowerCamelCase			,     )

          __A	:       str      =							do_lower_case
          __A	:       str      =							remove_space
          __A	:       str      =							keep_accents
          __A	:       Optional[int]      =							vocab_file

          __A	:       List[Any]      =							spm.SentencePieceProcessor(**self.sp_model_kwargs		)
          self.sp_model.Load(lowerCamelCase		)

          # Used for whitespace normalization in input texts
          # fmt : off
          __A	:       str      =							{" ", " ", " ", " ", " ", "　", " ", " ", " ", " ", "￼", ""}
          # fmt : on

          # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
          __A	:       int      =							re.compile(
              f"[{''.join(map(lowerCamelCase			,     list(range(0			,     9		)		) + list(range(11			,     32		)		) + list(range(127			,     160		)		) + [160, 173, 8203]		)		)}]"		)


     def __getstate__(   self		):


          '''simple docstring'''





          __A	:       Optional[Any]      =							self.__dict__.copy()
          __A	:       List[Any]      =							None
          return state


     def __setstate__(   self			,     lowerCamelCase		):


          '''simple docstring'''





          __A	:       List[str]      =							d

          # for backward compatibility
          if not hasattr(self			,     "sp_model_kwargs"		):
               __A	:       Tuple      =							{}

          __A	:       str      =							spm.SentencePieceProcessor(**self.sp_model_kwargs		)
          self.sp_model.Load(self.vocab_file		)


     @property
     # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          return len(self.sp_model		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase		):


          '''simple docstring'''





          __A	:       Any      =							self.non_printing_characters_re.sub(""			,     lowerCamelCase		)

          # Normalize whitespaces
          __A	:       Union[str, Any]      =							"".join([char if char not in self.whitespaces else " " for char in text]		)

          # NFC Unicode normalization
          __A	:       Optional[int]      =							unicodedata.normalize("NFC"			,     lowerCamelCase		)
          return text


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     **lowerCamelCase		):


          '''simple docstring'''





          __A	:       List[Any]      =							self.preprocess_text(lowerCamelCase		)
          return self.sp_model.encode(lowerCamelCase			,     out_type=lowerCamelCase		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase		):


          '''simple docstring'''





          return self.sp_model.PieceToId(lowerCamelCase		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase		):


          '''simple docstring'''





          return self.sp_model.IdToPiece(lowerCamelCase		)


     @staticmethod
     def lowerCAmelCase__  (   lowerCamelCase		):


          '''simple docstring'''





          return out_string


     def lowerCAmelCase__  (   self			,     lowerCamelCase		):


          '''simple docstring'''





          __A	:       Optional[int]      =							[]
          __A	:       int      =							""
          __A	:       Optional[int]      =							False
          for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                    # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                    if not prev_is_special:
                         out_string += " "

                    out_string += self.sp_model.decode(lowerCamelCase		) + token
                    __A	:       Optional[Any]      =							True
                    __A	:       str      =							[]
               else:
                    current_sub_tokens.append(lowerCamelCase		)
                    __A	:       Union[str, Any]      =							False
          out_string += self.sp_model.decode(lowerCamelCase		)

          return out_string


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Dict      =							{self.convert_ids_to_tokens(lowerCamelCase		): i for i in range(self.vocab_size		)}
          vocab.update(self.added_tokens_encoder		)
          return vocab


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase = None		):


          '''simple docstring'''





          if not os.path.isdir(lowerCamelCase		):
               logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
               return
          __A	:       Union[str, Any]      =							os.path.join(
              lowerCamelCase			,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

          if os.path.abspath(self.vocab_file		) != os.path.abspath(lowerCamelCase		) and os.path.isfile(self.vocab_file		):
               copyfile(self.vocab_file			,     lowerCamelCase		)
          elif not os.path.isfile(self.vocab_file		):
               with open(lowerCamelCase			,     "wb"		) as fi:
                    __A	:       List[str]      =							self.sp_model.serialized_model_proto()
                    fi.write(lowerCamelCase		)

          return (out_vocab_file,)


     def lowerCAmelCase__  (   self			,     lowerCamelCase			,     lowerCamelCase = False		):


          '''simple docstring'''





          if isinstance(lowerCamelCase			,     lowerCamelCase		):
               __A	:       Optional[Any]      =							self.preprocess_text(lowerCamelCase		)
               __A	:       Tuple      =							self.sp_model.encode(lowerCamelCase		)
          else:
               __A	:       Any      =							[self.preprocess_text(lowerCamelCase		) for t in text]
               __A	:       List[Any]      =							self.sp_model.encode(lowerCamelCase		)

          if return_tensors is True or return_tensors == "pt":
               __A	:       Dict      =							torch.tensor(lowerCamelCase		)

          return token_ids


     def lowerCAmelCase__  (   self			,     lowerCamelCase		):


          '''simple docstring'''





          return self.sp_model.decode(lowerCamelCase		)


     def lowerCAmelCase__  (   self			,     lowerCamelCase		):


          '''simple docstring'''





          __A	:       Dict      =							[f"User: {text}" if is_user else f"Bot: {text}" for is_user, text in conversation.iter_texts()]
          __A	:       int      =							(
              f"{self.eos_token}{self.bos_token}" + f"{self.bos_token}".join(lowerCamelCase		) + f"{self.bos_token}Bot:"
          )
          return self.encode(text=lowerCamelCase		)





'''simple docstring'''


import unittest

from transformers import CamembertTokenizer, CamembertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.utils import is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCamelCase 							=  get_tests_dir("""fixtures/test_sentencepiece.model""")
_UpperCamelCase 							=  get_tests_dir("""fixtures/test_sentencepiece_bpe.model""")

_UpperCamelCase 							=  """pt""" if is_torch_available() else """tf"""







@require_sentencepiece
@require_tokenizers
class 							__magic_name__	(       lowerCAmelCase    ,		unittest.TestCase ):

     """simple docstring"""






     lowerCamelCase__       						=				CamembertTokenizer
     lowerCamelCase__       						=				CamembertTokenizerFast
     lowerCamelCase__       						=				True
     lowerCamelCase__       						=				True


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          super().setUp()

          # We have a SentencePiece fixture for testing
          __A	:       List[Any]      =							CamembertTokenizer(lowerCamelCase		)
          tokenizer.save_pretrained(self.tmpdirname		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Optional[Any]      =							"<pad>"
          __A	:       Union[str, Any]      =							1

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase		)			,     lowerCamelCase		)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase		)			,     lowerCamelCase		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Union[str, Any]      =							list(self.get_tokenizer().get_vocab().keys()		)

          self.assertEqual(vocab_keys[0]			,     "<s>NOTUSED"		)
          self.assertEqual(vocab_keys[1]			,     "<pad>"		)
          self.assertEqual(vocab_keys[-1]			,     "<mask>"		)
          self.assertEqual(len(lowerCamelCase		)			,     1004		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          self.assertEqual(self.get_tokenizer().vocab_size			,     1005		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Optional[Any]      =							CamembertTokenizer(lowerCamelCase		)
          tokenizer.save_pretrained(self.tmpdirname		)
          __A	:       Optional[int]      =							CamembertTokenizerFast.from_pretrained(self.tmpdirname		)

          __A	:       str      =							"I was born in 92000, and this is falsé."

          __A	:       List[Any]      =							tokenizer.encode(lowerCamelCase		)
          __A	:       Tuple      =							rust_tokenizer.encode(lowerCamelCase		)
          self.assertListEqual(lowerCamelCase			,     lowerCamelCase		)

          __A	:       Dict      =							tokenizer.encode(lowerCamelCase			,     add_special_tokens=lowerCamelCase		)
          __A	:       str      =							rust_tokenizer.encode(lowerCamelCase			,     add_special_tokens=lowerCamelCase		)
          self.assertListEqual(lowerCamelCase			,     lowerCamelCase		)

          # <unk> tokens are not the same for `rust` than for `slow`.
          # Because spm gives back raw token instead of `unk` in EncodeAsPieces
          # tokens = tokenizer.tokenize(sequence)
          __A	:       int      =							tokenizer.convert_ids_to_tokens(lowerCamelCase		)
          __A	:       Optional[int]      =							rust_tokenizer.tokenize(lowerCamelCase		)
          self.assertListEqual(lowerCamelCase			,     lowerCamelCase		)


     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          if not self.test_rust_tokenizer:
               return

          __A	:       Tuple      =							self.get_tokenizer()
          __A	:       List[Any]      =							self.get_rust_tokenizer()

          __A	:       Any      =							"I was born in 92000, and this is falsé."

          __A	:       int      =							tokenizer.tokenize(lowerCamelCase		)
          __A	:       Any      =							rust_tokenizer.tokenize(lowerCamelCase		)
          self.assertListEqual(lowerCamelCase			,     lowerCamelCase		)

          __A	:       Any      =							tokenizer.encode(lowerCamelCase			,     add_special_tokens=lowerCamelCase		)
          __A	:       Optional[int]      =							rust_tokenizer.encode(lowerCamelCase			,     add_special_tokens=lowerCamelCase		)
          self.assertListEqual(lowerCamelCase			,     lowerCamelCase		)

          __A	:       Union[str, Any]      =							self.get_rust_tokenizer()
          __A	:       Optional[Any]      =							tokenizer.encode(lowerCamelCase		)
          __A	:       Dict      =							rust_tokenizer.encode(lowerCamelCase		)
          self.assertListEqual(lowerCamelCase			,     lowerCamelCase		)


     @slow
     def lowerCAmelCase__  (   self		):


          '''simple docstring'''





          __A	:       Any      =							{"input_ids": [[5, 54, 7196, 297, 30, 23, 776, 18, 11, 3215, 3705, 8252, 22, 3164, 1181, 2116, 29, 16, 813, 25, 791, 3314, 20, 3446, 38, 2_7575, 120, 6, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [5, 468, 17, 11, 9088, 20, 1517, 8, 2_2804, 1_8818, 10, 38, 629, 607, 607, 142, 19, 7196, 867, 56, 1_0326, 24, 2267, 20, 416, 5072, 1_5612, 233, 734, 7, 2399, 27, 16, 3015, 1649, 7, 24, 20, 4338, 2399, 27, 13, 3400, 14, 13, 6189, 8, 930, 9, 6]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
          # fmt: on

          # camembert is a french model. So we also use french texts.
          __A	:       int      =							[
              "Le transformeur est un modèle d'apprentissage profond introduit en 2017, "
              "utilisé principalement dans le domaine du traitement automatique des langues (TAL).",
              "À l'instar des réseaux de neurones récurrents (RNN), les transformeurs sont conçus "
              "pour gérer des données séquentielles, telles que le langage naturel, pour des tâches "
              "telles que la traduction et la synthèse de texte.",
          ]

          self.tokenizer_integration_test_util(
              expected_encoding=lowerCamelCase			,     model_name="camembert-base"			,     revision="3a0641d9a1aeb7e848a74299e7e4c4bca216b4cf"			,     sequences=lowerCamelCase			,     )

