





"""simple docstring"""





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
  import torch


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


lowercase__				:Optional[int]				   =       logging.get_logger(__name__)
lowercase__				:Any				   =       {'vocab_file': 'spiece.model'}

lowercase__				:List[Any]				   =       {
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

lowercase__				:int				   =       {
    'AI-Sweden/gpt-sw3-126m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-350m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-1.6b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-6.7b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-20b': 2_0_4_8,
}




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       List[Any]         =       VOCAB_FILES_NAMES
      _A     :       Dict         =       PRETRAINED_VOCAB_FILES_MAP
      _A     :       Optional[Any]         =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _A     :       Dict         =       ['input_ids', 'attention_mask']

      def __init__(							self       :      Optional[int]						,				__lowercase       :      List[str]						,				__lowercase       :      Tuple=False						,				__lowercase       :      List[str]=False						,				__lowercase       :      List[str]=False						,				__lowercase       :      str=None						,				__lowercase       :      str=None						,				__lowercase       :      int=None						,				__lowercase       :      Tuple=None						,				__lowercase       :      Optional[Dict[str, Any]] = None						,				**__lowercase       :      Dict						,				):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						{} if sp_model_kwargs is None else sp_model_kwargs

        __UpperCAmelCase    :						List[Any]       =						kwargs.get('''name_or_path''' )
        if name_or_path is None:
          logger.warning(
              '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
              ''' you are testing the model, this can safely be ignored''' )
          __UpperCAmelCase    :						List[Any]       =						'''None'''

        # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
        __UpperCAmelCase    :						List[str]       =						'''<|endoftext|>''' if eos_token is None else eos_token
        __UpperCAmelCase    :						Any       =						'''<unk>''' if unk_token is None else unk_token
        if "gpt-sw3-7b" in name_or_path:
          __UpperCAmelCase    :						Dict       =						unk_token if pad_token is None else pad_token
          __UpperCAmelCase    :						List[Any]       =						eos_token if bos_token is None else bos_token
        else:
          __UpperCAmelCase    :						List[Any]       =						'''<pad>''' if pad_token is None else pad_token
          __UpperCAmelCase    :						List[str]       =						'''<s>''' if bos_token is None else bos_token

        super().__init__(
            do_lower_case=__UpperCamelCase						,				remove_space=__UpperCamelCase						,				keep_accents=__UpperCamelCase						,				bos_token=__UpperCamelCase						,				eos_token=__UpperCamelCase						,				unk_token=__UpperCamelCase						,				pad_token=__UpperCamelCase						,				sp_model_kwargs=self.sp_model_kwargs						,				**__UpperCamelCase						,				)

        __UpperCAmelCase    :						List[Any]       =						do_lower_case
        __UpperCAmelCase    :						int       =						remove_space
        __UpperCAmelCase    :						Union[str, Any]       =						keep_accents
        __UpperCAmelCase    :						Dict       =						vocab_file

        __UpperCAmelCase    :						Optional[int]       =						spm.SentencePieceProcessor(**self.sp_model_kwargs )
        self.sp_model.Load(__UpperCamelCase )

        # Used for whitespace normalization in input texts
        # fmt : off
        __UpperCAmelCase    :						Dict       =						{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
        # fmt : on

        # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
        __UpperCAmelCase    :						List[Any]       =						re.compile(
            f'''[{''.join(map(__UpperCamelCase						,				list(range(0						,				9 ) ) + list(range(11						,				32 ) ) + list(range(127						,				160 ) ) + [160, 173, 8_203] ) )}]''' )

      def __getstate__(							self       :      List[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						int       =						self.__dict__.copy()
        __UpperCAmelCase    :						Dict       =						None
        return state

      def __setstate__(							self       :      int						,				__lowercase       :      Union[str, Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						d

        # for backward compatibility
        if not hasattr(self						,				'''sp_model_kwargs''' ):
          __UpperCAmelCase    :						Any       =						{}

        __UpperCAmelCase    :						List[str]       =						spm.SentencePieceProcessor(**self.sp_model_kwargs )
        self.sp_model.Load(self.vocab_file )

      @property
      # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
      def       A_					(							self       :      Union[str, Any] ):


        '''simple docstring'''




        return len(self.sp_model )

      def       A_					(							self       :      str						,				__lowercase       :      str ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						self.non_printing_characters_re.sub(''''''						,				__UpperCamelCase )

        # Normalize whitespaces
        __UpperCAmelCase    :						Optional[Any]       =						''''''.join([char if char not in self.whitespaces else ''' ''' for char in text] )

        # NFC Unicode normalization
        __UpperCAmelCase    :						Dict       =						unicodedata.normalize('''NFC'''						,				__UpperCamelCase )
        return text

      def       A_					(							self       :      Optional[Any]						,				__lowercase       :      str						,				**__lowercase       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						int       =						self.preprocess_text(__UpperCamelCase )
        return self.sp_model.encode(__UpperCamelCase						,				out_type=__UpperCamelCase )

      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      str ):


        '''simple docstring'''




        return self.sp_model.PieceToId(__UpperCamelCase )

      def       A_					(							self       :      Optional[Any]						,				__lowercase       :      int ):


        '''simple docstring'''




        return self.sp_model.IdToPiece(__UpperCamelCase )

      @staticmethod
      def       A_					(							__lowercase       :      str ):


        '''simple docstring'''




        return out_string

      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[int]       =						[]
        __UpperCAmelCase    :						List[str]       =						''''''
        __UpperCAmelCase    :						List[str]       =						False
        for token in tokens:
          # make sure that special tokens are not decoded using sentencepiece model
          if token in self.all_special_tokens:
            # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
            if not prev_is_special:
              out_string += " "

            out_string += self.sp_model.decode(__UpperCamelCase ) + token
            __UpperCAmelCase    :						str       =						True
            __UpperCAmelCase    :						List[Any]       =						[]
          else:
            current_sub_tokens.append(__UpperCamelCase )
            __UpperCAmelCase    :						Tuple       =						False
        out_string += self.sp_model.decode(__UpperCamelCase )

        return out_string

      def       A_					(							self       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						{self.convert_ids_to_tokens(__UpperCamelCase ): i for i in range(self.vocab_size )}
        vocab.update(self.added_tokens_encoder )
        return vocab

      def       A_					(							self       :      str						,				__lowercase       :      str						,				__lowercase       :      Optional[str] = None ):


        '''simple docstring'''




        if not os.path.isdir(__UpperCamelCase ):
          logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
          return
        __UpperCAmelCase    :						Tuple       =						os.path.join(
            __UpperCamelCase						,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )

        if os.path.abspath(self.vocab_file ) != os.path.abspath(__UpperCamelCase ) and os.path.isfile(self.vocab_file ):
          copyfile(self.vocab_file						,				__UpperCamelCase )
        elif not os.path.isfile(self.vocab_file ):
          with open(__UpperCamelCase						,				'''wb''' ) as fi:
            __UpperCAmelCase    :						Union[str, Any]       =						self.sp_model.serialized_model_proto()
            fi.write(__UpperCamelCase )

        return (out_vocab_file,)

      def       A_					(							self       :      Optional[int]						,				__lowercase       :      Union[str, List[str]]						,				__lowercase       :      Union[str, bool] = False ):


        '''simple docstring'''




        if isinstance(__UpperCamelCase						,				__UpperCamelCase ):
          __UpperCAmelCase    :						int       =						self.preprocess_text(__UpperCamelCase )
          __UpperCAmelCase    :						Union[str, Any]       =						self.sp_model.encode(__UpperCamelCase )
        else:
          __UpperCAmelCase    :						List[Any]       =						[self.preprocess_text(__UpperCamelCase ) for t in text]
          __UpperCAmelCase    :						List[str]       =						self.sp_model.encode(__UpperCamelCase )

        if return_tensors is True or return_tensors == "pt":
          __UpperCAmelCase    :						Tuple       =						torch.tensor(__UpperCamelCase )

        return token_ids

      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      Union[int, List[int]] ):


        '''simple docstring'''




        return self.sp_model.decode(__UpperCamelCase )







      def       A_					(							self       :      Optional[Any]						,				__lowercase       :      "Conversation" ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[int]       =						[f'''User: {text}''' if is_user else f'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
        __UpperCAmelCase    :						Dict       =						(
            f'''{self.eos_token}{self.bos_token}''' + f'''{self.bos_token}'''.join(__UpperCamelCase ) + f'''{self.bos_token}Bot:'''
        )
        return self.encode(text=__UpperCamelCase )










"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowercase__				:Tuple				   =       {
    'configuration_xlm': ['XLM_PRETRAINED_CONFIG_ARCHIVE_MAP', 'XLMConfig', 'XLMOnnxConfig'],
    'tokenization_xlm': ['XLMTokenizer'],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase__				:Optional[Any]				   =       [
      'XLM_PRETRAINED_MODEL_ARCHIVE_LIST',
      'XLMForMultipleChoice',
      'XLMForQuestionAnswering',
      'XLMForQuestionAnsweringSimple',
      'XLMForSequenceClassification',
      'XLMForTokenClassification',
      'XLMModel',
      'XLMPreTrainedModel',
      'XLMWithLMHeadModel',
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase__				:List[Any]				   =       [
      'TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST',
      'TFXLMForMultipleChoice',
      'TFXLMForQuestionAnsweringSimple',
      'TFXLMForSequenceClassification',
      'TFXLMForTokenClassification',
      'TFXLMMainLayer',
      'TFXLMModel',
      'TFXLMPreTrainedModel',
      'TFXLMWithLMHeadModel',
  ]


if TYPE_CHECKING:
  from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, XLMOnnxConfig
  from .tokenization_xlm import XLMTokenizer

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_xlm import (
        XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        XLMForMultipleChoice,
        XLMForQuestionAnswering,
        XLMForQuestionAnsweringSimple,
        XLMForSequenceClassification,
        XLMForTokenClassification,
        XLMModel,
        XLMPreTrainedModel,
        XLMWithLMHeadModel,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_xlm import (
        TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMForMultipleChoice,
        TFXLMForQuestionAnsweringSimple,
        TFXLMForSequenceClassification,
        TFXLMForTokenClassification,
        TFXLMMainLayer,
        TFXLMModel,
        TFXLMPreTrainedModel,
        TFXLMWithLMHeadModel,
    )

else:
  import sys

  lowercase__				:List[Any]				   =       _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)