





"""simple docstring"""





import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


lowercase__				:Optional[Any]				   =       logging.get_logger(__name__)

lowercase__				:str				   =       {'tokenizer_file': 'tokenizer.json'}

lowercase__				:Any				   =       {
    'tokenizer_file': {
        'bigscience/tokenizer': 'https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json',
        'bigscience/bloom-560m': 'https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json',
        'bigscience/bloom-1b1': 'https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json',
        'bigscience/bloom-1b7': 'https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json',
        'bigscience/bloom-3b': 'https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json',
        'bigscience/bloom-7b1': 'https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json',
        'bigscience/bloom': 'https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json',
    },
}




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       List[str]         =       VOCAB_FILES_NAMES
      _A     :       Tuple         =       PRETRAINED_VOCAB_FILES_MAP
      _A     :       Union[str, Any]         =       ['input_ids', 'attention_mask']
      _A     :       Any         =       None

      def __init__(							self       :      List[str]						,				__lowercase       :      int=None						,				__lowercase       :      int=None						,				__lowercase       :      Union[str, Any]=None						,				__lowercase       :      str="<unk>"						,				__lowercase       :      int="<s>"						,				__lowercase       :      Union[str, Any]="</s>"						,				__lowercase       :      Optional[Any]="<pad>"						,				__lowercase       :      Any=False						,				__lowercase       :      Dict=False						,				**__lowercase       :      Dict						,				):


        '''simple docstring'''




        super().__init__(
            __lowercase						,				__lowercase						,				tokenizer_file=__lowercase						,				unk_token=__lowercase						,				bos_token=__lowercase						,				eos_token=__lowercase						,				pad_token=__lowercase						,				add_prefix_space=__lowercase						,				clean_up_tokenization_spaces=__lowercase						,				**__lowercase						,				)
        __UpperCAmelCase    :						Tuple       =						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__() )
        if pre_tok_state.get('''add_prefix_space'''						,				__lowercase ) != add_prefix_space:
          __UpperCAmelCase    :						List[Any]       =						getattr(__lowercase						,				pre_tok_state.pop('''type''' ) )
          __UpperCAmelCase    :						List[str]       =						add_prefix_space
          __UpperCAmelCase    :						str       =						pre_tok_class(**__lowercase )

        __UpperCAmelCase    :						Union[str, Any]       =						add_prefix_space

      def       A_					(							self       :      int						,				*__lowercase       :      Dict						,				**__lowercase       :      int ):


        '''simple docstring'''




        __UpperCAmelCase    :						str       =						kwargs.get('''is_split_into_words'''						,				__lowercase )
        if not (self.add_prefix_space or not is_split_into_words):
          raise Exception(
              f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
              ''' pretokenized inputs.''' )

        return super()._batch_encode_plus(*__lowercase						,				**__lowercase )

      def       A_					(							self       :      Optional[int]						,				*__lowercase       :      int						,				**__lowercase       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Union[str, Any]       =						kwargs.get('''is_split_into_words'''						,				__lowercase )

        if not (self.add_prefix_space or not is_split_into_words):
          raise Exception(
              f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
              ''' pretokenized inputs.''' )

        return super()._encode_plus(*__lowercase						,				**__lowercase )

      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      str						,				__lowercase       :      Optional[str] = None ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						self._tokenizer.model.save(__lowercase						,				name=__lowercase )
        return tuple(__lowercase )







      def       A_					(							self       :      int						,				__lowercase       :      "Conversation" ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[Any]       =						[]
        for is_user, text in conversation.iter_texts():
          input_ids.extend(self.encode(__lowercase						,				add_special_tokens=__lowercase ) + [self.eos_token_id] )

        if len(__lowercase ) > self.model_max_length:
          __UpperCAmelCase    :						Optional[int]       =						input_ids[-self.model_max_length :]
        return input_ids






"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


lowercase__				:Tuple				   =       {
    'configuration_xlm': ['XLM_PRETRAINED_CONFIG_ARCHIVE_MAP', 'XLMConfig', 'XLMOnnxConfig'],
    'tokenization_xlm': ['XLMTokenizer'],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase__				:Optional[Any]				   =       [
      'XLM_PRETRAINED_MODEL_ARCHIVE_LIST',
      'XLMForMultipleChoice',
      'XLMForQuestionAnswering',
      'XLMForQuestionAnsweringSimple',
      'XLMForSequenceClassification',
      'XLMForTokenClassification',
      'XLMModel',
      'XLMPreTrainedModel',
      'XLMWithLMHeadModel',
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase__				:List[Any]				   =       [
      'TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST',
      'TFXLMForMultipleChoice',
      'TFXLMForQuestionAnsweringSimple',
      'TFXLMForSequenceClassification',
      'TFXLMForTokenClassification',
      'TFXLMMainLayer',
      'TFXLMModel',
      'TFXLMPreTrainedModel',
      'TFXLMWithLMHeadModel',
  ]


if TYPE_CHECKING:
  from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XLMConfig, XLMOnnxConfig
  from .tokenization_xlm import XLMTokenizer

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_xlm import (
        XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        XLMForMultipleChoice,
        XLMForQuestionAnswering,
        XLMForQuestionAnsweringSimple,
        XLMForSequenceClassification,
        XLMForTokenClassification,
        XLMModel,
        XLMPreTrainedModel,
        XLMWithLMHeadModel,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_xlm import (
        TF_XLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXLMForMultipleChoice,
        TFXLMForQuestionAnsweringSimple,
        TFXLMForSequenceClassification,
        TFXLMForTokenClassification,
        TFXLMMainLayer,
        TFXLMModel,
        TFXLMPreTrainedModel,
        TFXLMWithLMHeadModel,
    )

else:
  import sys

  lowercase__				:List[Any]				   =       _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)