


"""simple docstring"""






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_blenderbot import BlenderbotTokenizer


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation

__snake_case		=       logging.get_logger(__name__)


__snake_case		=       {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
    '''tokenizer_config_file''': '''tokenizer_config.json''',
}

__snake_case		=       {
    '''vocab_file''': {'''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json'''},
    '''merges_file''': {'''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt'''},
    '''tokenizer_config_file''': {
        '''facebook/blenderbot-3B''': '''https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json'''
    },
}

__snake_case		=       {'''facebook/blenderbot-3B''': 128}


class     __lowerCamelCase  (					a__				):

     '''simple docstring'''
     A_       :	int    =		VOCAB_FILES_NAMES
     A_       :	Optional[Any]    =		PRETRAINED_VOCAB_FILES_MAP
     A_       :	str    =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     A_       :	Optional[Any]    =		['input_ids', 'attention_mask']
     A_       :	Optional[int]    =		BlenderbotTokenizer


     def __init__(      self    ,				__UpperCAmelCase=None    ,				__UpperCAmelCase=None    ,				__UpperCAmelCase=None    ,				__UpperCAmelCase="replace"    ,				__UpperCAmelCase="<s>"    ,				__UpperCAmelCase="</s>"    ,				__UpperCAmelCase="</s>"    ,				__UpperCAmelCase="<s>"    ,				__UpperCAmelCase="<unk>"    ,				__UpperCAmelCase="<pad>"    ,				__UpperCAmelCase="<mask>"    ,				__UpperCAmelCase=False    ,				__UpperCAmelCase=True    ,				**__UpperCAmelCase    ,				)       ->							Any:
          super().__init__(
              __UpperCAmelCase    ,				__UpperCAmelCase    ,				tokenizer_file=__UpperCAmelCase    ,				errors=__UpperCAmelCase    ,				bos_token=__UpperCAmelCase    ,				eos_token=__UpperCAmelCase    ,				sep_token=__UpperCAmelCase    ,				cls_token=__UpperCAmelCase    ,				unk_token=__UpperCAmelCase    ,				pad_token=__UpperCAmelCase    ,				mask_token=__UpperCAmelCase    ,				add_prefix_space=__UpperCAmelCase    ,				trim_offsets=__UpperCAmelCase    ,				**__UpperCAmelCase    ,				)

          _a    	=				json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
          if pre_tok_state.get('''add_prefix_space'''    ,				__UpperCAmelCase							) != add_prefix_space:
               _a    	=				getattr(__UpperCAmelCase    ,				pre_tok_state.pop('''type'''							)							)
               _a    	=				add_prefix_space
               _a    	=				pre_tok_class(**__UpperCAmelCase							)

          _a    	=				add_prefix_space

          _a    	=				'''post_processor'''
          _a    	=				getattr(self.backend_tokenizer    ,				__UpperCAmelCase    ,				__UpperCAmelCase							)
          if tokenizer_component_instance:
               _a    	=				json.loads(tokenizer_component_instance.__getstate__()							)

               # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
               if "sep" in state:
                    _a    	=				tuple(state['''sep''']							)
               if "cls" in state:
                    _a    	=				tuple(state['''cls''']							)

               _a    	=				False

               if state.get('''add_prefix_space'''    ,				__UpperCAmelCase							) != add_prefix_space:
                    _a    	=				add_prefix_space
                    _a    	=				True

               if state.get('''trim_offsets'''    ,				__UpperCAmelCase							) != trim_offsets:
                    _a    	=				trim_offsets
                    _a    	=				True

               if changes_to_apply:
                    _a    	=				getattr(__UpperCAmelCase    ,				state.pop('''type'''							)							)
                    _a    	=				component_class(**__UpperCAmelCase							)
                    setattr(self.backend_tokenizer    ,				__UpperCAmelCase    ,				__UpperCAmelCase							)


     @property
     # Copied from transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast.mask_token with Roberta->Blenderbot, RoBERTa->Blenderbot
     def 							_UpperCAmelCase      (      self							)       ->							str:
          if self._mask_token is None:
               if self.verbose:
                    logger.error('''Using mask_token, but it is not set yet.'''							)
               return None
          return str(self._mask_token							)


     @mask_token.setter
     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							List[str]:
          _a    	=				AddedToken(__UpperCAmelCase    ,				lstrip=__UpperCAmelCase    ,				rstrip=__UpperCAmelCase							) if isinstance(__UpperCAmelCase    ,				__UpperCAmelCase							) else value
          _a    	=				value


     def 							_UpperCAmelCase      (      self    ,				*__UpperCAmelCase    ,				**__UpperCAmelCase							)       ->							BatchEncoding:
          _a    	=				kwargs.get('''is_split_into_words'''    ,				__UpperCAmelCase							)
          assert self.add_prefix_space or not is_split_into_words, (
              F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
              "to use it with pretokenized inputs."
          )

          return super()._batch_encode_plus(*__UpperCAmelCase    ,				**__UpperCAmelCase							)


     def 							_UpperCAmelCase      (      self    ,				*__UpperCAmelCase    ,				**__UpperCAmelCase							)       ->							BatchEncoding:
          _a    	=				kwargs.get('''is_split_into_words'''    ,				__UpperCAmelCase							)

          assert self.add_prefix_space or not is_split_into_words, (
              F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
              "to use it with pretokenized inputs."
          )

          return super()._encode_plus(*__UpperCAmelCase    ,				**__UpperCAmelCase							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							Tuple[str]:
          _a    	=				self._tokenizer.model.save(__UpperCAmelCase    ,				name=__UpperCAmelCase							)
          return tuple(__UpperCAmelCase							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							List[int]:
          _a    	=				[self.sep_token_id]
          _a    	=				[self.cls_token_id]

          if token_ids_a is None:
               return len(cls + token_ids_a + sep							) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							Union[str, Any]:
          return token_ids_a + [self.eos_token_id]





     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							List[int]:
          _a    	=				[]
          for is_user, text in conversation.iter_texts():
               if is_user:
                    # We need to space prefix as it's being done within blenderbot
                    inputs.append(''' ''' + text							)
               else:
                    # Generated responses should contain them already.
                    inputs.append(__UpperCAmelCase							)

          _a    	=				'''  '''.join(__UpperCAmelCase							)
          _a    	=				self.encode(__UpperCAmelCase							)
          if len(__UpperCAmelCase							) > self.model_max_length:
               _a    	=				input_ids[-self.model_max_length :]
               logger.warning(F'Trimmed input from conversation as it was longer than {self.model_max_length} tokens.'							)
          return input_ids



"""simple docstring"""






from . import __version__

# Backward compatibility imports, to make sure all those objects can be found in file_utils
from .utils import (
    CLOUDFRONT_DISTRIB_PREFIX,
    CONFIG_NAME,
    DISABLE_TELEMETRY,
    DUMMY_INPUTS,
    DUMMY_MASK,
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    FEATURE_EXTRACTOR_NAME,
    FLAX_WEIGHTS_NAME,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_PREFIX,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    MODEL_CARD_NAME,
    MULTIPLE_CHOICE_DUMMY_INPUTS,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    S3_BUCKET_PREFIX,
    SENTENCEPIECE_UNDERLINE,
    SPIECE_UNDERLINE,
    TF2_WEIGHTS_NAME,
    TF_WEIGHTS_NAME,
    TORCH_FX_REQUIRED_VERSION,
    TRANSFORMERS_CACHE,
    TRANSFORMERS_DYNAMIC_MODULE_NAME,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    WEIGHTS_INDEX_NAME,
    WEIGHTS_NAME,
    ContextManagers,
    DummyObject,
    EntryNotFoundError,
    ExplicitEnum,
    ModelOutput,
    PaddingStrategy,
    PushToHubMixin,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    TensorType,
    _LazyModule,
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    cached_property,
    copy_func,
    default_cache_path,
    define_sagemaker_information,
    get_cached_models,
    get_file_from_repo,
    get_full_repo_name,
    get_torch_version,
    has_file,
    http_user_agent,
    is_apex_available,
    is_bsa_available,
    is_coloredlogs_available,
    is_datasets_available,
    is_detectrona_available,
    is_faiss_available,
    is_flax_available,
    is_ftfy_available,
    is_in_notebook,
    is_ipex_available,
    is_librosa_available,
    is_offline_mode,
    is_onnx_available,
    is_pandas_available,
    is_phonemizer_available,
    is_protobuf_available,
    is_psutil_available,
    is_pyanvml_available,
    is_pyctcdecode_available,
    is_pytesseract_available,
    is_pytorch_quantization_available,
    is_rjieba_available,
    is_sagemaker_dp_enabled,
    is_sagemaker_mp_enabled,
    is_scipy_available,
    is_sentencepiece_available,
    is_seqio_available,
    is_sklearn_available,
    is_soundfile_availble,
    is_spacy_available,
    is_speech_available,
    is_tensor,
    is_tensorflow_probability_available,
    is_tfaonnx_available,
    is_tf_available,
    is_timm_available,
    is_tokenizers_available,
    is_torch_available,
    is_torch_bfaa_available,
    is_torch_cuda_available,
    is_torch_fx_available,
    is_torch_fx_proxy,
    is_torch_mps_available,
    is_torch_tfaa_available,
    is_torch_tpu_available,
    is_torchaudio_available,
    is_training_run_on_sagemaker,
    is_vision_available,
    replace_return_docstrings,
    requires_backends,
    to_numpy,
    to_py_obj,
    torch_only_method,
)