






'''simple docstring'''



import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_blenderbot import BlenderbotTokenizer


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation

SCREAMING_SNAKE_CASE							=						logging.get_logger(__name__)


SCREAMING_SNAKE_CASE							=						{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

SCREAMING_SNAKE_CASE							=						{
    'vocab_file': {'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json'},
    'merges_file': {'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt'},
    'tokenizer_config_file': {
        'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json'
    },
}

SCREAMING_SNAKE_CASE							=						{'facebook/blenderbot-3B': 128}
class UpperCAmelCase_       (     __snake_case					):




      """simple docstring"""




      UpperCamelCase_             =      VOCAB_FILES_NAMES
      UpperCamelCase_             =      PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase_             =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase_             =      ['''input_ids''', '''attention_mask''']
      UpperCamelCase_             =      BlenderbotTokenizer






      def __init__(			self      :		List[str]  ,   UpperCAmelCase      :		List[str]=None  ,   UpperCAmelCase      :		Any=None  ,   UpperCAmelCase      :		int=None  ,   UpperCAmelCase      :		str="replace"  ,   UpperCAmelCase      :		List[Any]="<s>"  ,   UpperCAmelCase      :		List[str]="</s>"  ,   UpperCAmelCase      :		Optional[Any]="</s>"  ,   UpperCAmelCase      :		List[Any]="<s>"  ,   UpperCAmelCase      :		Optional[int]="<unk>"  ,   UpperCAmelCase      :		int="<pad>"  ,   UpperCAmelCase      :		Union[str, Any]="<mask>"  ,   UpperCAmelCase      :		int=False  ,   UpperCAmelCase      :		Any=True  ,   **UpperCAmelCase      :		Dict  ,   )	->	Optional[Any]:


       '''simple docstring'''

       super().__init__(
           __UpperCamelCase  ,   __UpperCamelCase  ,   tokenizer_file=__UpperCamelCase  ,   errors=__UpperCamelCase  ,   bos_token=__UpperCamelCase  ,   eos_token=__UpperCamelCase  ,   sep_token=__UpperCamelCase  ,   cls_token=__UpperCamelCase  ,   unk_token=__UpperCamelCase  ,   pad_token=__UpperCamelCase  ,   mask_token=__UpperCamelCase  ,   add_prefix_space=__UpperCamelCase  ,   trim_offsets=__UpperCamelCase  ,   **__UpperCamelCase  ,   )

       lowercase       : Union[str, Any]						=json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
       if pre_tok_state.get('''add_prefix_space'''  ,   __UpperCamelCase		) != add_prefix_space:
        lowercase       : Dict						=getattr(__UpperCamelCase  ,   pre_tok_state.pop('''type'''		)		)
        lowercase       : Dict						=add_prefix_space
        lowercase       : List[Any]						=pre_tok_class(**__UpperCamelCase		)

       lowercase       : Union[str, Any]						=add_prefix_space

       lowercase       : Tuple						='''post_processor'''
       lowercase       : List[Any]						=getattr(self.backend_tokenizer  ,   __UpperCamelCase  ,   __UpperCamelCase		)
       if tokenizer_component_instance:
        lowercase       : Optional[Any]						=json.loads(tokenizer_component_instance.__getstate__()		)

        # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
        if "sep" in state:
         lowercase       : Union[str, Any]						=tuple(state['''sep''']		)
        if "cls" in state:
         lowercase       : Union[str, Any]						=tuple(state['''cls''']		)

        lowercase       : Any						=False

        if state.get('''add_prefix_space'''  ,   __UpperCamelCase		) != add_prefix_space:
         lowercase       : int						=add_prefix_space
         lowercase       : List[Any]						=True

        if state.get('''trim_offsets'''  ,   __UpperCamelCase		) != trim_offsets:
         lowercase       : Union[str, Any]						=trim_offsets
         lowercase       : int						=True

        if changes_to_apply:
         lowercase       : str						=getattr(__UpperCamelCase  ,   state.pop('''type'''		)		)
         lowercase       : List[str]						=component_class(**__UpperCamelCase		)
         setattr(self.backend_tokenizer  ,   __UpperCamelCase  ,   __UpperCamelCase		)






      @property
      # Copied from transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast.mask_token with Roberta->Blenderbot, RoBERTa->Blenderbot
      def 					A__   (			self      :		int		)	->	int:


       '''simple docstring'''

       if self._mask_token is None:
        if self.verbose:
         logger.error('''Using mask_token, but it is not set yet.'''		)
        return None
       return str(self._mask_token		)






      @mask_token.setter
      def 					A__   (			self      :		str  ,   UpperCAmelCase      :		str		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : int						=AddedToken(__UpperCamelCase  ,   lstrip=__UpperCamelCase  ,   rstrip=__UpperCamelCase		) if isinstance(__UpperCamelCase  ,   __UpperCamelCase		) else value
       lowercase       : Optional[int]						=value






      def 					A__   (			self      :		Dict  ,   *UpperCAmelCase      :		Union[str, Any]  ,   **UpperCAmelCase      :		Tuple		)	->	int:


       '''simple docstring'''

       lowercase       : str						=kwargs.get('''is_split_into_words'''  ,   __UpperCamelCase		)
       assert self.add_prefix_space or not is_split_into_words, (
           f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
           "to use it with pretokenized inputs."
       )

       return super()._batch_encode_plus(*__UpperCamelCase  ,   **__UpperCamelCase		)






      def 					A__   (			self      :		str  ,   *UpperCAmelCase      :		int  ,   **UpperCAmelCase      :		Optional[Any]		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : Optional[int]						=kwargs.get('''is_split_into_words'''  ,   __UpperCamelCase		)

       assert self.add_prefix_space or not is_split_into_words, (
           f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
           "to use it with pretokenized inputs."
       )

       return super()._encode_plus(*__UpperCamelCase  ,   **__UpperCamelCase		)






      def 					A__   (			self      :		Optional[int]  ,   UpperCAmelCase      :		Any  ,   UpperCAmelCase      :		List[Any] = None		)	->	List[str]:


       '''simple docstring'''

       lowercase       : Any						=self._tokenizer.model.save(__UpperCamelCase  ,   name=__UpperCamelCase		)
       return tuple(__UpperCamelCase		)






      def 					A__   (			self      :		Optional[Any]  ,   UpperCAmelCase      :		int  ,   UpperCAmelCase      :		int = None		)	->	List[str]:


       '''simple docstring'''

       lowercase       : str						=[self.sep_token_id]
       lowercase       : Any						=[self.cls_token_id]

       if token_ids_a is None:
        return len(cls + token_ids_a + sep		) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]






      def 					A__   (			self      :		Optional[int]  ,   UpperCAmelCase      :		List[str]  ,   UpperCAmelCase      :		Optional[int] = None		)	->	Dict:


       '''simple docstring'''

       return token_ids_a + [self.eos_token_id]






      def 					A__   (			self      :		Optional[Any]  ,   UpperCAmelCase      :		Tuple		)	->	Any:


       '''simple docstring'''

       lowercase       : Union[str, Any]						=[]
       for is_user, text in conversation.iter_texts():
        if is_user:
         # We need to space prefix as it's being done within blenderbot
         inputs.append(''' ''' + text		)
        else:
         # Generated responses should contain them already.
         inputs.append(__UpperCamelCase		)

       lowercase       : List[Any]						='''  '''.join(__UpperCamelCase		)
       lowercase       : Any						=self.encode(__UpperCamelCase		)
       if len(__UpperCamelCase		) > self.model_max_length:
        lowercase       : List[Any]						=input_ids[-self.model_max_length :]
        logger.warning(f'Trimmed input from conversation as it was longer than {self.model_max_length} tokens.'		)
       return input_ids






from ..utils import DummyObject, requires_backends


class SCREAMING_SNAKE_CASE	(		metaclass=__snake_case				):






			"""simple docstring"""




			__A    	=	["""flax""", """transformers"""]


			def __init__(     self			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(self			,					['flax', 'transformers']				)


			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







class SCREAMING_SNAKE_CASE	(		metaclass=__snake_case				):






			"""simple docstring"""




			__A    	=	["""flax""", """transformers"""]


			def __init__(     self			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(self			,					['flax', 'transformers']				)


			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







class SCREAMING_SNAKE_CASE	(		metaclass=__snake_case				):






			"""simple docstring"""




			__A    	=	["""flax""", """transformers"""]


			def __init__(     self			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(self			,					['flax', 'transformers']				)


			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







class SCREAMING_SNAKE_CASE	(		metaclass=__snake_case				):






			"""simple docstring"""




			__A    	=	["""flax""", """transformers"""]


			def __init__(     self			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(self			,					['flax', 'transformers']				)


			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)







			@classmethod
			def       __lowerCAmelCase			(     cls			,					*__UpperCamelCase			,					**__UpperCamelCase				):

				"""simple docstring"""


				requires_backends(cls			,					['flax', 'transformers']				)
