






"""simple docstring"""




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation

A_		:		Union[str, Any]		     =			logging.get_logger(__name__)


A_		:		str		     =			{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

A_		:		Tuple		     =			{
    """vocab_file""": {"""facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"""},
    """merges_file""": {"""facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"""},
    """tokenizer_config_file""": {
        """facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"""
    },
}

A_		:		Dict		     =			{"""facebook/blenderbot-3B""": 128}

@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def   snake_case ()	->    Optional[int]:
  UpperCamelCase_: Union[str, Any]   				=    (
      list(range(ord('!'  )     ,   ord('~'  ) + 1  )  ) + list(range(ord('¡'  )     ,   ord('¬'  ) + 1  )  ) + list(range(ord('®'  )     ,   ord('ÿ'  ) + 1  )  )
  )
  UpperCamelCase_: Union[str, Any]   				=    bs[:]
  UpperCamelCase_: Any   				=    0
  for b in range(2**8  ):
    if b not in bs:
      bs.append(_A  )
      cs.append(2**8 + n  )
      n += 1
  UpperCamelCase_: str   				=    [chr(_A  ) for n in cs]
  return dict(zip(_A     ,   _A  )  )

def   snake_case (UpperCAmelCase__  )	->    Tuple:
  UpperCamelCase_: Union[str, Any]   				=    set()
  UpperCamelCase_: Union[str, Any]   				=    word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)  )
    UpperCamelCase_: Union[str, Any]   				=    char
  return pairs




class     _lowerCAmelCase(	SCREAMING_SNAKE_CASE_ ):





       """simple docstring"""







       a    :		int									=VOCAB_FILES_NAMES
       a    :		List[str]									=PRETRAINED_VOCAB_FILES_MAP
       a    :		str									=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       a    :		Optional[Any]									=['''input_ids''', '''attention_mask''']





       def __init__( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase="replace"  ,      _lowerCamelCase="<s>"  ,      _lowerCamelCase="</s>"  ,      _lowerCamelCase="</s>"  ,      _lowerCamelCase="<s>"  ,      _lowerCamelCase="<unk>"  ,      _lowerCamelCase="<pad>"  ,      _lowerCamelCase="<mask>"  ,      _lowerCamelCase=False  ,      **_lowerCamelCase  ,      ):
         UpperCamelCase_: Any   				=    AddedToken(snake_case__  ,      lstrip=snake_case__  ,      rstrip=snake_case__							) if isinstance(snake_case__  ,      snake_case__							) else bos_token
         UpperCamelCase_: List[str]   				=    AddedToken(snake_case__  ,      lstrip=snake_case__  ,      rstrip=snake_case__							) if isinstance(snake_case__  ,      snake_case__							) else eos_token
         UpperCamelCase_: int   				=    AddedToken(snake_case__  ,      lstrip=snake_case__  ,      rstrip=snake_case__							) if isinstance(snake_case__  ,      snake_case__							) else sep_token
         UpperCamelCase_: Optional[Any]   				=    AddedToken(snake_case__  ,      lstrip=snake_case__  ,      rstrip=snake_case__							) if isinstance(snake_case__  ,      snake_case__							) else cls_token
         UpperCamelCase_: Any   				=    AddedToken(snake_case__  ,      lstrip=snake_case__  ,      rstrip=snake_case__							) if isinstance(snake_case__  ,      snake_case__							) else unk_token
         UpperCamelCase_: Any   				=    AddedToken(snake_case__  ,      lstrip=snake_case__  ,      rstrip=snake_case__							) if isinstance(snake_case__  ,      snake_case__							) else pad_token

         # Mask token behave like a normal word, i.e. include the space before it
         UpperCamelCase_: Tuple   				=    AddedToken(snake_case__  ,      lstrip=snake_case__  ,      rstrip=snake_case__							) if isinstance(snake_case__  ,      snake_case__							) else mask_token

         super().__init__(
             errors=snake_case__  ,      bos_token=snake_case__  ,      eos_token=snake_case__  ,      unk_token=snake_case__  ,      sep_token=snake_case__  ,      cls_token=snake_case__  ,      pad_token=snake_case__  ,      mask_token=snake_case__  ,      add_prefix_space=snake_case__  ,      **snake_case__  ,      )

         with open(snake_case__  ,      encoding='utf-8'							) as vocab_handle:
           UpperCamelCase_: str   				=    json.load(snake_case__							)
         UpperCamelCase_: List[Any]   				=    {v: k for k, v in self.encoder.items()}
         UpperCamelCase_: Dict   				=    errors  # how to handle errors in decoding
         UpperCamelCase_: int   				=    bytes_to_unicode()
         UpperCamelCase_: Tuple   				=    {v: k for k, v in self.byte_encoder.items()}
         with open(snake_case__  ,      encoding='utf-8'							) as merges_handle:
           UpperCamelCase_: str   				=    merges_handle.read().split('\n'							)[1:-1]
         UpperCamelCase_: str   				=    [tuple(merge.split()							) for merge in bpe_merges]
         UpperCamelCase_: Tuple   				=    dict(zip(snake_case__  ,      range(len(snake_case__							)							)							)							)
         UpperCamelCase_: List[str]   				=    {}
         UpperCamelCase_: Optional[int]   				=    add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         UpperCamelCase_: Optional[int]   				=    re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'							)





       @property
       # Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
       def       _a   ( self							):
         return len(self.encoder							)





       def       _a   ( self							):
         return dict(self.encoder  ,      **self.added_tokens_encoder							)





       def       _a   ( self  ,      _lowerCamelCase							):

         if token in self.cache:
           return self.cache[token]
         UpperCamelCase_: Optional[Any]   				=    tuple(snake_case__							)
         UpperCamelCase_: Tuple   				=    get_pairs(snake_case__							)

         if not pairs:
           return token

         while True:
           UpperCamelCase_: List[str]   				=    min(snake_case__  ,      key=lambda _lowerCamelCase							: self.bpe_ranks.get(snake_case__  ,      float('inf'							)							)							)
           if bigram not in self.bpe_ranks:
             break
           UpperCamelCase_: Optional[Any]   				=    bigram
           UpperCamelCase_: Tuple   				=    []
           UpperCamelCase_: str   				=    0
           while i < len(snake_case__							):
             try:
               UpperCamelCase_: List[Any]   				=    word.index(snake_case__  ,      snake_case__							)
             except ValueError:
               new_word.extend(word[i:]							)
               break
             else:
               new_word.extend(word[i:j]							)
               UpperCamelCase_: str   				=    j

             if word[i] == first and i < len(snake_case__							) - 1 and word[i + 1] == second:
               new_word.append(first + second							)
               i += 2
             else:
               new_word.append(word[i]							)
               i += 1
           UpperCamelCase_: List[str]   				=    tuple(snake_case__							)
           UpperCamelCase_: int   				=    new_word
           if len(snake_case__							) == 1:
             break
           else:
             UpperCamelCase_: Tuple   				=    get_pairs(snake_case__							)
         UpperCamelCase_: Union[str, Any]   				=    ' '.join(snake_case__							)
         UpperCamelCase_: Union[str, Any]   				=    word
         return word





       def       _a   ( self  ,      _lowerCamelCase							):
         UpperCamelCase_: int   				=    []
         for token in re.findall(self.pat  ,      snake_case__							):
           UpperCamelCase_: Optional[int]   				=    ''.join(
               self.byte_encoder[b] for b in token.encode('utf-8'							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
           bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case__							).split(' '							)							)
         return bpe_tokens





       def       _a   ( self  ,      _lowerCamelCase							):
         return self.encoder.get(snake_case__  ,      self.encoder.get(self.unk_token							)							)





       def       _a   ( self  ,      _lowerCamelCase							):
         return self.decoder.get(snake_case__							)





       def       _a   ( self  ,      _lowerCamelCase							):
         UpperCamelCase_: Dict   				=    ''.join(snake_case__							)
         UpperCamelCase_: Tuple   				=    bytearray([self.byte_decoder[c] for c in text]							).decode('utf-8'  ,      errors=self.errors							)
         return text





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase = None							):

         if not os.path.isdir(snake_case__							):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''							)
           return
         UpperCamelCase_: Any   				=    os.path.join(
             snake_case__  ,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)
         UpperCamelCase_: str   				=    os.path.join(
             snake_case__  ,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']							)

         with open(snake_case__  ,      'w'  ,      encoding='utf-8'							) as f:
           f.write(json.dumps(self.encoder  ,      indent=2  ,      sort_keys=snake_case__  ,      ensure_ascii=snake_case__							) + '\n'							)

         UpperCamelCase_: int   				=    0
         with open(snake_case__  ,      'w'  ,      encoding='utf-8'							) as writer:
           writer.write('#version: 0.2\n'							)
           for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,      key=lambda _lowerCamelCase							: kv[1]							):
             if index != token_index:
               logger.warning(
                   f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                   ' Please check that the tokenizer is not corrupted!'							)
               UpperCamelCase_: Tuple   				=    token_index
             writer.write(' '.join(snake_case__							) + '\n'							)
             index += 1

         return vocab_file, merge_file





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase = None  ,      _lowerCamelCase = False							):

         if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=snake_case__  ,      token_ids_a=snake_case__  ,      already_has_special_tokens=snake_case__							)

         if token_ids_a is None:
           return [1] + ([0] * len(snake_case__							)) + [1]
         return [1] + ([0] * len(snake_case__							)) + [1, 1] + ([0] * len(snake_case__							)) + [1]





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase = None							):
         UpperCamelCase_: List[str]   				=    [self.sep_token_id]
         UpperCamelCase_: str   				=    [self.cls_token_id]

         if token_ids_a is None:
           return len(cls + token_ids_a + sep							) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase=False  ,      **_lowerCamelCase							):
         UpperCamelCase_: Optional[int]   				=    kwargs.pop('add_prefix_space'  ,      self.add_prefix_space							)
         if (is_split_into_words or add_prefix_space) and (len(snake_case__							) > 0 and not text[0].isspace()):
           UpperCamelCase_: Tuple   				=    ' ' + text
         return (text, kwargs)





       def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase = None							):
         return token_ids_a + [self.eos_token_id]





       def       _a   ( self  ,      _lowerCamelCase							):
         UpperCamelCase_: Any   				=    []
         for is_user, text in conversation.iter_texts():
           if is_user:
             # We need to space prefix as it's being done within blenderbot
             inputs.append(' ' + text							)
           else:
             # Generated responses should contain them already.
             inputs.append(snake_case__							)

         UpperCamelCase_: List[Any]   				=    '  '.join(snake_case__							)
         UpperCamelCase_: str   				=    self.encode(snake_case__							)
         if len(snake_case__							) > self.model_max_length:
           UpperCamelCase_: Any   				=    input_ids[-self.model_max_length :]
           logger.warning(f'''Trimmed input from conversation as it was longer than {self.model_max_length} tokens.'''							)
         return input_ids







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


A_		:		Tuple		     =			{
    'configuration_funnel': ['FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP', 'FunnelConfig'],
    'convert_funnel_original_tf_checkpoint_to_pytorch': [],
    'tokenization_funnel': ['FunnelTokenizer'],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_		:		List[str]		     =			['FunnelTokenizerFast']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_		:		List[str]		     =			[
       'FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST',
       'FunnelBaseModel',
       'FunnelForMaskedLM',
       'FunnelForMultipleChoice',
       'FunnelForPreTraining',
       'FunnelForQuestionAnswering',
       'FunnelForSequenceClassification',
       'FunnelForTokenClassification',
       'FunnelModel',
       'FunnelPreTrainedModel',
       'load_tf_weights_in_funnel',
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A_		:		str		     =			[
       'TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST',
       'TFFunnelBaseModel',
       'TFFunnelForMaskedLM',
       'TFFunnelForMultipleChoice',
       'TFFunnelForPreTraining',
       'TFFunnelForQuestionAnswering',
       'TFFunnelForSequenceClassification',
       'TFFunnelForTokenClassification',
       'TFFunnelModel',
       'TFFunnelPreTrainedModel',
   ]


if TYPE_CHECKING:
   from .configuration_funnel import FUNNEL_PRETRAINED_CONFIG_ARCHIVE_MAP, FunnelConfig
   from .tokenization_funnel import FunnelTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_funnel_fast import FunnelTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_funnel import (
          FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
          FunnelBaseModel,
          FunnelForMaskedLM,
          FunnelForMultipleChoice,
          FunnelForPreTraining,
          FunnelForQuestionAnswering,
          FunnelForSequenceClassification,
          FunnelForTokenClassification,
          FunnelModel,
          FunnelPreTrainedModel,
          load_tf_weights_in_funnel,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_funnel import (
          TF_FUNNEL_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFFunnelBaseModel,
          TFFunnelForMaskedLM,
          TFFunnelForMultipleChoice,
          TFFunnelForPreTraining,
          TFFunnelForQuestionAnswering,
          TFFunnelForSequenceClassification,
          TFFunnelForTokenClassification,
          TFFunnelModel,
          TFFunnelPreTrainedModel,
      )

else:
   import sys

   A_		:		Any		     =			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)