





'''simple docstring'''
import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a__				:				Union[str, Any]					=logging.get_logger(__name__)


a__				:				int					={'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all BART models at https://huggingface.co/models?filter=bart
a__				:				List[str]					={
    'vocab_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/vocab.json',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/vocab.json',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json',
    },
    'merges_file': {
        'facebook/bart-base': 'https://huggingface.co/facebook/bart-base/resolve/main/merges.txt',
        'facebook/bart-large': 'https://huggingface.co/facebook/bart-large/resolve/main/merges.txt',
        'facebook/bart-large-mnli': 'https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt',
        'facebook/bart-large-cnn': 'https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt',
        'facebook/bart-large-xsum': 'https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt',
        'yjernite/bart_eli5': 'https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt',
    },
}

a__				:				Any					={
    'facebook/bart-base': 1_024,
    'facebook/bart-large': 1_024,
    'facebook/bart-large-mnli': 1_024,
    'facebook/bart-large-cnn': 1_024,
    'facebook/bart-large-xsum': 1_024,
    'yjernite/bart_eli5': 1_024,
}
@lru_cache()
def 							lowercase__  (  )					-> Optional[Any]:



       """simple docstring"""






       __UpperCamelCase			=    (
           list(range(ord('!'	)							,    ord('~'	) + 1	)	) + list(range(ord('¡'	)							,    ord('¬'	) + 1	)	) + list(range(ord('®'	)							,    ord('ÿ'	) + 1	)	)
       )
       __UpperCamelCase			=    bs[:]
       __UpperCamelCase			=    0
       for b in range(2**8	):
              if b not in bs:
                     bs.append(__lowercase	)
                     cs.append(2**8 + n	)
                     n += 1
       __UpperCamelCase			=    [chr(__lowercase	) for n in cs]
       return dict(zip(__lowercase							,    __lowercase	)	)
def 							lowercase__  (  __lowercase		:    Any	)					-> Optional[Any]:



       """simple docstring"""






       __UpperCamelCase			=    set()
       __UpperCamelCase			=    word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)	)
              __UpperCamelCase			=    char
       return pairs


class 	snake_case	(       __lowerCamelCase			):




 """simple docstring"""



 SCREAMING_SNAKE_CASE_      :   Tuple     						=VOCAB_FILES_NAMES
 SCREAMING_SNAKE_CASE_      :   List[Any]     						=PRETRAINED_VOCAB_FILES_MAP
 SCREAMING_SNAKE_CASE_      :   Dict     						=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 SCREAMING_SNAKE_CASE_      :   List[Any]     						=['input_ids', 'attention_mask']






 def __init__(			self			:				List[str]				,      __A			:				Union[str, Any]				,      __A			:				Optional[Any]				,      __A			:				Optional[int]="replace"				,      __A			:				Optional[int]="<s>"				,      __A			:				List[str]="</s>"				,      __A			:				Optional[Any]="</s>"				,      __A			:				List[str]="<s>"				,      __A			:				Union[str, Any]="<unk>"				,      __A			:				str="<pad>"				,      __A			:				Optional[Any]="<mask>"				,      __A			:				int=False				,      **__A			:				List[Any]				,      ):
        __UpperCamelCase			=    AddedToken(lowerCAmelCase__				,      lstrip=lowerCAmelCase__				,      rstrip=lowerCAmelCase__      ) if isinstance(lowerCAmelCase__				,      lowerCAmelCase__      ) else bos_token
        __UpperCamelCase			=    AddedToken(lowerCAmelCase__				,      lstrip=lowerCAmelCase__				,      rstrip=lowerCAmelCase__      ) if isinstance(lowerCAmelCase__				,      lowerCAmelCase__      ) else eos_token
        __UpperCamelCase			=    AddedToken(lowerCAmelCase__				,      lstrip=lowerCAmelCase__				,      rstrip=lowerCAmelCase__      ) if isinstance(lowerCAmelCase__				,      lowerCAmelCase__      ) else sep_token
        __UpperCamelCase			=    AddedToken(lowerCAmelCase__				,      lstrip=lowerCAmelCase__				,      rstrip=lowerCAmelCase__      ) if isinstance(lowerCAmelCase__				,      lowerCAmelCase__      ) else cls_token
        __UpperCamelCase			=    AddedToken(lowerCAmelCase__				,      lstrip=lowerCAmelCase__				,      rstrip=lowerCAmelCase__      ) if isinstance(lowerCAmelCase__				,      lowerCAmelCase__      ) else unk_token
        __UpperCamelCase			=    AddedToken(lowerCAmelCase__				,      lstrip=lowerCAmelCase__				,      rstrip=lowerCAmelCase__      ) if isinstance(lowerCAmelCase__				,      lowerCAmelCase__      ) else pad_token

        # Mask token behave like a normal word, i.e. include the space before it
        __UpperCamelCase			=    AddedToken(lowerCAmelCase__				,      lstrip=lowerCAmelCase__				,      rstrip=lowerCAmelCase__      ) if isinstance(lowerCAmelCase__				,      lowerCAmelCase__      ) else mask_token

        super().__init__(
            errors=lowerCAmelCase__				,      bos_token=lowerCAmelCase__				,      eos_token=lowerCAmelCase__				,      unk_token=lowerCAmelCase__				,      sep_token=lowerCAmelCase__				,      cls_token=lowerCAmelCase__				,      pad_token=lowerCAmelCase__				,      mask_token=lowerCAmelCase__				,      add_prefix_space=lowerCAmelCase__				,      **lowerCAmelCase__				,      )

        with open(lowerCAmelCase__				,      encoding='utf-8'      ) as vocab_handle:
               __UpperCamelCase			=    json.load(lowerCAmelCase__      )
        __UpperCamelCase			=    {v: k for k, v in self.encoder.items()}
        __UpperCamelCase			=    errors  # how to handle errors in decoding
        __UpperCamelCase			=    bytes_to_unicode()
        __UpperCamelCase			=    {v: k for k, v in self.byte_encoder.items()}
        with open(lowerCAmelCase__				,      encoding='utf-8'      ) as merges_handle:
               __UpperCamelCase			=    merges_handle.read().split('\n'      )[1:-1]
        __UpperCamelCase			=    [tuple(merge.split()      ) for merge in bpe_merges]
        __UpperCamelCase			=    dict(zip(lowerCAmelCase__				,      range(len(lowerCAmelCase__      )      )      )      )
        __UpperCamelCase			=    {}
        __UpperCamelCase			=    add_prefix_space

        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
        __UpperCamelCase			=    re.compile(R'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'      )






 @property
 def 			_lowerCamelCase   (			self			:				str      ):
        return len(self.encoder      )






 def 			_lowerCamelCase   (			self			:				Tuple      ):
        return dict(self.encoder				,      **self.added_tokens_encoder      )






 def 			_lowerCamelCase   (			self			:				Optional[int]				,      __A			:				Any      ):




        if token in self.cache:
               return self.cache[token]
        __UpperCamelCase			=    tuple(lowerCAmelCase__      )
        __UpperCamelCase			=    get_pairs(lowerCAmelCase__      )

        if not pairs:
               return token

        while True:
               __UpperCamelCase			=    min(lowerCAmelCase__				,      key=lambda __A      : self.bpe_ranks.get(lowerCAmelCase__				,      float('inf'      )      )      )
               if bigram not in self.bpe_ranks:
                      break
               __UpperCamelCase			,		__UpperCamelCase			=    bigram
               __UpperCamelCase			=    []
               __UpperCamelCase			=    0
               while i < len(lowerCAmelCase__      ):
                      try:
                             __UpperCamelCase			=    word.index(lowerCAmelCase__				,      lowerCAmelCase__      )
                      except ValueError:
                             new_word.extend(word[i:]      )
                             break
                      else:
                             new_word.extend(word[i:j]      )
                             __UpperCamelCase			=    j

                      if word[i] == first and i < len(lowerCAmelCase__      ) - 1 and word[i + 1] == second:
                             new_word.append(first + second      )
                             i += 2
                      else:
                             new_word.append(word[i]      )
                             i += 1
               __UpperCamelCase			=    tuple(lowerCAmelCase__      )
               __UpperCamelCase			=    new_word
               if len(lowerCAmelCase__      ) == 1:
                      break
               else:
                      __UpperCamelCase			=    get_pairs(lowerCAmelCase__      )
        __UpperCamelCase			=    ' '.join(lowerCAmelCase__      )
        __UpperCamelCase			=    word
        return word






 def 			_lowerCamelCase   (			self			:				Union[str, Any]				,      __A			:				Tuple      ):
        __UpperCamelCase			=    []
        for token in re.findall(self.pat				,      lowerCAmelCase__      ):
               __UpperCamelCase			=    ''.join(
                   self.byte_encoder[b] for b in token.encode('utf-8'      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
               bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase__      ).split(' '      )      )
        return bpe_tokens






 def 			_lowerCamelCase   (			self			:				str				,      __A			:				List[str]      ):
        return self.encoder.get(lowerCAmelCase__				,      self.encoder.get(self.unk_token      )      )






 def 			_lowerCamelCase   (			self			:				List[str]				,      __A			:				Optional[int]      ):
        return self.decoder.get(lowerCAmelCase__      )






 def 			_lowerCamelCase   (			self			:				Tuple				,      __A			:				List[str]      ):
        __UpperCamelCase			=    ''.join(lowerCAmelCase__      )
        __UpperCamelCase			=    bytearray([self.byte_decoder[c] for c in text]      ).decode('utf-8'				,      errors=self.errors      )
        return text






 def 			_lowerCamelCase   (			self			:				Dict				,      __A			:				str				,      __A			:				Optional[str] = None      ):




        if not os.path.isdir(lowerCAmelCase__      ):
               logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''      )
               return
        __UpperCamelCase			=    os.path.join(
            lowerCAmelCase__				,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
        __UpperCamelCase			=    os.path.join(
            lowerCAmelCase__				,      (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']      )

        with open(lowerCAmelCase__				,      'w'				,      encoding='utf-8'      ) as f:
               f.write(json.dumps(self.encoder				,      indent=2				,      sort_keys=lowerCAmelCase__				,      ensure_ascii=lowerCAmelCase__      ) + '\n'      )

        __UpperCamelCase			=    0
        with open(lowerCAmelCase__				,      'w'				,      encoding='utf-8'      ) as writer:
               writer.write('#version: 0.2\n'      )
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,      key=lambda __A      : kv[1]      ):
                      if index != token_index:
                             logger.warning(
                                 f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                 ' Please check that the tokenizer is not corrupted!'      )
                             __UpperCamelCase			=    token_index
                      writer.write(' '.join(lowerCAmelCase__      ) + '\n'      )
                      index += 1

        return vocab_file, merge_file






 def 			_lowerCamelCase   (			self			:				Dict				,      __A			:				List[int]				,      __A			:				Optional[List[int]] = None      ):




        if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        __UpperCamelCase			=    [self.cls_token_id]
        __UpperCamelCase			=    [self.sep_token_id]
        return cls + token_ids_a + sep + sep + token_ids_a + sep






 def 			_lowerCamelCase   (			self			:				Dict				,      __A			:				List[int]				,      __A			:				Optional[List[int]] = None				,      __A			:				bool = False      ):




        if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=lowerCAmelCase__				,      token_ids_a=lowerCAmelCase__				,      already_has_special_tokens=lowerCAmelCase__      )

        if token_ids_a is None:
               return [1] + ([0] * len(lowerCAmelCase__      )) + [1]
        return [1] + ([0] * len(lowerCAmelCase__      )) + [1, 1] + ([0] * len(lowerCAmelCase__      )) + [1]






 def 			_lowerCamelCase   (			self			:				Optional[int]				,      __A			:				List[int]				,      __A			:				Optional[List[int]] = None      ):
        __UpperCamelCase			=    [self.sep_token_id]
        __UpperCamelCase			=    [self.cls_token_id]

        if token_ids_a is None:
               return len(cls + token_ids_a + sep      ) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]






 def 			_lowerCamelCase   (			self			:				Optional[int]				,      __A			:				Tuple				,      __A			:				Tuple=False				,      **__A			:				Optional[Any]      ):
        __UpperCamelCase			=    kwargs.pop('add_prefix_space'				,      self.add_prefix_space      )
        if (is_split_into_words or add_prefix_space) and (len(lowerCAmelCase__      ) > 0 and not text[0].isspace()):
               __UpperCamelCase			=    ' ' + text
        return (text, kwargs)







'''simple docstring'''

from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowercase__		:     Tuple			=							{'configuration_mmbt': ['MMBTConfig']}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowercase__		:     int			=							['MMBTForClassification', 'MMBTModel', 'ModalEmbeddings']


if TYPE_CHECKING:
   from .configuration_mmbt import MMBTConfig

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_mmbt import MMBTForClassification, MMBTModel, ModalEmbeddings

else:
   import sys

   lowercase__		:     List[Any]			=							_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



