


'''simple docstring'''

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


lowercase__				     =		logging.get_logger(__name__)


lowercase__				     =		{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

# See all BART models at https://huggingface.co/models?filter=bart
lowercase__				     =		{
    "vocab_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/vocab.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/vocab.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json",
    },
    "merges_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/merges.txt",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/merges.txt",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json",
    },
}

lowercase__				     =		{
    "facebook/bart-base": 1_024,
    "facebook/bart-large": 1_024,
    "facebook/bart-large-mnli": 1_024,
    "facebook/bart-large-cnn": 1_024,
    "facebook/bart-large-xsum": 1_024,
    "yjernite/bart_eli5": 1_024,
}





class 						__SCREAMING_SNAKE_CASE					(      lowerCamelCase__ ):
     UpperCAmelCase 					=				VOCAB_FILES_NAMES
     UpperCAmelCase 					=				PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase 					=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase 					=				['''input_ids''', '''attention_mask''']
     UpperCAmelCase 					=				BartTokenizer






     def __init__(			self     ,							__UpperCamelCase=None     ,							__UpperCamelCase=None     ,							__UpperCamelCase=None     ,							__UpperCamelCase="replace"     ,							__UpperCamelCase="<s>"     ,							__UpperCamelCase="</s>"     ,							__UpperCamelCase="</s>"     ,							__UpperCamelCase="<s>"     ,							__UpperCamelCase="<unk>"     ,							__UpperCamelCase="<pad>"     ,							__UpperCamelCase="<mask>"     ,							__UpperCamelCase=False     ,							__UpperCamelCase=True     ,							**__UpperCamelCase     ,							)       ->    List[str]:
          super().__init__(
              __UpperCamelCase     ,							__UpperCamelCase     ,							tokenizer_file=__UpperCamelCase     ,							errors=__UpperCamelCase     ,							bos_token=__UpperCamelCase     ,							eos_token=__UpperCamelCase     ,							sep_token=__UpperCamelCase     ,							cls_token=__UpperCamelCase     ,							unk_token=__UpperCamelCase     ,							pad_token=__UpperCamelCase     ,							mask_token=__UpperCamelCase     ,							add_prefix_space=__UpperCamelCase     ,							trim_offsets=__UpperCamelCase     ,							**__UpperCamelCase     ,							)

          _a						=		json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
          if pre_tok_state.get("add_prefix_space"     ,							__UpperCamelCase		) != add_prefix_space:
               _a						=		getattr(__UpperCamelCase     ,							pre_tok_state.pop("type"		)		)
               _a						=		add_prefix_space
               _a						=		pre_tok_class(**__UpperCamelCase		)

          _a						=		add_prefix_space

          # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
          _a						=		"post_processor"
          _a						=		getattr(self.backend_tokenizer     ,							__UpperCamelCase     ,							__UpperCamelCase		)
          if tokenizer_component_instance:
               _a						=		json.loads(tokenizer_component_instance.__getstate__()		)

               # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
               if "sep" in state:
                    _a						=		tuple(state["sep"]		)
               if "cls" in state:
                    _a						=		tuple(state["cls"]		)

               _a						=		False

               if state.get("add_prefix_space"     ,							__UpperCamelCase		) != add_prefix_space:
                    _a						=		add_prefix_space
                    _a						=		True

               if state.get("trim_offsets"     ,							__UpperCamelCase		) != trim_offsets:
                    _a						=		trim_offsets
                    _a						=		True

               if changes_to_apply:
                    _a						=		getattr(__UpperCamelCase     ,							state.pop("type"		)		)
                    _a						=		component_class(**__UpperCamelCase		)
                    setattr(self.backend_tokenizer     ,							__UpperCamelCase     ,							__UpperCamelCase		)






     @property
     def  a_					(			self		)       ->    str:
          if self._mask_token is None:
               if self.verbose:
                    logger.error("Using mask_token, but it is not set yet."		)
               return None
          return str(self._mask_token		)






     @mask_token.setter
     def  a_					(			self     ,							__UpperCamelCase		)       ->    Any:
          _a						=		AddedToken(__UpperCamelCase     ,							lstrip=__UpperCamelCase     ,							rstrip=__UpperCamelCase		) if isinstance(__UpperCamelCase     ,							__UpperCamelCase		) else value
          _a						=		value






     def  a_					(			self     ,							*__UpperCamelCase     ,							**__UpperCamelCase		)       ->    BatchEncoding:
          _a						=		kwargs.get("is_split_into_words"     ,							__UpperCamelCase		)

          if is_split_into_words and not self.add_prefix_space:
               raise ValueError(
                   f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
                   "to use it with pretokenized inputs."		)

          return super()._batch_encode_plus(*__UpperCamelCase     ,							**__UpperCamelCase		)






     def  a_					(			self     ,							*__UpperCamelCase     ,							**__UpperCamelCase		)       ->    BatchEncoding:
          _a						=		kwargs.get("is_split_into_words"     ,							__UpperCamelCase		)

          if is_split_into_words and not self.add_prefix_space:
               raise ValueError(
                   f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
                   "to use it with pretokenized inputs."		)

          return super()._encode_plus(*__UpperCamelCase     ,							**__UpperCamelCase		)






     def  a_					(			self     ,							__UpperCamelCase     ,							__UpperCamelCase = None		)       ->    Tuple[str]:
          _a						=		self._tokenizer.model.save(__UpperCamelCase     ,							name=__UpperCamelCase		)
          return tuple(__UpperCamelCase		)






     def  a_					(			self     ,							__UpperCamelCase     ,							__UpperCamelCase=None		)       ->    Tuple:
          _a						=		[self.bos_token_id] + token_ids_a + [self.eos_token_id]
          if token_ids_a is None:
               return output

          return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






     def  a_					(			self     ,							__UpperCamelCase     ,							__UpperCamelCase = None		)       ->    List[int]:
          _a						=		[self.sep_token_id]
          _a						=		[self.cls_token_id]

          if token_ids_a is None:
               return len(cls + token_ids_a + sep		) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]





'''simple docstring'''

from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowercase__				     =		{
    "configuration_mctct": ["MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP", "MCTCTConfig"],
    "feature_extraction_mctct": ["MCTCTFeatureExtractor"],
    "processing_mctct": ["MCTCTProcessor"],
}


try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   lowercase__				     =		[
       "MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST",
       "MCTCTForCTC",
       "MCTCTModel",
       "MCTCTPreTrainedModel",
   ]


if TYPE_CHECKING:
   from .configuration_mctct import MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP, MCTCTConfig
   from .feature_extraction_mctct import MCTCTFeatureExtractor
   from .processing_mctct import MCTCTProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_mctct import MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST, MCTCTForCTC, MCTCTModel, MCTCTPreTrainedModel

else:
   import sys

   lowercase__				     =		_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

