






"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


_lowerCAmelCase	:	List[str]				=      logging.get_logger(__name__)


_lowerCAmelCase	:	Optional[Any]				=      {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

# See all BART models at https://huggingface.co/models?filter=bart
_lowerCAmelCase	:	int				=      {
    """vocab_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/vocab.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/vocab.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/merges.txt""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/merges.txt""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """facebook/bart-base""": """https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json""",
        """facebook/bart-large""": """https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json""",
        """facebook/bart-large-mnli""": """https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json""",
        """facebook/bart-large-cnn""": """https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json""",
        """facebook/bart-large-xsum""": """https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json""",
        """yjernite/bart_eli5""": """https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json""",
    },
}

_lowerCAmelCase	:	Tuple				=      {
    """facebook/bart-base""": 1_024,
    """facebook/bart-large""": 1_024,
    """facebook/bart-large-mnli""": 1_024,
    """facebook/bart-large-cnn""": 1_024,
    """facebook/bart-large-xsum""": 1_024,
    """yjernite/bart_eli5""": 1_024,
}




class 			lowerCAmelCase__			(    __magic_name__   ):
   SCREAMING_SNAKE_CASE_    =VOCAB_FILES_NAMES
   SCREAMING_SNAKE_CASE_    =PRETRAINED_VOCAB_FILES_MAP
   SCREAMING_SNAKE_CASE_    =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   SCREAMING_SNAKE_CASE_    =['''input_ids''', '''attention_mask''']
   SCREAMING_SNAKE_CASE_    =BartTokenizer
   def __init__(		self      :	Tuple							,			snake_case__      :	Tuple=None							,			snake_case__      :	Tuple=None							,			snake_case__      :	Any=None							,			snake_case__      :	Tuple="replace"							,			snake_case__      :	Optional[Any]="<s>"							,			snake_case__      :	List[str]="</s>"							,			snake_case__      :	Union[str, Any]="</s>"							,			snake_case__      :	Union[str, Any]="<s>"							,			snake_case__      :	List[str]="<unk>"							,			snake_case__      :	Any="<pad>"							,			snake_case__      :	Optional[int]="<mask>"							,			snake_case__      :	Union[str, Any]=False							,			snake_case__      :	Union[str, Any]=True							,			**snake_case__      :	Optional[int]							,			):


        '''simple docstring'''
        super().__init__(
            snake_case__							,			snake_case__							,			tokenizer_file=snake_case__							,			errors=snake_case__							,			bos_token=snake_case__							,			eos_token=snake_case__							,			sep_token=snake_case__							,			cls_token=snake_case__							,			unk_token=snake_case__							,			pad_token=snake_case__							,			mask_token=snake_case__							,			add_prefix_space=snake_case__							,			trim_offsets=snake_case__							,			**snake_case__							,			)

        UpperCAmelCase__						:    str							=							json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
        if pre_tok_state.get("add_prefix_space"							,			snake_case__							) != add_prefix_space:
             UpperCAmelCase__						:    int							=							getattr(snake_case__							,			pre_tok_state.pop("type"							)							)
             UpperCAmelCase__						:    Union[str, Any]							=							add_prefix_space
             UpperCAmelCase__						:    List[str]							=							pre_tok_class(**snake_case__							)

        UpperCAmelCase__						:    str							=							add_prefix_space

        # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
        UpperCAmelCase__						:    str							=							"post_processor"
        UpperCAmelCase__						:    str							=							getattr(self.backend_tokenizer							,			snake_case__							,			snake_case__							)
        if tokenizer_component_instance:
             UpperCAmelCase__						:    Union[str, Any]							=							json.loads(tokenizer_component_instance.__getstate__()							)

             # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
             if "sep" in state:
                  UpperCAmelCase__						:    List[Any]							=							tuple(state["sep"]							)
             if "cls" in state:
                  UpperCAmelCase__						:    Any							=							tuple(state["cls"]							)

             UpperCAmelCase__						:    Union[str, Any]							=							False

             if state.get("add_prefix_space"							,			snake_case__							) != add_prefix_space:
                  UpperCAmelCase__						:    Union[str, Any]							=							add_prefix_space
                  UpperCAmelCase__						:    Any							=							True

             if state.get("trim_offsets"							,			snake_case__							) != trim_offsets:
                  UpperCAmelCase__						:    List[Any]							=							trim_offsets
                  UpperCAmelCase__						:    str							=							True

             if changes_to_apply:
                  UpperCAmelCase__						:    List[str]							=							getattr(snake_case__							,			state.pop("type"							)							)
                  UpperCAmelCase__						:    str							=							component_class(**snake_case__							)
                  setattr(self.backend_tokenizer							,			snake_case__							,			snake_case__							)
   @property
   def       __a							(		self      :	List[Any]							):


        '''simple docstring'''
        if self._mask_token is None:
             if self.verbose:
                  logger.error("Using mask_token, but it is not set yet."							)
             return None
        return str(self._mask_token							)
   @mask_token.setter
   def       __a							(		self      :	Optional[int]							,			snake_case__      :	Union[str, Any]							):


        '''simple docstring'''
        UpperCAmelCase__						:    int							=							AddedToken(snake_case__							,			lstrip=snake_case__							,			rstrip=snake_case__							) if isinstance(snake_case__							,			snake_case__							) else value
        UpperCAmelCase__						:    Any							=							value
   def       __a							(		self      :	Optional[Any]							,			*snake_case__      :	str							,			**snake_case__      :	Tuple							):


        '''simple docstring'''
        UpperCAmelCase__						:    Tuple							=							kwargs.get("is_split_into_words"							,			snake_case__							)

        if is_split_into_words and not self.add_prefix_space:
             raise ValueError(
                 f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
                 "to use it with pretokenized inputs."							)

        return super()._batch_encode_plus(*snake_case__							,			**snake_case__							)
   def       __a							(		self      :	List[Any]							,			*snake_case__      :	Union[str, Any]							,			**snake_case__      :	Any							):


        '''simple docstring'''
        UpperCAmelCase__						:    List[Any]							=							kwargs.get("is_split_into_words"							,			snake_case__							)

        if is_split_into_words and not self.add_prefix_space:
             raise ValueError(
                 f'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
                 "to use it with pretokenized inputs."							)

        return super()._encode_plus(*snake_case__							,			**snake_case__							)
   def       __a							(		self      :	Union[str, Any]							,			snake_case__      :	str							,			snake_case__      :	Optional[str] = None							):


        '''simple docstring'''
        UpperCAmelCase__						:    Tuple							=							self._tokenizer.model.save(snake_case__							,			name=snake_case__							)
        return tuple(snake_case__							)
   def       __a							(		self      :	str							,			snake_case__      :	Optional[int]							,			snake_case__      :	List[str]=None							):


        '''simple docstring'''
        UpperCAmelCase__						:    str							=							[self.bos_token_id] + token_ids_a + [self.eos_token_id]
        if token_ids_a is None:
             return output

        return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]




   def       __a							(		self      :	Union[str, Any]							,			snake_case__      :	List[int]							,			snake_case__      :	Optional[List[int]] = None							):


        '''simple docstring'''
        UpperCAmelCase__						:    Optional[int]							=							[self.sep_token_id]
        UpperCAmelCase__						:    List[Any]							=							[self.cls_token_id]

        if token_ids_a is None:
             return len(cls + token_ids_a + sep							) * [0]
        return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]










"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_lowerCAmelCase	:	Tuple				=      {
    """configuration_lxmert""": ["""LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """LxmertConfig"""],
    """tokenization_lxmert""": ["""LxmertTokenizer"""],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase	:	int				=      ["""LxmertTokenizerFast"""]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase	:	Tuple				=      [
     """LxmertEncoder""",
     """LxmertForPreTraining""",
     """LxmertForQuestionAnswering""",
     """LxmertModel""",
     """LxmertPreTrainedModel""",
     """LxmertVisualFeatureEncoder""",
     """LxmertXLayer""",
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _lowerCAmelCase	:	Optional[int]				=      [
     """TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """TFLxmertForPreTraining""",
     """TFLxmertMainLayer""",
     """TFLxmertModel""",
     """TFLxmertPreTrainedModel""",
     """TFLxmertVisualFeatureEncoder""",
 ]


if TYPE_CHECKING:
 from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig
 from .tokenization_lxmert import LxmertTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_lxmert_fast import LxmertTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_lxmert import (
      LxmertEncoder,
      LxmertForPreTraining,
      LxmertForQuestionAnswering,
      LxmertModel,
      LxmertPreTrainedModel,
      LxmertVisualFeatureEncoder,
      LxmertXLayer,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_lxmert import (
      TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFLxmertForPreTraining,
      TFLxmertMainLayer,
      TFLxmertModel,
      TFLxmertPreTrainedModel,
      TFLxmertVisualFeatureEncoder,
  )

else:
 import sys

 _lowerCAmelCase	:	Any				=      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


