


"""simple docstring"""






import json
from typing import Dict, List, Optional, Tuple, Union

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding, EncodedInput
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_led import LEDTokenizer


__snake_case		=       logging.get_logger(__name__)


__snake_case		=       {"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

__snake_case		=       {
    "vocab_file": {
        "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json",
    },
    "merges_file": {
        "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "allenai/led-base-16384": "https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json",
    },
}

__snake_case		=       {
    "allenai/led-base-16384": 16384,
}


class     __lowerCamelCase  (					lowercase__				):

     '''simple docstring'''
     A_       :	Optional[int]    =		VOCAB_FILES_NAMES
     A_       :	Optional[Any]    =		PRETRAINED_VOCAB_FILES_MAP
     A_       :	Optional[Any]    =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     A_       :	List[str]    =		LEDTokenizer
     A_       :	List[str]    =		['input_ids', 'attention_mask']


     def __init__(      self    ,				__UpperCAmelCase=None    ,				__UpperCAmelCase=None    ,				__UpperCAmelCase=None    ,				__UpperCAmelCase="replace"    ,				__UpperCAmelCase="<s>"    ,				__UpperCAmelCase="</s>"    ,				__UpperCAmelCase="</s>"    ,				__UpperCAmelCase="<s>"    ,				__UpperCAmelCase="<unk>"    ,				__UpperCAmelCase="<pad>"    ,				__UpperCAmelCase="<mask>"    ,				__UpperCAmelCase=False    ,				__UpperCAmelCase=True    ,				**__UpperCAmelCase    ,				)       ->							Optional[Any]:
          super().__init__(
              __lowerCamelCase    ,				__lowerCamelCase    ,				tokenizer_file=__lowerCamelCase    ,				errors=__lowerCamelCase    ,				bos_token=__lowerCamelCase    ,				eos_token=__lowerCamelCase    ,				sep_token=__lowerCamelCase    ,				cls_token=__lowerCamelCase    ,				unk_token=__lowerCamelCase    ,				pad_token=__lowerCamelCase    ,				mask_token=__lowerCamelCase    ,				add_prefix_space=__lowerCamelCase    ,				trim_offsets=__lowerCamelCase    ,				**__lowerCamelCase    ,				)

          _a    	=				json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
          if pre_tok_state.get('''add_prefix_space'''    ,				__lowerCamelCase							) != add_prefix_space:
               _a    	=				getattr(__lowerCamelCase    ,				pre_tok_state.pop('''type'''							)							)
               _a    	=				add_prefix_space
               _a    	=				pre_tok_class(**__lowerCamelCase							)

          _a    	=				add_prefix_space

          # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
          _a    	=				"post_processor"
          _a    	=				getattr(self.backend_tokenizer    ,				__lowerCamelCase    ,				__lowerCamelCase							)
          if tokenizer_component_instance:
               _a    	=				json.loads(tokenizer_component_instance.__getstate__()							)

               # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
               if "sep" in state:
                    _a    	=				tuple(state['''sep''']							)
               if "cls" in state:
                    _a    	=				tuple(state['''cls''']							)

               _a    	=				False

               if state.get('''add_prefix_space'''    ,				__lowerCamelCase							) != add_prefix_space:
                    _a    	=				add_prefix_space
                    _a    	=				True

               if state.get('''trim_offsets'''    ,				__lowerCamelCase							) != trim_offsets:
                    _a    	=				trim_offsets
                    _a    	=				True

               if changes_to_apply:
                    _a    	=				getattr(__lowerCamelCase    ,				state.pop('''type'''							)							)
                    _a    	=				component_class(**__lowerCamelCase							)
                    setattr(self.backend_tokenizer    ,				__lowerCamelCase    ,				__lowerCamelCase							)


     @property
     # Copied from transformers.models.bart.tokenization_bart_fast.BartTokenizerFast.mask_token with BART->LED
     def 							_UpperCAmelCase      (      self							)       ->							List[Any]:



          if self._mask_token is None:
               if self.verbose:
                    logger.error('''Using mask_token, but it is not set yet.'''							)
               return None
          return str(self._mask_token							)


     @mask_token.setter
     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							int:
          _a    	=				AddedToken(__lowerCamelCase    ,				lstrip=__lowerCamelCase    ,				rstrip=__lowerCamelCase							) if isinstance(__lowerCamelCase    ,				__lowerCamelCase							) else value
          _a    	=				value


     def 							_UpperCAmelCase      (      self    ,				*__UpperCAmelCase    ,				**__UpperCAmelCase							)       ->							str:
          _a    	=				kwargs.get('''is_split_into_words'''    ,				__lowerCamelCase							)

          if is_split_into_words and not self.add_prefix_space:
               raise ValueError(
                   F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
                   '''to use it with pretokenized inputs.'''							)

          return super()._batch_encode_plus(*__lowerCamelCase    ,				**__lowerCamelCase							)


     def 							_UpperCAmelCase      (      self    ,				*__UpperCAmelCase    ,				**__UpperCAmelCase							)       ->							str:
          _a    	=				kwargs.get('''is_split_into_words'''    ,				__lowerCamelCase							)

          if is_split_into_words and not self.add_prefix_space:
               raise ValueError(
                   F'You need to instantiate {self.__class__.__name__} with add_prefix_space=True '
                   '''to use it with pretokenized inputs.'''							)

          return super()._encode_plus(*__lowerCamelCase    ,				**__lowerCamelCase							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							Union[str, Any]:
          _a    	=				self._tokenizer.model.save(__lowerCamelCase    ,				name=__lowerCamelCase							)
          return tuple(__lowerCamelCase							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase=None							)       ->							List[Any]:
          _a    	=				[self.bos_token_id] + token_ids_a + [self.eos_token_id]
          if token_ids_a is None:
               return output

          return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							Union[str, Any]:
          _a    	=				[self.sep_token_id]
          _a    	=				[self.cls_token_id]

          if token_ids_a is None:
               return len(cls + token_ids_a + sep							) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]





     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = PaddingStrategy.DO_NOT_PAD    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				)       ->							Optional[Any]:
          _a    	=				super()._pad(
              encoded_inputs=__lowerCamelCase    ,				max_length=__lowerCamelCase    ,				padding_strategy=__lowerCamelCase    ,				pad_to_multiple_of=__lowerCamelCase    ,				return_attention_mask=__lowerCamelCase    ,				)

          # Load from model defaults
          if return_attention_mask is None:
               _a    	=				"attention_mask" in self.model_input_names

          if return_attention_mask and "global_attention_mask" in encoded_inputs:
               _a    	=				encoded_inputs[self.model_input_names[0]]
               # `global_attention_mask` need to have the same length as other (sequential) inputs.
               _a    	=				len(encoded_inputs['''global_attention_mask''']							) != len(__lowerCamelCase							)

               if needs_to_be_padded:
                    _a    	=				len(__lowerCamelCase							) - len(encoded_inputs['''global_attention_mask''']							)

                    if self.padding_side == "right":
                         # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
                         _a    	=				(
                             encoded_inputs["global_attention_mask"] + [-1] * difference
                         )
                    elif self.padding_side == "left":
                         _a    	=				[-1] * difference + encoded_inputs[
                             "global_attention_mask"
                         ]
                    else:
                         raise ValueError('''Invalid padding strategy:''' + str(self.padding_side							)							)

          return encoded_inputs









"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__snake_case		=       {
    '''configuration_lxmert''': ['''LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''LxmertConfig'''],
    '''tokenization_lxmert''': ['''LxmertTokenizer'''],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __snake_case		=       ['''LxmertTokenizerFast''']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __snake_case		=       [
       '''LxmertEncoder''',
       '''LxmertForPreTraining''',
       '''LxmertForQuestionAnswering''',
       '''LxmertModel''',
       '''LxmertPreTrainedModel''',
       '''LxmertVisualFeatureEncoder''',
       '''LxmertXLayer''',
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   __snake_case		=       [
       '''TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''TFLxmertForPreTraining''',
       '''TFLxmertMainLayer''',
       '''TFLxmertModel''',
       '''TFLxmertPreTrainedModel''',
       '''TFLxmertVisualFeatureEncoder''',
   ]


if TYPE_CHECKING:
   from .configuration_lxmert import LXMERT_PRETRAINED_CONFIG_ARCHIVE_MAP, LxmertConfig
   from .tokenization_lxmert import LxmertTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_lxmert_fast import LxmertTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_lxmert import (
          LxmertEncoder,
          LxmertForPreTraining,
          LxmertForQuestionAnswering,
          LxmertModel,
          LxmertPreTrainedModel,
          LxmertVisualFeatureEncoder,
          LxmertXLayer,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_lxmert import (
          TF_LXMERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFLxmertForPreTraining,
          TFLxmertMainLayer,
          TFLxmertModel,
          TFLxmertPreTrainedModel,
          TFLxmertVisualFeatureEncoder,
      )

else:
   import sys

   __snake_case		=       _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)