





"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


_lowerCamelCase    	=      logging.get_logger(__name__)


_lowerCamelCase    	=      {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

# See all BART models at https://huggingface.co/models?filter=bart
_lowerCamelCase    	=      {
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json''',
    },
}

_lowerCamelCase    	=      {
    '''facebook/bart-base''': 10_24,
    '''facebook/bart-large''': 10_24,
    '''facebook/bart-large-mnli''': 10_24,
    '''facebook/bart-large-cnn''': 10_24,
    '''facebook/bart-large-xsum''': 10_24,
    '''yjernite/bart_eli5''': 10_24,
}





class      snake_case						(  __UpperCAmelCase  ):
      lowerCamelCase__         =					VOCAB_FILES_NAMES
      lowerCamelCase__         =					PRETRAINED_VOCAB_FILES_MAP
      lowerCamelCase__         =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCamelCase__         =					["input_ids", "attention_mask"]
      lowerCamelCase__         =					BartTokenizer

      def __init__(     self  :int		,				_lowerCamelCase  :int=None		,				_lowerCamelCase  :Union[str, Any]=None		,				_lowerCamelCase  :int=None		,				_lowerCamelCase  :str="replace"		,				_lowerCamelCase  :str="<s>"		,				_lowerCamelCase  :Any="</s>"		,				_lowerCamelCase  :Optional[Any]="</s>"		,				_lowerCamelCase  :List[Any]="<s>"		,				_lowerCamelCase  :Optional[int]="<unk>"		,				_lowerCamelCase  :Optional[Any]="<pad>"		,				_lowerCamelCase  :Any="<mask>"		,				_lowerCamelCase  :List[str]=False		,				_lowerCamelCase  :Tuple=True		,				**_lowerCamelCase  :int		,				):
       super().__init__(
           SCREAMING_SNAKE_CASE__		,				SCREAMING_SNAKE_CASE__		,				tokenizer_file=SCREAMING_SNAKE_CASE__		,				errors=SCREAMING_SNAKE_CASE__		,				bos_token=SCREAMING_SNAKE_CASE__		,				eos_token=SCREAMING_SNAKE_CASE__		,				sep_token=SCREAMING_SNAKE_CASE__		,				cls_token=SCREAMING_SNAKE_CASE__		,				unk_token=SCREAMING_SNAKE_CASE__		,				pad_token=SCREAMING_SNAKE_CASE__		,				mask_token=SCREAMING_SNAKE_CASE__		,				add_prefix_space=SCREAMING_SNAKE_CASE__		,				trim_offsets=SCREAMING_SNAKE_CASE__		,				**SCREAMING_SNAKE_CASE__		,				)

       __SCREAMING_SNAKE_CASE					:     Dict   				=			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
       if pre_tok_state.get('''add_prefix_space'''		,				SCREAMING_SNAKE_CASE__		) != add_prefix_space:
        __SCREAMING_SNAKE_CASE					:     str   				=			getattr(SCREAMING_SNAKE_CASE__		,				pre_tok_state.pop('''type'''		)		)
        __SCREAMING_SNAKE_CASE					:     str   				=			add_prefix_space
        __SCREAMING_SNAKE_CASE					:     int   				=			pre_tok_class(**SCREAMING_SNAKE_CASE__		)

       __SCREAMING_SNAKE_CASE					:     Any   				=			add_prefix_space

       # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
       __SCREAMING_SNAKE_CASE					:     List[Any]   				=			'''post_processor'''
       __SCREAMING_SNAKE_CASE					:     Dict   				=			getattr(self.backend_tokenizer		,				SCREAMING_SNAKE_CASE__		,				SCREAMING_SNAKE_CASE__		)
       if tokenizer_component_instance:
        __SCREAMING_SNAKE_CASE					:     int   				=			json.loads(tokenizer_component_instance.__getstate__()		)

        # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
        if "sep" in state:
         __SCREAMING_SNAKE_CASE					:     List[Any]   				=			tuple(state['''sep''']		)
        if "cls" in state:
         __SCREAMING_SNAKE_CASE					:     List[str]   				=			tuple(state['''cls''']		)

        __SCREAMING_SNAKE_CASE					:     str   				=			False

        if state.get('''add_prefix_space'''		,				SCREAMING_SNAKE_CASE__		) != add_prefix_space:
         __SCREAMING_SNAKE_CASE					:     Optional[int]   				=			add_prefix_space
         __SCREAMING_SNAKE_CASE					:     Dict   				=			True

        if state.get('''trim_offsets'''		,				SCREAMING_SNAKE_CASE__		) != trim_offsets:
         __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			trim_offsets
         __SCREAMING_SNAKE_CASE					:     Tuple   				=			True

        if changes_to_apply:
         __SCREAMING_SNAKE_CASE					:     str   				=			getattr(SCREAMING_SNAKE_CASE__		,				state.pop('''type'''		)		)
         __SCREAMING_SNAKE_CASE					:     Dict   				=			component_class(**SCREAMING_SNAKE_CASE__		)
         setattr(self.backend_tokenizer		,				SCREAMING_SNAKE_CASE__		,				SCREAMING_SNAKE_CASE__		)

      @property
      def 		SCREAMING_SNAKE_CASE_     (     self  :str		):



       if self._mask_token is None:
        if self.verbose:
         logger.error('''Using mask_token, but it is not set yet.'''		)
        return None
       return str(self._mask_token		)

      @mask_token.setter
      def 		SCREAMING_SNAKE_CASE_     (     self  :List[Any]		,				_lowerCamelCase  :int		):
       __SCREAMING_SNAKE_CASE					:     Any   				=			AddedToken(SCREAMING_SNAKE_CASE__		,				lstrip=SCREAMING_SNAKE_CASE__		,				rstrip=SCREAMING_SNAKE_CASE__		) if isinstance(SCREAMING_SNAKE_CASE__		,				SCREAMING_SNAKE_CASE__		) else value
       __SCREAMING_SNAKE_CASE					:     str   				=			value

      def 		SCREAMING_SNAKE_CASE_     (     self  :str		,				*_lowerCamelCase  :Optional[int]		,				**_lowerCamelCase  :Dict		):
       __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			kwargs.get('''is_split_into_words'''		,				SCREAMING_SNAKE_CASE__		)

       if is_split_into_words and not self.add_prefix_space:
        raise ValueError(
            f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
            '''to use it with pretokenized inputs.'''		)

       return super()._batch_encode_plus(*SCREAMING_SNAKE_CASE__		,				**SCREAMING_SNAKE_CASE__		)

      def 		SCREAMING_SNAKE_CASE_     (     self  :Dict		,				*_lowerCamelCase  :List[Any]		,				**_lowerCamelCase  :List[str]		):
       __SCREAMING_SNAKE_CASE					:     Optional[Any]   				=			kwargs.get('''is_split_into_words'''		,				SCREAMING_SNAKE_CASE__		)

       if is_split_into_words and not self.add_prefix_space:
        raise ValueError(
            f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
            '''to use it with pretokenized inputs.'''		)

       return super()._encode_plus(*SCREAMING_SNAKE_CASE__		,				**SCREAMING_SNAKE_CASE__		)

      def 		SCREAMING_SNAKE_CASE_     (     self  :Optional[int]		,				_lowerCamelCase  :str		,				_lowerCamelCase  :Optional[str] = None		):
       __SCREAMING_SNAKE_CASE					:     List[str]   				=			self._tokenizer.model.save(SCREAMING_SNAKE_CASE__		,				name=SCREAMING_SNAKE_CASE__		)
       return tuple(SCREAMING_SNAKE_CASE__		)

      def 		SCREAMING_SNAKE_CASE_     (     self  :List[Any]		,				_lowerCamelCase  :Any		,				_lowerCamelCase  :Optional[int]=None		):
       __SCREAMING_SNAKE_CASE					:     Tuple   				=			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
       if token_ids_a is None:
        return output

       return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]



      def 		SCREAMING_SNAKE_CASE_     (     self  :Optional[Any]		,				_lowerCamelCase  :List[int]		,				_lowerCamelCase  :Optional[List[int]] = None		):
       __SCREAMING_SNAKE_CASE					:     Union[str, Any]   				=			[self.sep_token_id]
       __SCREAMING_SNAKE_CASE					:     Optional[Any]   				=			[self.cls_token_id]

       if token_ids_a is None:
        return len(cls + token_ids_a + sep		) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]



# Copyright 2023 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


_snake_case      = {
    '''configuration_efficientnet''': [
        '''EFFICIENTNET_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''EfficientNetConfig''',
        '''EfficientNetOnnxConfig''',
    ]
}

try:
 if not is_vision_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _snake_case      = ['''EfficientNetImageProcessor''']

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 _snake_case      = [
     '''EFFICIENTNET_PRETRAINED_MODEL_ARCHIVE_LIST''',
     '''EfficientNetForImageClassification''',
     '''EfficientNetModel''',
     '''EfficientNetPreTrainedModel''',
 ]

if TYPE_CHECKING:
 from .configuration_efficientnet import (
     EFFICIENTNET_PRETRAINED_CONFIG_ARCHIVE_MAP,
     EfficientNetConfig,
     EfficientNetOnnxConfig,
 )

 try:
  if not is_vision_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .image_processing_efficientnet import EfficientNetImageProcessor

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_efficientnet import (
      EFFICIENTNET_PRETRAINED_MODEL_ARCHIVE_LIST,
      EfficientNetForImageClassification,
      EfficientNetModel,
      EfficientNetPreTrainedModel,
  )

else:
 import sys

 _snake_case      = _LazyModule(__name__, globals()['''__file__'''], _import_structure)


