



"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


snake_case   					=				logging.get_logger(__name__)


snake_case   					=				{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

# See all BART models at https://huggingface.co/models?filter=bart
snake_case   					=				{
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json''',
    },
}

snake_case   					=				{
    '''facebook/bart-base''': 1_0_2_4,
    '''facebook/bart-large''': 1_0_2_4,
    '''facebook/bart-large-mnli''': 1_0_2_4,
    '''facebook/bart-large-cnn''': 1_0_2_4,
    '''facebook/bart-large-xsum''': 1_0_2_4,
    '''yjernite/bart_eli5''': 1_0_2_4,
}



class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):
   A__		:	Dict           =  VOCAB_FILES_NAMES
   A__		:	str           =  PRETRAINED_VOCAB_FILES_MAP
   A__		:	Union[str, Any]           =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A__		:	Any           =  ['''input_ids''', '''attention_mask''']
   A__		:	Optional[int]           =  BartTokenizer



   def __init__(       self    :	List[Any]							,       __lowerCamelCase    :	List[Any]=None							,       __lowerCamelCase    :	Dict=None							,       __lowerCamelCase    :	Any=None							,       __lowerCamelCase    :	Any="replace"							,       __lowerCamelCase    :	Any="<s>"							,       __lowerCamelCase    :	int="</s>"							,       __lowerCamelCase    :	List[str]="</s>"							,       __lowerCamelCase    :	Optional[Any]="<s>"							,       __lowerCamelCase    :	Optional[int]="<unk>"							,       __lowerCamelCase    :	Optional[int]="<pad>"							,       __lowerCamelCase    :	Any="<mask>"							,       __lowerCamelCase    :	Optional[Any]=False							,       __lowerCamelCase    :	int=True							,       **__lowerCamelCase    :	Dict							,       ):



     """simple docstring"""
     super().__init__(
         __lowerCamelCase							,       __lowerCamelCase							,       tokenizer_file=__lowerCamelCase							,       errors=__lowerCamelCase							,       bos_token=__lowerCamelCase							,       eos_token=__lowerCamelCase							,       sep_token=__lowerCamelCase							,       cls_token=__lowerCamelCase							,       unk_token=__lowerCamelCase							,       pad_token=__lowerCamelCase							,       mask_token=__lowerCamelCase							,       add_prefix_space=__lowerCamelCase							,       trim_offsets=__lowerCamelCase							,       **__lowerCamelCase							,       )

     _snake_case   	=      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
     if pre_tok_state.get('''add_prefix_space'''							,       __lowerCamelCase				) != add_prefix_space:
       _snake_case   	=      getattr(__lowerCamelCase							,       pre_tok_state.pop('''type'''				)				)
       _snake_case   	=      add_prefix_space
       _snake_case   	=      pre_tok_class(**__lowerCamelCase				)

     _snake_case   	=      add_prefix_space

     # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
     _snake_case   	=      '''post_processor'''
     _snake_case   	=      getattr(self.backend_tokenizer							,       __lowerCamelCase							,       __lowerCamelCase				)
     if tokenizer_component_instance:
       _snake_case   	=      json.loads(tokenizer_component_instance.__getstate__()				)

       # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
       if "sep" in state:
         _snake_case   	=      tuple(state['''sep''']				)
       if "cls" in state:
         _snake_case   	=      tuple(state['''cls''']				)

       _snake_case   	=      False

       if state.get('''add_prefix_space'''							,       __lowerCamelCase				) != add_prefix_space:
         _snake_case   	=      add_prefix_space
         _snake_case   	=      True

       if state.get('''trim_offsets'''							,       __lowerCamelCase				) != trim_offsets:
         _snake_case   	=      trim_offsets
         _snake_case   	=      True

       if changes_to_apply:
         _snake_case   	=      getattr(__lowerCamelCase							,       state.pop('''type'''				)				)
         _snake_case   	=      component_class(**__lowerCamelCase				)
         setattr(self.backend_tokenizer							,       __lowerCamelCase							,       __lowerCamelCase				)



   @property
   def    __UpperCAmelCase    (       self    :	Optional[int]				):



     """simple docstring"""
     if self._mask_token is None:
       if self.verbose:
         logger.error('''Using mask_token, but it is not set yet.'''				)
       return None
     return str(self._mask_token				)



   @mask_token.setter
   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	str				):



     """simple docstring"""
     _snake_case   	=      AddedToken(__lowerCamelCase							,       lstrip=__lowerCamelCase							,       rstrip=__lowerCamelCase				) if isinstance(__lowerCamelCase							,       __lowerCamelCase				) else value
     _snake_case   	=      value



   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       *__lowerCamelCase    :	List[str]							,       **__lowerCamelCase    :	Any				):



     """simple docstring"""
     _snake_case   	=      kwargs.get('''is_split_into_words'''							,       __lowerCamelCase				)

     if is_split_into_words and not self.add_prefix_space:
       raise ValueError(
           f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
           '''to use it with pretokenized inputs.'''				)

     return super()._batch_encode_plus(*__lowerCamelCase							,       **__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	List[Any]							,       *__lowerCamelCase    :	Union[str, Any]							,       **__lowerCamelCase    :	Optional[Any]				):



     """simple docstring"""
     _snake_case   	=      kwargs.get('''is_split_into_words'''							,       __lowerCamelCase				)

     if is_split_into_words and not self.add_prefix_space:
       raise ValueError(
           f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
           '''to use it with pretokenized inputs.'''				)

     return super()._encode_plus(*__lowerCamelCase							,       **__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Tuple							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	Optional[str] = None				):



     """simple docstring"""
     _snake_case   	=      self._tokenizer.model.save(__lowerCamelCase							,       name=__lowerCamelCase				)
     return tuple(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Tuple							,       __lowerCamelCase    :	Tuple							,       __lowerCamelCase    :	Any=None				):



     """simple docstring"""
     _snake_case   	=      [self.bos_token_id] + token_ids_a + [self.eos_token_id]
     if token_ids_a is None:
       return output

     return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None				):



     """simple docstring"""
     _snake_case   	=      [self.sep_token_id]
     _snake_case   	=      [self.cls_token_id]

     if token_ids_a is None:
       return len(cls + token_ids_a + sep				) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]










"""simple docstring"""






def 			snake_case		(		lowerCAmelCase_  ,						lowerCAmelCase_	)						->				int:
  return 1 if input_a == input_a else 0




def 			snake_case		(		)						->				None:
  assert xnor_gate(0  ,						0	) == 1
  assert xnor_gate(0  ,						1	) == 0
  assert xnor_gate(1  ,						0	) == 0
  assert xnor_gate(1  ,						1	) == 1


if __name__ == "__main__":
     print(xnor_gate(0, 0))
     print(xnor_gate(0, 1))
     print(xnor_gate(1, 0))
     print(xnor_gate(1, 1))





