



"""simple docstring"""






import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


snake_case   					=				logging.get_logger(__name__)


snake_case   					=				{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

snake_case   					=				{
    '''vocab_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
    },
}

snake_case   					=				{
    '''allenai/longformer-base-4096''': 4_0_9_6,
    '''allenai/longformer-large-4096''': 4_0_9_6,
    '''allenai/longformer-large-4096-finetuned-triviaqa''': 4_0_9_6,
    '''allenai/longformer-base-4096-extra.pos.embd.only''': 4_0_9_6,
    '''allenai/longformer-large-4096-extra.pos.embd.only''': 4_0_9_6,
}




@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 			snake_case		(		)						->				Tuple:
  _snake_case   	=      (
      list(range(ord('''!'''	)  ,						ord('''~'''	) + 1	)	) + list(range(ord('''¡'''	)  ,						ord('''¬'''	) + 1	)	) + list(range(ord('''®'''	)  ,						ord('''ÿ'''	) + 1	)	)
  )
  _snake_case   	=      bs[:]
  _snake_case   	=      0
  for b in range(2**8	):
    if b not in bs:
      bs.append(lowerCAmelCase_	)
      cs.append(2**8 + n	)
      n += 1
  _snake_case   	=      [chr(lowerCAmelCase_	) for n in cs]
  return dict(zip(lowerCAmelCase_  ,						lowerCAmelCase_	)	)




def 			snake_case		(		lowerCAmelCase_	)						->				Optional[Any]:
  _snake_case   	=      set()
  _snake_case   	=      word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)	)
    _snake_case   	=      char
  return pairs







class     UpperCAmelCase    (	__SCREAMING_SNAKE_CASE  ):
   A__		:	Dict           =  VOCAB_FILES_NAMES
   A__		:	Union[str, Any]           =  PRETRAINED_VOCAB_FILES_MAP
   A__		:	Optional[int]           =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   A__		:	Optional[int]           =  ['''input_ids''', '''attention_mask''']



   def __init__(       self    :	Optional[int]							,       __lowerCamelCase    :	Union[str, Any]							,       __lowerCamelCase    :	Optional[int]							,       __lowerCamelCase    :	Optional[Any]="replace"							,       __lowerCamelCase    :	Union[str, Any]="<s>"							,       __lowerCamelCase    :	Optional[int]="</s>"							,       __lowerCamelCase    :	List[Any]="</s>"							,       __lowerCamelCase    :	str="<s>"							,       __lowerCamelCase    :	Dict="<unk>"							,       __lowerCamelCase    :	Any="<pad>"							,       __lowerCamelCase    :	Optional[Any]="<mask>"							,       __lowerCamelCase    :	List[str]=False							,       **__lowerCamelCase    :	Optional[Any]							,       ):



     """simple docstring"""
     _snake_case   	=      AddedToken(__lowerCamelCase							,       lstrip=__lowerCamelCase							,       rstrip=__lowerCamelCase				) if isinstance(__lowerCamelCase							,       __lowerCamelCase				) else bos_token
     _snake_case   	=      AddedToken(__lowerCamelCase							,       lstrip=__lowerCamelCase							,       rstrip=__lowerCamelCase				) if isinstance(__lowerCamelCase							,       __lowerCamelCase				) else eos_token
     _snake_case   	=      AddedToken(__lowerCamelCase							,       lstrip=__lowerCamelCase							,       rstrip=__lowerCamelCase				) if isinstance(__lowerCamelCase							,       __lowerCamelCase				) else sep_token
     _snake_case   	=      AddedToken(__lowerCamelCase							,       lstrip=__lowerCamelCase							,       rstrip=__lowerCamelCase				) if isinstance(__lowerCamelCase							,       __lowerCamelCase				) else cls_token
     _snake_case   	=      AddedToken(__lowerCamelCase							,       lstrip=__lowerCamelCase							,       rstrip=__lowerCamelCase				) if isinstance(__lowerCamelCase							,       __lowerCamelCase				) else unk_token
     _snake_case   	=      AddedToken(__lowerCamelCase							,       lstrip=__lowerCamelCase							,       rstrip=__lowerCamelCase				) if isinstance(__lowerCamelCase							,       __lowerCamelCase				) else pad_token

     # Mask token behave like a normal word, i.e. include the space before it
     _snake_case   	=      AddedToken(__lowerCamelCase							,       lstrip=__lowerCamelCase							,       rstrip=__lowerCamelCase				) if isinstance(__lowerCamelCase							,       __lowerCamelCase				) else mask_token

     super().__init__(
         errors=__lowerCamelCase							,       bos_token=__lowerCamelCase							,       eos_token=__lowerCamelCase							,       unk_token=__lowerCamelCase							,       sep_token=__lowerCamelCase							,       cls_token=__lowerCamelCase							,       pad_token=__lowerCamelCase							,       mask_token=__lowerCamelCase							,       add_prefix_space=__lowerCamelCase							,       **__lowerCamelCase							,       )

     with open(__lowerCamelCase							,       encoding='''utf-8'''				) as vocab_handle:
       _snake_case   	=      json.load(__lowerCamelCase				)
     _snake_case   	=      {v: k for k, v in self.encoder.items()}
     _snake_case   	=      errors  # how to handle errors in decoding
     _snake_case   	=      bytes_to_unicode()
     _snake_case   	=      {v: k for k, v in self.byte_encoder.items()}
     with open(__lowerCamelCase							,       encoding='''utf-8'''				) as merges_handle:
       _snake_case   	=      merges_handle.read().split('''\n'''				)[1:-1]
     _snake_case   	=      [tuple(merge.split()				) for merge in bpe_merges]
     _snake_case   	=      dict(zip(__lowerCamelCase							,       range(len(__lowerCamelCase				)				)				)				)
     _snake_case   	=      {}
     _snake_case   	=      add_prefix_space

     # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
     _snake_case   	=      re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''				)



   @property
   def    __UpperCAmelCase    (       self    :	List[Any]				):



     """simple docstring"""
     return len(self.encoder				)



   def    __UpperCAmelCase    (       self    :	int				):



     """simple docstring"""
     return dict(self.encoder							,       **self.added_tokens_encoder				)



   def    __UpperCAmelCase    (       self    :	int							,       __lowerCamelCase    :	List[Any]				):



     """simple docstring"""
     if token in self.cache:
       return self.cache[token]
     _snake_case   	=      tuple(__lowerCamelCase				)
     _snake_case   	=      get_pairs(__lowerCamelCase				)

     if not pairs:
       return token

     while True:
       _snake_case   	=      min(__lowerCamelCase							,       key=lambda __lowerCamelCase				: self.bpe_ranks.get(__lowerCamelCase							,       float('''inf'''				)				)				)
       if bigram not in self.bpe_ranks:
         break
       _snake_case	,     _snake_case   	=      bigram
       _snake_case   	=      []
       _snake_case   	=      0
       while i < len(__lowerCamelCase				):
         try:
           _snake_case   	=      word.index(__lowerCamelCase							,       __lowerCamelCase				)
         except ValueError:
           new_word.extend(word[i:]				)
           break
         else:
           new_word.extend(word[i:j]				)
           _snake_case   	=      j

         if word[i] == first and i < len(__lowerCamelCase				) - 1 and word[i + 1] == second:
           new_word.append(first + second				)
           i += 2
         else:
           new_word.append(word[i]				)
           i += 1
       _snake_case   	=      tuple(__lowerCamelCase				)
       _snake_case   	=      new_word
       if len(__lowerCamelCase				) == 1:
         break
       else:
         _snake_case   	=      get_pairs(__lowerCamelCase				)
     _snake_case   	=      ''' '''.join(__lowerCamelCase				)
     _snake_case   	=      word
     return word



   def    __UpperCAmelCase    (       self    :	Union[str, Any]							,       __lowerCamelCase    :	int				):



     """simple docstring"""
     _snake_case   	=      []
     for token in re.findall(self.pat							,       __lowerCamelCase				):
       _snake_case   	=      ''''''.join(
           self.byte_encoder[b] for b in token.encode('''utf-8'''				)				)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__lowerCamelCase				).split(''' '''				)				)
     return bpe_tokens



   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	Optional[Any]				):



     """simple docstring"""
     return self.encoder.get(__lowerCamelCase							,       self.encoder.get(self.unk_token				)				)



   def    __UpperCAmelCase    (       self    :	int							,       __lowerCamelCase    :	Dict				):



     """simple docstring"""
     return self.decoder.get(__lowerCamelCase				)



   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	Union[str, Any]				):



     """simple docstring"""
     _snake_case   	=      ''''''.join(__lowerCamelCase				)
     _snake_case   	=      bytearray([self.byte_decoder[c] for c in text]				).decode('''utf-8'''							,       errors=self.errors				)
     return text



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	str							,       __lowerCamelCase    :	Optional[str] = None				):



     """simple docstring"""
     if not os.path.isdir(__lowerCamelCase				):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
       return
     _snake_case   	=      os.path.join(
         __lowerCamelCase							,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)
     _snake_case   	=      os.path.join(
         __lowerCamelCase							,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']				)

     with open(__lowerCamelCase							,       '''w'''							,       encoding='''utf-8'''				) as f:
       f.write(json.dumps(self.encoder							,       indent=2							,       sort_keys=__lowerCamelCase							,       ensure_ascii=__lowerCamelCase				) + '''\n'''				)

     _snake_case   	=      0
     with open(__lowerCamelCase							,       '''w'''							,       encoding='''utf-8'''				) as writer:
       writer.write('''#version: 0.2\n'''				)
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,       key=lambda __lowerCamelCase				: kv[1]				):
         if index != token_index:
           logger.warning(
               f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
               ''' Please check that the tokenizer is not corrupted!'''				)
           _snake_case   	=      token_index
         writer.write(''' '''.join(__lowerCamelCase				) + '''\n'''				)
         index += 1

     return vocab_file, merge_file



   def    __UpperCAmelCase    (       self    :	Any							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None				):



     """simple docstring"""
     if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
     _snake_case   	=      [self.cls_token_id]
     _snake_case   	=      [self.sep_token_id]
     return cls + token_ids_a + sep + sep + token_ids_a + sep



   def    __UpperCAmelCase    (       self    :	Optional[Any]							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None							,       __lowerCamelCase    :	bool = False				):



     """simple docstring"""
     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=__lowerCamelCase							,       token_ids_a=__lowerCamelCase							,       already_has_special_tokens=__lowerCamelCase				)

     if token_ids_a is None:
       return [1] + ([0] * len(__lowerCamelCase				)) + [1]
     return [1] + ([0] * len(__lowerCamelCase				)) + [1, 1] + ([0] * len(__lowerCamelCase				)) + [1]



   def    __UpperCAmelCase    (       self    :	str							,       __lowerCamelCase    :	List[int]							,       __lowerCamelCase    :	Optional[List[int]] = None				):



     """simple docstring"""
     _snake_case   	=      [self.sep_token_id]
     _snake_case   	=      [self.cls_token_id]

     if token_ids_a is None:
       return len(cls + token_ids_a + sep				) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]





   def    __UpperCAmelCase    (       self    :	Tuple							,       __lowerCamelCase    :	Tuple							,       __lowerCamelCase    :	List[Any]=False							,       **__lowerCamelCase    :	Dict				):



     """simple docstring"""
     _snake_case   	=      kwargs.pop('''add_prefix_space'''							,       self.add_prefix_space				)
     if (is_split_into_words or add_prefix_space) and (len(__lowerCamelCase				) > 0 and not text[0].isspace()):
       _snake_case   	=      ''' ''' + text
     return (text, kwargs)












"""simple docstring"""
from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__UpperCamelCase     :						Union[str, Any]						       =  logging.get_logger(__name__)

__UpperCamelCase     :						Tuple						       =  {
    '''bert-base-uncased''': '''https://huggingface.co/bert-base-uncased/resolve/main/config.json''',
    '''bert-large-uncased''': '''https://huggingface.co/bert-large-uncased/resolve/main/config.json''',
    '''bert-base-cased''': '''https://huggingface.co/bert-base-cased/resolve/main/config.json''',
    '''bert-large-cased''': '''https://huggingface.co/bert-large-cased/resolve/main/config.json''',
    '''bert-base-multilingual-uncased''': '''https://huggingface.co/bert-base-multilingual-uncased/resolve/main/config.json''',
    '''bert-base-multilingual-cased''': '''https://huggingface.co/bert-base-multilingual-cased/resolve/main/config.json''',
    '''bert-base-chinese''': '''https://huggingface.co/bert-base-chinese/resolve/main/config.json''',
    '''bert-base-german-cased''': '''https://huggingface.co/bert-base-german-cased/resolve/main/config.json''',
    '''bert-large-uncased-whole-word-masking''': (
        '''https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/config.json'''
    ),
    '''bert-large-cased-whole-word-masking''': (
        '''https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/config.json'''
    ),
    '''bert-large-uncased-whole-word-masking-finetuned-squad''': (
        '''https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/config.json'''
    ),
    '''bert-large-cased-whole-word-masking-finetuned-squad''': (
        '''https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/config.json'''
    ),
    '''bert-base-cased-finetuned-mrpc''': '''https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/config.json''',
    '''bert-base-german-dbmdz-cased''': '''https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/config.json''',
    '''bert-base-german-dbmdz-uncased''': '''https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/config.json''',
    '''cl-tohoku/bert-base-japanese''': '''https://huggingface.co/cl-tohoku/bert-base-japanese/resolve/main/config.json''',
    '''cl-tohoku/bert-base-japanese-whole-word-masking''': (
        '''https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking/resolve/main/config.json'''
    ),
    '''cl-tohoku/bert-base-japanese-char''': (
        '''https://huggingface.co/cl-tohoku/bert-base-japanese-char/resolve/main/config.json'''
    ),
    '''cl-tohoku/bert-base-japanese-char-whole-word-masking''': (
        '''https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking/resolve/main/config.json'''
    ),
    '''TurkuNLP/bert-base-finnish-cased-v1''': (
        '''https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/config.json'''
    ),
    '''TurkuNLP/bert-base-finnish-uncased-v1''': (
        '''https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/config.json'''
    ),
    '''wietsedv/bert-base-dutch-cased''': '''https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/config.json''',
    # See all BERT models at https://huggingface.co/models?filter=bert
}







class       a							(   a__  ):
 snake_case__             =       '''bert'''



 def __init__(     self				,       _snake_case=3_05_22				,       _snake_case=7_68				,       _snake_case=12				,       _snake_case=12				,       _snake_case=30_72				,       _snake_case="gelu"				,       _snake_case=0.1				,       _snake_case=0.1				,       _snake_case=5_12				,       _snake_case=2				,       _snake_case=0.02				,       _snake_case=1E-12				,       _snake_case=0				,       _snake_case="absolute"				,       _snake_case=True				,       _snake_case=None				,       **_snake_case				,       ):







  """simple docstring"""






  super().__init__(pad_token_id=_snake_case				,       **_snake_case     )

  lowerCAmelCase    				= vocab_size
  lowerCAmelCase    				= hidden_size
  lowerCAmelCase    				= num_hidden_layers
  lowerCAmelCase    				= num_attention_heads
  lowerCAmelCase    				= hidden_act
  lowerCAmelCase    				= intermediate_size
  lowerCAmelCase    				= hidden_dropout_prob
  lowerCAmelCase    				= attention_probs_dropout_prob
  lowerCAmelCase    				= max_position_embeddings
  lowerCAmelCase    				= type_vocab_size
  lowerCAmelCase    				= initializer_range
  lowerCAmelCase    				= layer_norm_eps
  lowerCAmelCase    				= position_embedding_type
  lowerCAmelCase    				= use_cache
  lowerCAmelCase    				= classifier_dropout







class       a							(   a__  ):



 @property
 def    UpperCamelCase__  (     self     ):







  """simple docstring"""






  if self.task == "multiple-choice":
   lowerCAmelCase    				= {0: 'batch', 1: 'choice', 2: 'sequence'}
  else:
   lowerCAmelCase    				= {0: 'batch', 1: 'sequence'}
  return OrderedDict(
      [
          ('input_ids', dynamic_axis),
          ('attention_mask', dynamic_axis),
          ('token_type_ids', dynamic_axis),
      ]     )





