






import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase								=		logging.get_logger(__name__)


UpperCamelCase								=		{"vocab_file": "vocab.json", "merges_file": "merges.txt"}

UpperCamelCase								=		{
    "vocab_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
    },
}

UpperCamelCase								=		{
    "allenai/longformer-base-4096": 4_096,
    "allenai/longformer-large-4096": 4_096,
    "allenai/longformer-large-4096-finetuned-triviaqa": 4_096,
    "allenai/longformer-base-4096-extra.pos.embd.only": 4_096,
    "allenai/longformer-large-4096-extra.pos.embd.only": 4_096,
}






@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def    __magic_name__    (	)     ->    Tuple:
							_lowercase							:   List[Any] 							=   (
							    list(range(ord('!'				)     ,     ord('~'				) + 1				)				) + list(range(ord('¡'				)     ,     ord('¬'				) + 1				)				) + list(range(ord('®'				)     ,     ord('ÿ'				) + 1				)				)
							)
							_lowercase							:   str 							=   bs[:]
							_lowercase							:   Optional[Any] 							=   0
							for b in range(2**8				):
														if b not in bs:
																					bs.append(SCREAMING_SNAKE_CASE				)
																					cs.append(2**8 + n				)
																					n += 1
							_lowercase							:   str 							=   [chr(SCREAMING_SNAKE_CASE				) for n in cs]
							return dict(zip(SCREAMING_SNAKE_CASE     ,     SCREAMING_SNAKE_CASE				)				)






def    __magic_name__    (	SCREAMING_SNAKE_CASE				)     ->    Any:
							_lowercase							:   int 							=   set()
							_lowercase							:   Union[str, Any] 							=   word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)				)
														_lowercase							:   str 							=   char
							return pairs






class 				lowerCAmelCase_	(		__snake_case			):
				_UpperCamelCase					:       List[Any]							       = VOCAB_FILES_NAMES
				_UpperCamelCase					:       Union[str, Any]							       = PRETRAINED_VOCAB_FILES_MAP
				_UpperCamelCase					:       Optional[Any]							       = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_UpperCamelCase					:       Dict							       = ["input_ids", "attention_mask"]


				def __init__(					self ,						_lowerCAmelCase ,						_lowerCAmelCase ,						_lowerCAmelCase="replace" ,						_lowerCAmelCase="<s>" ,						_lowerCAmelCase="</s>" ,						_lowerCAmelCase="</s>" ,						_lowerCAmelCase="<s>" ,						_lowerCAmelCase="<unk>" ,						_lowerCAmelCase="<pad>" ,						_lowerCAmelCase="<mask>" ,						_lowerCAmelCase=False ,						**_lowerCAmelCase ,						):
											_lowercase							:   List[str] 							=   AddedToken(_lowerCAmelCase ,						lstrip=_lowerCAmelCase ,						rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase ,						_lowerCAmelCase      ) else bos_token
											_lowercase							:   str 							=   AddedToken(_lowerCAmelCase ,						lstrip=_lowerCAmelCase ,						rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase ,						_lowerCAmelCase      ) else eos_token
											_lowercase							:   int 							=   AddedToken(_lowerCAmelCase ,						lstrip=_lowerCAmelCase ,						rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase ,						_lowerCAmelCase      ) else sep_token
											_lowercase							:   Union[str, Any] 							=   AddedToken(_lowerCAmelCase ,						lstrip=_lowerCAmelCase ,						rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase ,						_lowerCAmelCase      ) else cls_token
											_lowercase							:   Optional[int] 							=   AddedToken(_lowerCAmelCase ,						lstrip=_lowerCAmelCase ,						rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase ,						_lowerCAmelCase      ) else unk_token
											_lowercase							:   str 							=   AddedToken(_lowerCAmelCase ,						lstrip=_lowerCAmelCase ,						rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase ,						_lowerCAmelCase      ) else pad_token

											# Mask token behave like a normal word, i.e. include the space before it
											_lowercase							:   Dict 							=   AddedToken(_lowerCAmelCase ,						lstrip=_lowerCAmelCase ,						rstrip=_lowerCAmelCase      ) if isinstance(_lowerCAmelCase ,						_lowerCAmelCase      ) else mask_token

											super().__init__(
											    errors=_lowerCAmelCase ,						bos_token=_lowerCAmelCase ,						eos_token=_lowerCAmelCase ,						unk_token=_lowerCAmelCase ,						sep_token=_lowerCAmelCase ,						cls_token=_lowerCAmelCase ,						pad_token=_lowerCAmelCase ,						mask_token=_lowerCAmelCase ,						add_prefix_space=_lowerCAmelCase ,						**_lowerCAmelCase ,						)

											with open(_lowerCAmelCase ,						encoding='utf-8'      ) as vocab_handle:
																		_lowercase							:   Optional[int] 							=   json.load(_lowerCAmelCase      )
											_lowercase							:   int 							=   {v: k for k, v in self.encoder.items()}
											_lowercase							:   str 							=   errors  # how to handle errors in decoding
											_lowercase							:   int 							=   bytes_to_unicode()
											_lowercase							:   Dict 							=   {v: k for k, v in self.byte_encoder.items()}
											with open(_lowerCAmelCase ,						encoding='utf-8'      ) as merges_handle:
																		_lowercase							:   Dict 							=   merges_handle.read().split('\n'      )[1:-1]
											_lowercase							:   Tuple 							=   [tuple(merge.split()      ) for merge in bpe_merges]
											_lowercase							:   Union[str, Any] 							=   dict(zip(_lowerCAmelCase ,						range(len(_lowerCAmelCase      )      )      )      )
											_lowercase							:   Dict 							=   {}
											_lowercase							:   Optional[int] 							=   add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											_lowercase							:   Optional[Any] 							=   re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'      )


				@property
				def 						__a					(					self      ):
											return len(self.encoder      )


				def 						__a					(					self      ):
											return dict(self.encoder ,						**self.added_tokens_encoder      )


				def 						__a					(					self ,						_lowerCAmelCase      ):
											if token in self.cache:
																		return self.cache[token]
											_lowercase							:   Any 							=   tuple(_lowerCAmelCase      )
											_lowercase							:   List[str] 							=   get_pairs(_lowerCAmelCase      )

											if not pairs:
																		return token

											while True:
																		_lowercase							:   Union[str, Any] 							=   min(_lowerCAmelCase ,						key=lambda _lowerCAmelCase      : self.bpe_ranks.get(_lowerCAmelCase ,						float('inf'      )      )      )
																		if bigram not in self.bpe_ranks:
																									break
																		_lowercase							,							_lowercase							:   Dict 							=   bigram
																		_lowercase							:   int 							=   []
																		_lowercase							:   Union[str, Any] 							=   0
																		while i < len(_lowerCAmelCase      ):
																									try:
																																_lowercase							:   Optional[Any] 							=   word.index(_lowerCAmelCase ,						_lowerCAmelCase      )
																									except ValueError:
																																new_word.extend(word[i:]      )
																																break
																									else:
																																new_word.extend(word[i:j]      )
																																_lowercase							:   List[str] 							=   j

																									if word[i] == first and i < len(_lowerCAmelCase      ) - 1 and word[i + 1] == second:
																																new_word.append(first + second      )
																																i += 2
																									else:
																																new_word.append(word[i]      )
																																i += 1
																		_lowercase							:   List[str] 							=   tuple(_lowerCAmelCase      )
																		_lowercase							:   Any 							=   new_word
																		if len(_lowerCAmelCase      ) == 1:
																									break
																		else:
																									_lowercase							:   List[str] 							=   get_pairs(_lowerCAmelCase      )
											_lowercase							:   Tuple 							=   ' '.join(_lowerCAmelCase      )
											_lowercase							:   Any 							=   word
											return word


				def 						__a					(					self ,						_lowerCAmelCase      ):
											_lowercase							:   Tuple 							=   []
											for token in re.findall(self.pat ,						_lowerCAmelCase      ):
																		_lowercase							:   Optional[int] 							=   ''.join(
																		    self.byte_encoder[b] for b in token.encode('utf-8'      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																		bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCAmelCase      ).split(' '      )      )
											return bpe_tokens


				def 						__a					(					self ,						_lowerCAmelCase      ):
											return self.encoder.get(_lowerCAmelCase ,						self.encoder.get(self.unk_token      )      )


				def 						__a					(					self ,						_lowerCAmelCase      ):
											return self.decoder.get(_lowerCAmelCase      )


				def 						__a					(					self ,						_lowerCAmelCase      ):
											_lowercase							:   List[Any] 							=   ''.join(_lowerCAmelCase      )
											_lowercase							:   Dict 							=   bytearray([self.byte_decoder[c] for c in text]      ).decode('utf-8' ,						errors=self.errors      )
											return text


				def 						__a					(					self ,						_lowerCAmelCase ,						_lowerCAmelCase = None      ):
											if not os.path.isdir(_lowerCAmelCase      ):
																		logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
																		return
											_lowercase							:   List[Any] 							=   os.path.join(
											    _lowerCAmelCase ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )
											_lowercase							:   Optional[Any] 							=   os.path.join(
											    _lowerCAmelCase ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']      )

											with open(_lowerCAmelCase ,						'w' ,						encoding='utf-8'      ) as f:
																		f.write(json.dumps(self.encoder ,						indent=2 ,						sort_keys=_lowerCAmelCase ,						ensure_ascii=_lowerCAmelCase      ) + '\n'      )

											_lowercase							:   str 							=   0
											with open(_lowerCAmelCase ,						'w' ,						encoding='utf-8'      ) as writer:
																		writer.write('#version: 0.2\n'      )
																		for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,						key=lambda _lowerCAmelCase      : kv[1]      ):
																									if index != token_index:
																																logger.warning(
																																    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																																    ' Please check that the tokenizer is not corrupted!'      )
																																_lowercase							:   int 							=   token_index
																									writer.write(' '.join(_lowerCAmelCase      ) + '\n'      )
																									index += 1

											return vocab_file, merge_file


				def 						__a					(					self ,						_lowerCAmelCase ,						_lowerCAmelCase = None      ):
											if token_ids_a is None:
																		return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											_lowercase							:   Optional[int] 							=   [self.cls_token_id]
											_lowercase							:   Any 							=   [self.sep_token_id]
											return cls + token_ids_a + sep + sep + token_ids_a + sep


				def 						__a					(					self ,						_lowerCAmelCase ,						_lowerCAmelCase = None ,						_lowerCAmelCase = False      ):
											if already_has_special_tokens:
																		return super().get_special_tokens_mask(
																		    token_ids_a=_lowerCAmelCase ,						token_ids_a=_lowerCAmelCase ,						already_has_special_tokens=_lowerCAmelCase      )

											if token_ids_a is None:
																		return [1] + ([0] * len(_lowerCAmelCase      )) + [1]
											return [1] + ([0] * len(_lowerCAmelCase      )) + [1, 1] + ([0] * len(_lowerCAmelCase      )) + [1]


				def 						__a					(					self ,						_lowerCAmelCase ,						_lowerCAmelCase = None      ):
											_lowercase							:   Any 							=   [self.sep_token_id]
											_lowercase							:   int 							=   [self.cls_token_id]

											if token_ids_a is None:
																		return len(cls + token_ids_a + sep      ) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]


				def 						__a					(					self ,						_lowerCAmelCase ,						_lowerCAmelCase=False ,						**_lowerCAmelCase      ):
											_lowercase							:   Optional[int] 							=   kwargs.pop('add_prefix_space' ,						self.add_prefix_space      )
											if (is_split_into_words or add_prefix_space) and (len(_lowerCAmelCase      ) > 0 and not text[0].isspace()):
																		_lowercase							:   Dict 							=   ' ' + text
											return (text, kwargs)






"""simple docstring"""

from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_torch_available,
    is_vision_available,
)


SCREAMING_SNAKE_CASE_            =    {'''configuration_beit''': ['''BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''BeitConfig''', '''BeitOnnxConfig''']}

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   SCREAMING_SNAKE_CASE_            =    ['''BeitFeatureExtractor''']
   SCREAMING_SNAKE_CASE_            =    ['''BeitImageProcessor''']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   SCREAMING_SNAKE_CASE_            =    [
       '''BEIT_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''BeitForImageClassification''',
       '''BeitForMaskedImageModeling''',
       '''BeitForSemanticSegmentation''',
       '''BeitModel''',
       '''BeitPreTrainedModel''',
   ]


try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   SCREAMING_SNAKE_CASE_            =    [
       '''FlaxBeitForImageClassification''',
       '''FlaxBeitForMaskedImageModeling''',
       '''FlaxBeitModel''',
       '''FlaxBeitPreTrainedModel''',
   ]

if TYPE_CHECKING:
   from .configuration_beit import BEIT_PRETRAINED_CONFIG_ARCHIVE_MAP, BeitConfig, BeitOnnxConfig

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .feature_extraction_beit import BeitFeatureExtractor
      from .image_processing_beit import BeitImageProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_beit import (
          BEIT_PRETRAINED_MODEL_ARCHIVE_LIST,
          BeitForImageClassification,
          BeitForMaskedImageModeling,
          BeitForSemanticSegmentation,
          BeitModel,
          BeitPreTrainedModel,
      )

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_beit import (
          FlaxBeitForImageClassification,
          FlaxBeitForMaskedImageModeling,
          FlaxBeitModel,
          FlaxBeitPreTrainedModel,
      )


else:
   import sys

   SCREAMING_SNAKE_CASE_            =    _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)



