





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_        =							logging.get_logger(__name__)


lowerCAmelCase_        =							{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

lowerCAmelCase_        =							{
    '''vocab_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'''
        ),
    },
    '''merges_file''': {
        '''allenai/longformer-base-4096''': '''https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt''',
        '''allenai/longformer-large-4096''': (
            '''https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-finetuned-triviaqa''': (
            '''https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-base-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
        '''allenai/longformer-large-4096-extra.pos.embd.only''': (
            '''https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'''
        ),
    },
}

lowerCAmelCase_        =							{
    '''allenai/longformer-base-4096''': 4_0_9_6,
    '''allenai/longformer-large-4096''': 4_0_9_6,
    '''allenai/longformer-large-4096-finetuned-triviaqa''': 4_0_9_6,
    '''allenai/longformer-base-4096-extra.pos.embd.only''': 4_0_9_6,
    '''allenai/longformer-large-4096-extra.pos.embd.only''': 4_0_9_6,
}






@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def 	lowerCamelCase_		(							)						->		List[str]:






				"""simple docstring"""


				snake_case_		:       str           =    (
				    list(range(ord('''!'''					)					,							ord('''~'''					) + 1					)					) + list(range(ord('''¡'''					)					,							ord('''¬'''					) + 1					)					) + list(range(ord('''®'''					)					,							ord('''ÿ'''					) + 1					)					)
				)
				snake_case_		:       Union[str, Any]           =    bs[:]
				snake_case_		:       Dict           =    0
				for b in range(2**8					):
								if b not in bs:
												bs.append(_UpperCamelCase					)
												cs.append(2**8 + n					)
												n += 1
				snake_case_		:       Any           =    [chr(_UpperCamelCase					) for n in cs]
				return dict(zip(_UpperCamelCase					,							_UpperCamelCase					)					)






def 	lowerCamelCase_		(							_UpperCamelCase					)						->		Dict:






				"""simple docstring"""


				snake_case_		:       List[str]           =    set()
				snake_case_		:       List[Any]           =    word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)					)
								snake_case_		:       Optional[int]           =    char
				return pairs





class    __lowerCAmelCase			(     _a	):
			lowerCamelCase_	:	Tuple			     = VOCAB_FILES_NAMES
			lowerCamelCase_	:	Optional[int]			     = PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase_	:	Any			     = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase_	:	List[Any]			     = ['''input_ids''', '''attention_mask''']






			def __init__(self   ,				__magic_name__   ,				__magic_name__   ,				__magic_name__="replace"   ,				__magic_name__="<s>"   ,				__magic_name__="</s>"   ,				__magic_name__="</s>"   ,				__magic_name__="<s>"   ,				__magic_name__="<unk>"   ,				__magic_name__="<pad>"   ,				__magic_name__="<mask>"   ,				__magic_name__=False   ,				**__magic_name__   ,				)	->  Union[str, Any]:


							'''simple docstring'''



							snake_case_		:       int           =    AddedToken(__magic_name__   ,				lstrip=__magic_name__   ,				rstrip=__magic_name__       ) if isinstance(__magic_name__   ,				__magic_name__       ) else bos_token
							snake_case_		:       List[Any]           =    AddedToken(__magic_name__   ,				lstrip=__magic_name__   ,				rstrip=__magic_name__       ) if isinstance(__magic_name__   ,				__magic_name__       ) else eos_token
							snake_case_		:       Dict           =    AddedToken(__magic_name__   ,				lstrip=__magic_name__   ,				rstrip=__magic_name__       ) if isinstance(__magic_name__   ,				__magic_name__       ) else sep_token
							snake_case_		:       Optional[int]           =    AddedToken(__magic_name__   ,				lstrip=__magic_name__   ,				rstrip=__magic_name__       ) if isinstance(__magic_name__   ,				__magic_name__       ) else cls_token
							snake_case_		:       Union[str, Any]           =    AddedToken(__magic_name__   ,				lstrip=__magic_name__   ,				rstrip=__magic_name__       ) if isinstance(__magic_name__   ,				__magic_name__       ) else unk_token
							snake_case_		:       Any           =    AddedToken(__magic_name__   ,				lstrip=__magic_name__   ,				rstrip=__magic_name__       ) if isinstance(__magic_name__   ,				__magic_name__       ) else pad_token

							# Mask token behave like a normal word, i.e. include the space before it
							snake_case_		:       str           =    AddedToken(__magic_name__   ,				lstrip=__magic_name__   ,				rstrip=__magic_name__       ) if isinstance(__magic_name__   ,				__magic_name__       ) else mask_token

							super().__init__(
							    errors=__magic_name__   ,				bos_token=__magic_name__   ,				eos_token=__magic_name__   ,				unk_token=__magic_name__   ,				sep_token=__magic_name__   ,				cls_token=__magic_name__   ,				pad_token=__magic_name__   ,				mask_token=__magic_name__   ,				add_prefix_space=__magic_name__   ,				**__magic_name__   ,				)

							with open(__magic_name__   ,				encoding='''utf-8'''       ) as vocab_handle:
											snake_case_		:       Optional[Any]           =    json.load(__magic_name__       )
							snake_case_		:       Optional[int]           =    {v: k for k, v in self.encoder.items()}
							snake_case_		:       Optional[Any]           =    errors  # how to handle errors in decoding
							snake_case_		:       Union[str, Any]           =    bytes_to_unicode()
							snake_case_		:       str           =    {v: k for k, v in self.byte_encoder.items()}
							with open(__magic_name__   ,				encoding='''utf-8'''       ) as merges_handle:
											snake_case_		:       Dict           =    merges_handle.read().split('''\n'''       )[1:-1]
							snake_case_		:       Optional[int]           =    [tuple(merge.split()       ) for merge in bpe_merges]
							snake_case_		:       Union[str, Any]           =    dict(zip(__magic_name__   ,				range(len(__magic_name__       )       )       )       )
							snake_case_		:       int           =    {}
							snake_case_		:       Tuple           =    add_prefix_space

							# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
							snake_case_		:       Optional[Any]           =    re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''       )






			@property
			def      lowerCamelCase    (self       )	->  Any:


							'''simple docstring'''



							return len(self.encoder       )






			def      lowerCamelCase    (self       )	->  Dict:


							'''simple docstring'''



							return dict(self.encoder   ,				**self.added_tokens_encoder       )






			def      lowerCamelCase    (self   ,				__magic_name__       )	->  Union[str, Any]:


							'''simple docstring'''



							if token in self.cache:
											return self.cache[token]
							snake_case_		:       str           =    tuple(__magic_name__       )
							snake_case_		:       Dict           =    get_pairs(__magic_name__       )

							if not pairs:
											return token

							while True:
											snake_case_		:       Optional[int]           =    min(__magic_name__   ,				key=lambda __magic_name__       : self.bpe_ranks.get(__magic_name__   ,				float('''inf'''       )       )       )
											if bigram not in self.bpe_ranks:
															break
											snake_case_     ,				snake_case_		:       List[Any]           =    bigram
											snake_case_		:       int           =    []
											snake_case_		:       int           =    0
											while i < len(__magic_name__       ):
															try:
																			snake_case_		:       Union[str, Any]           =    word.index(__magic_name__   ,				__magic_name__       )
															except ValueError:
																			new_word.extend(word[i:]       )
																			break
															else:
																			new_word.extend(word[i:j]       )
																			snake_case_		:       Any           =    j

															if word[i] == first and i < len(__magic_name__       ) - 1 and word[i + 1] == second:
																			new_word.append(first + second       )
																			i += 2
															else:
																			new_word.append(word[i]       )
																			i += 1
											snake_case_		:       Dict           =    tuple(__magic_name__       )
											snake_case_		:       List[str]           =    new_word
											if len(__magic_name__       ) == 1:
															break
											else:
															snake_case_		:       Optional[int]           =    get_pairs(__magic_name__       )
							snake_case_		:       List[str]           =    ''' '''.join(__magic_name__       )
							snake_case_		:       List[Any]           =    word
							return word






			def      lowerCamelCase    (self   ,				__magic_name__       )	->  str:


							'''simple docstring'''



							snake_case_		:       Tuple           =    []
							for token in re.findall(self.pat   ,				__magic_name__       ):
											snake_case_		:       Optional[Any]           =    ''''''.join(
											    self.byte_encoder[b] for b in token.encode('''utf-8'''       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__magic_name__       ).split(''' '''       )       )
							return bpe_tokens






			def      lowerCamelCase    (self   ,				__magic_name__       )	->  Any:


							'''simple docstring'''



							return self.encoder.get(__magic_name__   ,				self.encoder.get(self.unk_token       )       )






			def      lowerCamelCase    (self   ,				__magic_name__       )	->  Dict:


							'''simple docstring'''



							return self.decoder.get(__magic_name__       )






			def      lowerCamelCase    (self   ,				__magic_name__       )	->  Dict:


							'''simple docstring'''



							snake_case_		:       Tuple           =    ''''''.join(__magic_name__       )
							snake_case_		:       Dict           =    bytearray([self.byte_decoder[c] for c in text]       ).decode('''utf-8'''   ,				errors=self.errors       )
							return text






			def      lowerCamelCase    (self   ,				__magic_name__   ,				__magic_name__ = None       )	->  Tuple[str]:


							'''simple docstring'''



							if not os.path.isdir(__magic_name__       ):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
											return
							snake_case_		:       Dict           =    os.path.join(
							    __magic_name__   ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )
							snake_case_		:       Optional[int]           =    os.path.join(
							    __magic_name__   ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']       )

							with open(__magic_name__   ,				'''w'''   ,				encoding='''utf-8'''       ) as f:
											f.write(json.dumps(self.encoder   ,				indent=2   ,				sort_keys=__magic_name__   ,				ensure_ascii=__magic_name__       ) + '''\n'''       )

							snake_case_		:       List[str]           =    0
							with open(__magic_name__   ,				'''w'''   ,				encoding='''utf-8'''       ) as writer:
											writer.write('''#version: 0.2\n'''       )
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,				key=lambda __magic_name__       : kv[1]       ):
															if index != token_index:
																			logger.warning(
																			    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																			    ''' Please check that the tokenizer is not corrupted!'''       )
																			snake_case_		:       Optional[int]           =    token_index
															writer.write(''' '''.join(__magic_name__       ) + '''\n'''       )
															index += 1

							return vocab_file, merge_file






			def      lowerCamelCase    (self   ,				__magic_name__   ,				__magic_name__ = None       )	->  List[int]:


							'''simple docstring'''



							if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
							snake_case_		:       Any           =    [self.cls_token_id]
							snake_case_		:       List[str]           =    [self.sep_token_id]
							return cls + token_ids_a + sep + sep + token_ids_a + sep






			def      lowerCamelCase    (self   ,				__magic_name__   ,				__magic_name__ = None   ,				__magic_name__ = False       )	->  List[int]:


							'''simple docstring'''



							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=__magic_name__   ,				token_ids_a=__magic_name__   ,				already_has_special_tokens=__magic_name__       )

							if token_ids_a is None:
											return [1] + ([0] * len(__magic_name__       )) + [1]
							return [1] + ([0] * len(__magic_name__       )) + [1, 1] + ([0] * len(__magic_name__       )) + [1]






			def      lowerCamelCase    (self   ,				__magic_name__   ,				__magic_name__ = None       )	->  List[int]:


							'''simple docstring'''



							snake_case_		:       Any           =    [self.sep_token_id]
							snake_case_		:       Union[str, Any]           =    [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep       ) * [0]
							return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]






			def      lowerCamelCase    (self   ,				__magic_name__   ,				__magic_name__=False   ,				**__magic_name__       )	->  Dict:


							'''simple docstring'''



							snake_case_		:       Optional[int]           =    kwargs.pop('''add_prefix_space'''   ,				self.add_prefix_space       )
							if (is_split_into_words or add_prefix_space) and (len(__magic_name__       ) > 0 and not text[0].isspace()):
											snake_case_		:       Optional[Any]           =    ''' ''' + text
							return (text, kwargs)








import pytest
import requests

from datasets.utils.file_utils import http_head

from .utils import OfflineSimulationMode, RequestWouldHangIndefinitelyError, offline

@pytest.mark.integration
def   __lowercase       (			):







				"""simple docstring"""






				with offline(OfflineSimulationMode.CONNECTION_TIMES_OUT			):
								with pytest.raises(snake_case			):
												requests.request('''GET''',	'''https://huggingface.co'''			)
								with pytest.raises(requests.exceptions.ConnectTimeout			):
												requests.request('''GET''',	'''https://huggingface.co''',	timeout=1.0			)

@pytest.mark.integration
def   __lowercase       (			):







				"""simple docstring"""






				with offline(OfflineSimulationMode.CONNECTION_FAILS			):
								with pytest.raises(requests.exceptions.ConnectionError			):
												requests.request('''GET''',	'''https://huggingface.co'''			)

def   __lowercase       (			):







				"""simple docstring"""






				with offline(OfflineSimulationMode.HF_DATASETS_OFFLINE_SET_TO_1			):
								with pytest.raises(snake_case			):
												http_head('''https://huggingface.co'''			)






