





import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_ 							=       logging.get_logger(__name__)


UpperCAmelCase_ 							=       {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

UpperCAmelCase_ 							=       {
    """vocab_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt""",
        """allenai/longformer-large-4096""": (
            """https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-finetuned-triviaqa""": (
            """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"""
        ),
        """allenai/longformer-base-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
        """allenai/longformer-large-4096-extra.pos.embd.only""": (
            """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"""
        ),
    },
}

UpperCAmelCase_ 							=       {
    """allenai/longformer-base-4096""": 4096,
    """allenai/longformer-large-4096""": 4096,
    """allenai/longformer-large-4096-finetuned-triviaqa""": 4096,
    """allenai/longformer-base-4096-extra.pos.embd.only""": 4096,
    """allenai/longformer-large-4096-extra.pos.embd.only""": 4096,
}





@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def   lowerCamelCase__       (       )     ->	Dict:



					'''simple docstring'''




					_snake_case													=			(
					    list(range(ord('!'  )    ,      ord('~'  ) + 1  )  ) + list(range(ord('¡'  )    ,      ord('¬'  ) + 1  )  ) + list(range(ord('®'  )    ,      ord('ÿ'  ) + 1  )  )
					)
					_snake_case													=			bs[:]
					_snake_case													=			0
					for b in range(2**8  ):
										if b not in bs:
															bs.append(UpperCamelCase__  )
															cs.append(2**8 + n  )
															n += 1
					_snake_case													=			[chr(UpperCamelCase__  ) for n in cs]
					return dict(zip(UpperCamelCase__    ,      UpperCamelCase__  )  )





def   lowerCamelCase__       (       UpperCamelCase__     :    str  )     ->	Any:



					'''simple docstring'''




					_snake_case													=			set()
					_snake_case													=			word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)  )
										_snake_case													=			char
					return pairs




class      UpperCamelCase_						(					_lowerCamelCase   ):
					lowerCAmelCase_							=		VOCAB_FILES_NAMES
					lowerCAmelCase_							=		PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_							=		['''input_ids''', '''attention_mask''']




					def __init__(						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_="replace"			,  lowerCAmelCase_="<s>"			,  lowerCAmelCase_="</s>"			,  lowerCAmelCase_="</s>"			,  lowerCAmelCase_="<s>"			,  lowerCAmelCase_="<unk>"			,  lowerCAmelCase_="<pad>"			,  lowerCAmelCase_="<mask>"			,  lowerCAmelCase_=False			,  **lowerCAmelCase_			,  )						->							Optional[int]:
										_snake_case													=			AddedToken(lowerCAmelCase_			,  lstrip=lowerCAmelCase_			,  rstrip=lowerCAmelCase_					) if isinstance(lowerCAmelCase_			,  lowerCAmelCase_					) else bos_token
										_snake_case													=			AddedToken(lowerCAmelCase_			,  lstrip=lowerCAmelCase_			,  rstrip=lowerCAmelCase_					) if isinstance(lowerCAmelCase_			,  lowerCAmelCase_					) else eos_token
										_snake_case													=			AddedToken(lowerCAmelCase_			,  lstrip=lowerCAmelCase_			,  rstrip=lowerCAmelCase_					) if isinstance(lowerCAmelCase_			,  lowerCAmelCase_					) else sep_token
										_snake_case													=			AddedToken(lowerCAmelCase_			,  lstrip=lowerCAmelCase_			,  rstrip=lowerCAmelCase_					) if isinstance(lowerCAmelCase_			,  lowerCAmelCase_					) else cls_token
										_snake_case													=			AddedToken(lowerCAmelCase_			,  lstrip=lowerCAmelCase_			,  rstrip=lowerCAmelCase_					) if isinstance(lowerCAmelCase_			,  lowerCAmelCase_					) else unk_token
										_snake_case													=			AddedToken(lowerCAmelCase_			,  lstrip=lowerCAmelCase_			,  rstrip=lowerCAmelCase_					) if isinstance(lowerCAmelCase_			,  lowerCAmelCase_					) else pad_token

										# Mask token behave like a normal word, i.e. include the space before it
										_snake_case													=			AddedToken(lowerCAmelCase_			,  lstrip=lowerCAmelCase_			,  rstrip=lowerCAmelCase_					) if isinstance(lowerCAmelCase_			,  lowerCAmelCase_					) else mask_token

										super().__init__(
										    errors=lowerCAmelCase_			,  bos_token=lowerCAmelCase_			,  eos_token=lowerCAmelCase_			,  unk_token=lowerCAmelCase_			,  sep_token=lowerCAmelCase_			,  cls_token=lowerCAmelCase_			,  pad_token=lowerCAmelCase_			,  mask_token=lowerCAmelCase_			,  add_prefix_space=lowerCAmelCase_			,  **lowerCAmelCase_			,  )

										with open(lowerCAmelCase_			,  encoding='utf-8'					) as vocab_handle:
															_snake_case													=			json.load(lowerCAmelCase_					)
										_snake_case													=			{v: k for k, v in self.encoder.items()}
										_snake_case													=			errors  # how to handle errors in decoding
										_snake_case													=			bytes_to_unicode()
										_snake_case													=			{v: k for k, v in self.byte_encoder.items()}
										with open(lowerCAmelCase_			,  encoding='utf-8'					) as merges_handle:
															_snake_case													=			merges_handle.read().split('\n'					)[1:-1]
										_snake_case													=			[tuple(merge.split()					) for merge in bpe_merges]
										_snake_case													=			dict(zip(lowerCAmelCase_			,  range(len(lowerCAmelCase_					)					)					)					)
										_snake_case													=			{}
										_snake_case													=			add_prefix_space

										# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
										_snake_case													=			re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'					)




					@property
					def 		lowerCAmelCase     (						self					)						->							int:
										return len(self.encoder					)




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										return dict(self.encoder			,  **self.added_tokens_encoder					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							List[str]:
										if token in self.cache:
															return self.cache[token]
										_snake_case													=			tuple(lowerCAmelCase_					)
										_snake_case													=			get_pairs(lowerCAmelCase_					)

										if not pairs:
															return token

										while True:
															_snake_case													=			min(lowerCAmelCase_			,  key=lambda lowerCAmelCase_					: self.bpe_ranks.get(lowerCAmelCase_			,  float('inf'					)					)					)
															if bigram not in self.bpe_ranks:
																				break
															_snake_case							,       _snake_case													=			bigram
															_snake_case													=			[]
															_snake_case													=			0
															while i < len(lowerCAmelCase_					):
																				try:
																									_snake_case													=			word.index(lowerCAmelCase_			,  lowerCAmelCase_					)
																				except ValueError:
																									new_word.extend(word[i:]					)
																									break
																				else:
																									new_word.extend(word[i:j]					)
																									_snake_case													=			j

																				if word[i] == first and i < len(lowerCAmelCase_					) - 1 and word[i + 1] == second:
																									new_word.append(first + second					)
																									i += 2
																				else:
																									new_word.append(word[i]					)
																									i += 1
															_snake_case													=			tuple(lowerCAmelCase_					)
															_snake_case													=			new_word
															if len(lowerCAmelCase_					) == 1:
																				break
															else:
																				_snake_case													=			get_pairs(lowerCAmelCase_					)
										_snake_case													=			' '.join(lowerCAmelCase_					)
										_snake_case													=			word
										return word




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							Tuple:
										_snake_case													=			[]
										for token in re.findall(self.pat			,  lowerCAmelCase_					):
															_snake_case													=			''.join(
															    self.byte_encoder[b] for b in token.encode('utf-8'					)					)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
															bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCAmelCase_					).split(' '					)					)
										return bpe_tokens




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							List[Any]:
										return self.encoder.get(lowerCAmelCase_			,  self.encoder.get(self.unk_token					)					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							Union[str, Any]:
										return self.decoder.get(lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							List[str]:
										_snake_case													=			''.join(lowerCAmelCase_					)
										_snake_case													=			bytearray([self.byte_decoder[c] for c in text]					).decode('utf-8'			,  errors=self.errors					)
										return text




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_ = None					)						->							Tuple[str]:
										if not os.path.isdir(lowerCAmelCase_					):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
															return
										_snake_case													=			os.path.join(
										    lowerCAmelCase_			,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)
										_snake_case													=			os.path.join(
										    lowerCAmelCase_			,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']					)

										with open(lowerCAmelCase_			,  'w'			,  encoding='utf-8'					) as f:
															f.write(json.dumps(self.encoder			,  indent=2			,  sort_keys=lowerCAmelCase_			,  ensure_ascii=lowerCAmelCase_					) + '\n'					)

										_snake_case													=			0
										with open(lowerCAmelCase_			,  'w'			,  encoding='utf-8'					) as writer:
															writer.write('#version: 0.2\n'					)
															for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,  key=lambda lowerCAmelCase_					: kv[1]					):
																				if index != token_index:
																									logger.warning(
																									    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																									    ' Please check that the tokenizer is not corrupted!'					)
																									_snake_case													=			token_index
																				writer.write(' '.join(lowerCAmelCase_					) + '\n'					)
																				index += 1

										return vocab_file, merge_file




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_ = None					)						->							List[int]:
										if token_ids_a is None:
															return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										_snake_case													=			[self.cls_token_id]
										_snake_case													=			[self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_ = None			,  lowerCAmelCase_ = False					)						->							List[int]:
										if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=lowerCAmelCase_			,  token_ids_a=lowerCAmelCase_			,  already_has_special_tokens=lowerCAmelCase_					)

										if token_ids_a is None:
															return [1] + ([0] * len(lowerCAmelCase_					)) + [1]
										return [1] + ([0] * len(lowerCAmelCase_					)) + [1, 1] + ([0] * len(lowerCAmelCase_					)) + [1]




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_ = None					)						->							List[int]:
										_snake_case													=			[self.sep_token_id]
										_snake_case													=			[self.cls_token_id]

										if token_ids_a is None:
															return len(cls + token_ids_a + sep					) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep					) * [0]






					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_=False			,  **lowerCAmelCase_					)						->							List[Any]:
										_snake_case													=			kwargs.pop('add_prefix_space'			,  self.add_prefix_space					)
										if (is_split_into_words or add_prefix_space) and (len(lowerCAmelCase_					) > 0 and not text[0].isspace()):
															_snake_case													=			' ' + text
										return (text, kwargs)










import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      UpperCamelCase_						(					_lowerCamelCase							,					unittest.TestCase   ):
					lowerCAmelCase_							=		BertTokenizer
					lowerCAmelCase_							=		BertTokenizerFast
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		True
					lowerCAmelCase_							=		filter_non_english




					def 		lowerCAmelCase     (						self					)						->							Optional[int]:
										super().setUp()

										_snake_case													=			[
										    '[UNK]',
										    '[CLS]',
										    '[SEP]',
										    '[PAD]',
										    '[MASK]',
										    'want',
										    '##want',
										    '##ed',
										    'wa',
										    'un',
										    'runn',
										    '##ing',
										    ',',
										    'low',
										    'lowest',
										]
										_snake_case													=			os.path.join(self.tmpdirname			,  VOCAB_FILES_NAMES['vocab_file']					)
										with open(self.vocab_file			,  'w'			,  encoding='utf-8'					) as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]					)					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							Optional[Any]:
										_snake_case													=			'UNwant\u00E9d,running'
										_snake_case													=			'unwanted, running'
										return input_text, output_text




					def 		lowerCAmelCase     (						self					)						->							List[Any]:
										_snake_case													=			self.tokenizer_class(self.vocab_file					)

										_snake_case													=			tokenizer.tokenize('UNwant\u00E9d,running'					)
										self.assertListEqual(lowerCAmelCase_			,  ['un', '##want', '##ed', ',', 'runn', '##ing']					)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_					)			,  [9, 6, 7, 12, 10, 11]					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										if not self.test_rust_tokenizer:
															return

										_snake_case													=			self.get_tokenizer()
										_snake_case													=			self.get_rust_tokenizer()

										_snake_case													=			'UNwant\u00E9d,running'

										_snake_case													=			tokenizer.tokenize(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.tokenize(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			self.get_rust_tokenizer()
										_snake_case													=			tokenizer.encode(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										# With lower casing
										_snake_case													=			self.get_tokenizer(do_lower_case=lowerCAmelCase_					)
										_snake_case													=			self.get_rust_tokenizer(do_lower_case=lowerCAmelCase_					)

										_snake_case													=			'UNwant\u00E9d,running'

										_snake_case													=			tokenizer.tokenize(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.tokenize(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			self.get_rust_tokenizer()
										_snake_case													=			tokenizer.encode(lowerCAmelCase_					)
										_snake_case													=			rust_tokenizer.encode(lowerCAmelCase_					)
										self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										_snake_case													=			BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'					)			,  ['ah', '\u535A', '\u63A8', 'zz']					)




					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)			,  ['hello', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							List[Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hällo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['h\u00E9llo']					)




					def 		lowerCAmelCase     (						self					)						->							Any:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hallo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							List[str]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['hallo', '!', 'how', 'are', 'you', '?']					)
										self.assertListEqual(tokenizer.tokenize('H\u00E9llo'					)			,  ['hello']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '					)			,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['HäLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  strip_accents=lowerCAmelCase_					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '					)			,  ['HaLLo', '!', 'how', 'Are', 'yoU', '?']					)




					def 		lowerCAmelCase     (						self					)						->							Dict:
										_snake_case													=			BasicTokenizer(do_lower_case=lowerCAmelCase_			,  never_split=['[UNK]']					)

										self.assertListEqual(
										    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'					)			,  ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			BasicTokenizer()
										_snake_case													=			'a\n\'ll !!to?\'d of, can\'t.'
										_snake_case													=			['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
										self.assertListEqual(tokenizer.tokenize(lowerCAmelCase_					)			,  lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										_snake_case													=			['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

										_snake_case													=			{}
										for i, token in enumerate(lowerCAmelCase_					):
															_snake_case													=			i
										_snake_case													=			WordpieceTokenizer(vocab=lowerCAmelCase_			,  unk_token='[UNK]'					)

										self.assertListEqual(tokenizer.tokenize(''					)			,  []					)

										self.assertListEqual(tokenizer.tokenize('unwanted running'					)			,  ['un', '##want', '##ed', 'runn', '##ing']					)

										self.assertListEqual(tokenizer.tokenize('unwantedX running'					)			,  ['[UNK]', 'runn', '##ing']					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										self.assertTrue(_is_whitespace(' '					)					)
										self.assertTrue(_is_whitespace('\t'					)					)
										self.assertTrue(_is_whitespace('\r'					)					)
										self.assertTrue(_is_whitespace('\n'					)					)
										self.assertTrue(_is_whitespace('\u00A0'					)					)

										self.assertFalse(_is_whitespace('A'					)					)
										self.assertFalse(_is_whitespace('-'					)					)




					def 		lowerCAmelCase     (						self					)						->							Dict:
										self.assertTrue(_is_control('\u0005'					)					)

										self.assertFalse(_is_control('A'					)					)
										self.assertFalse(_is_control(' '					)					)
										self.assertFalse(_is_control('\t'					)					)
										self.assertFalse(_is_control('\r'					)					)




					def 		lowerCAmelCase     (						self					)						->							int:
										self.assertTrue(_is_punctuation('-'					)					)
										self.assertTrue(_is_punctuation('$'					)					)
										self.assertTrue(_is_punctuation('`'					)					)
										self.assertTrue(_is_punctuation('.'					)					)

										self.assertFalse(_is_punctuation('A'					)					)
										self.assertFalse(_is_punctuation(' '					)					)




					def 		lowerCAmelCase     (						self					)						->							Tuple:
										_snake_case													=			self.get_tokenizer()
										_snake_case													=			self.get_rust_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(lowerCAmelCase_					) for t in ['Test', '\xad', 'test']]			,  [['[UNK]'], [], ['[UNK]']]					)

										self.assertListEqual(
										    [rust_tokenizer.tokenize(lowerCAmelCase_					) for t in ['Test', '\xad', 'test']]			,  [['[UNK]'], [], ['[UNK]']]					)




					@slow
					def 		lowerCAmelCase     (						self					)						->							Optional[Any]:
										_snake_case													=			self.tokenizer_class.from_pretrained('bert-base-uncased'					)

										_snake_case													=			tokenizer.encode('sequence builders'			,  add_special_tokens=lowerCAmelCase_					)
										_snake_case													=			tokenizer.encode('multi-sequence build'			,  add_special_tokens=lowerCAmelCase_					)

										_snake_case													=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_					)
										_snake_case													=			tokenizer.build_inputs_with_special_tokens(lowerCAmelCase_			,  lowerCAmelCase_					)

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]




					def 		lowerCAmelCase     (						self					)						->							Union[str, Any]:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																				_snake_case													=			tokenizer_r.encode_plus(
																				    lowerCAmelCase_			,  return_attention_mask=lowerCAmelCase_			,  return_token_type_ids=lowerCAmelCase_			,  return_offsets_mapping=lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_			,  )

																				_snake_case													=			tokenizer_r.do_lower_case if hasattr(lowerCAmelCase_			,  'do_lower_case'					) else False
																				_snake_case													=			(
																				    [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'A'),
																				        ((1, 2), ','),
																				        ((3, 5), 'na'),
																				        ((5, 6), '##ï'),
																				        ((6, 8), '##ve'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'Allen'),
																				        ((21, 23), '##NL'),
																				        ((23, 24), '##P'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				    if not do_lower_case
																				    else [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'a'),
																				        ((1, 2), ','),
																				        ((3, 8), 'naive'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'allen'),
																				        ((21, 23), '##nl'),
																				        ((23, 24), '##p'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				)

																				self.assertEqual(
																				    [e[1] for e in expected_results]			,  tokenizer_r.convert_ids_to_tokens(tokens['input_ids']					)					)
																				self.assertEqual([e[0] for e in expected_results]			,  tokens['offset_mapping']					)






					def 		lowerCAmelCase     (						self					)						->							str:
										_snake_case													=			['的', '人', '有']
										_snake_case													=			''.join(lowerCAmelCase_					)
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
																				_snake_case													=			True
																				_snake_case													=			self.tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			tokenizer_p.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
																				_snake_case													=			tokenizer_r.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_					)

																				# it is expected that each Chinese character is not preceded by "##"
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)

																				_snake_case													=			False
																				_snake_case													=			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)
																				_snake_case													=			self.tokenizer_class.from_pretrained(lowerCAmelCase_			,  **lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.encode(lowerCAmelCase_			,  add_special_tokens=lowerCAmelCase_					)

																				_snake_case													=			tokenizer_r.convert_ids_to_tokens(lowerCAmelCase_					)
																				_snake_case													=			tokenizer_p.convert_ids_to_tokens(lowerCAmelCase_					)

																				# it is expected that only the first Chinese character is not preceded by "##".
																				_snake_case													=			[
																				    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(lowerCAmelCase_					)
																				]
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)
																				self.assertListEqual(lowerCAmelCase_			,  lowerCAmelCase_					)



