
"""simple docstring"""


import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE			  =  logging.get_logger(__name__)

SCREAMING_SNAKE_CASE			  =  {
    """vocab_file""": """vocab.txt""",
    """merges_file""": """bpe.codes""",
}

SCREAMING_SNAKE_CASE			  =  {
    """vocab_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt""",
    },
    """merges_file""": {
        """vinai/phobert-base""": """https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes""",
        """vinai/phobert-large""": """https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes""",
    },
}

SCREAMING_SNAKE_CASE			  =  {
    """vinai/phobert-base""": 256,
    """vinai/phobert-large""": 256,
}



def 						lowerCamelCase__      (				UpperCAmelCase_					)->				Tuple:







       """simple docstring"""
       UpperCamelCase       			=		set()
       UpperCamelCase       			=		word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)					)
              UpperCamelCase       			=		char

       UpperCamelCase       			=		set(UpperCAmelCase_					)
       return pairs



class 	__a	(   _lowerCAmelCase						):
       UpperCamelCase_		:       int				=      VOCAB_FILES_NAMES
       UpperCamelCase_		:       int				=      PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase_		:       Tuple				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






       def __init__(	self			:				Optional[int]							,   UpperCAmelCase_			:				int							,   UpperCAmelCase_			:				Union[str, Any]							,   UpperCAmelCase_			:				Optional[int]="<s>"							,   UpperCAmelCase_			:				int="</s>"							,   UpperCAmelCase_			:				Tuple="</s>"							,   UpperCAmelCase_			:				Any="<s>"							,   UpperCAmelCase_			:				str="<unk>"							,   UpperCAmelCase_			:				Optional[int]="<pad>"							,   UpperCAmelCase_			:				Any="<mask>"							,   **UpperCAmelCase_			:				Optional[int]							,   )->	Union[str, Any]:







              """simple docstring"""
              super().__init__(
                  bos_token=UpperCAmelCase_							,   eos_token=UpperCAmelCase_							,   unk_token=UpperCAmelCase_							,   sep_token=UpperCAmelCase_							,   cls_token=UpperCAmelCase_							,   pad_token=UpperCAmelCase_							,   mask_token=UpperCAmelCase_							,   **UpperCAmelCase_							,   )

              UpperCamelCase       			=		vocab_file
              UpperCamelCase       			=		merges_file

              UpperCamelCase       			=		{}
              UpperCamelCase       			=		0
              UpperCamelCase       			=		1
              UpperCamelCase       			=		2
              UpperCamelCase       			=		3

              self.add_from_file(UpperCAmelCase_      )

              UpperCamelCase       			=		{v: k for k, v in self.encoder.items()}

              with open(UpperCAmelCase_							,   encoding="utf-8"      ) as merges_handle:
                     UpperCamelCase       			=		merges_handle.read().split("\n"      )[:-1]
              UpperCamelCase       			=		[tuple(merge.split()[:-1]      ) for merge in merges]
              UpperCamelCase       			=		dict(zip(UpperCAmelCase_							,   range(len(UpperCAmelCase_      )      )      )      )
              UpperCamelCase       			=		{}






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[Any]							,   UpperCAmelCase_			:				List[int]							,   UpperCAmelCase_			:				Optional[List[int]] = None      )->	List[int]:







              """simple docstring"""

              if token_ids_a is None:
                     return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
              UpperCamelCase       			=		[self.cls_token_id]
              UpperCamelCase       			=		[self.sep_token_id]
              return cls + token_ids_a + sep + sep + token_ids_a + sep






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[Any]							,   UpperCAmelCase_			:				List[int]							,   UpperCAmelCase_			:				Optional[List[int]] = None							,   UpperCAmelCase_			:				bool = False      )->	List[int]:







              """simple docstring"""

              if already_has_special_tokens:
                     return super().get_special_tokens_mask(
                         token_ids_a=UpperCAmelCase_							,   token_ids_a=UpperCAmelCase_							,   already_has_special_tokens=UpperCAmelCase_      )

              if token_ids_a is None:
                     return [1] + ([0] * len(UpperCAmelCase_      )) + [1]
              return [1] + ([0] * len(UpperCAmelCase_      )) + [1, 1] + ([0] * len(UpperCAmelCase_      )) + [1]






       def 						_SCREAMING_SNAKE_CASE							(	self			:				str							,   UpperCAmelCase_			:				List[int]							,   UpperCAmelCase_			:				Optional[List[int]] = None      )->	List[int]:







              """simple docstring"""
              UpperCamelCase       			=		[self.sep_token_id]
              UpperCamelCase       			=		[self.cls_token_id]

              if token_ids_a is None:
                     return len(cls + token_ids_a + sep      ) * [0]
              return len(cls + token_ids_a + sep + sep + token_ids_a + sep      ) * [0]






       @property
       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[str]      )->	List[str]:







              """simple docstring"""
              return len(self.encoder      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Union[str, Any]      )->	Tuple:







              """simple docstring"""
              return dict(self.encoder							,   **self.added_tokens_encoder      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Any							,   UpperCAmelCase_			:				Tuple      )->	Any:







              """simple docstring"""
              if token in self.cache:
                     return self.cache[token]
              UpperCamelCase       			=		tuple(UpperCAmelCase_      )
              UpperCamelCase       			=		tuple(list(word[:-1]      ) + [word[-1] + "</w>"]      )
              UpperCamelCase       			=		get_pairs(UpperCAmelCase_      )

              if not pairs:
                     return token

              while True:
                     UpperCamelCase       			=		min(UpperCAmelCase_							,   key=lambda UpperCAmelCase_      : self.bpe_ranks.get(UpperCAmelCase_							,   float("inf"      )      )      )
                     if bigram not in self.bpe_ranks:
                            break
                     UpperCamelCase			,							UpperCamelCase       			=		bigram
                     UpperCamelCase       			=		[]
                     UpperCamelCase       			=		0
                     while i < len(UpperCAmelCase_      ):
                            try:
                                   UpperCamelCase       			=		word.index(UpperCAmelCase_							,   UpperCAmelCase_      )
                            except ValueError:
                                   new_word.extend(word[i:]      )
                                   break
                            else:
                                   new_word.extend(word[i:j]      )
                                   UpperCamelCase       			=		j

                            if word[i] == first and i < len(UpperCAmelCase_      ) - 1 and word[i + 1] == second:
                                   new_word.append(first + second      )
                                   i += 2
                            else:
                                   new_word.append(word[i]      )
                                   i += 1
                     UpperCamelCase       			=		tuple(UpperCAmelCase_      )
                     UpperCamelCase       			=		new_word
                     if len(UpperCAmelCase_      ) == 1:
                            break
                     else:
                            UpperCamelCase       			=		get_pairs(UpperCAmelCase_      )
              UpperCamelCase       			=		"@@ ".join(UpperCAmelCase_      )
              UpperCamelCase       			=		word[:-4]
              UpperCamelCase       			=		word
              return word






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[int]							,   UpperCAmelCase_			:				Dict      )->	int:







              """simple docstring"""
              UpperCamelCase       			=		[]

              UpperCamelCase       			=		re.findall(r"\S+\n?"							,   UpperCAmelCase_      )

              for token in words:
                     split_tokens.extend(list(self.bpe(UpperCAmelCase_      ).split(" "      )      )      )
              return split_tokens






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Tuple							,   UpperCAmelCase_			:				int      )->	str:







              """simple docstring"""
              return self.encoder.get(UpperCAmelCase_							,   self.encoder.get(self.unk_token      )      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Tuple							,   UpperCAmelCase_			:				Optional[int]      )->	Union[str, Any]:







              """simple docstring"""
              return self.decoder.get(UpperCAmelCase_							,   self.unk_token      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Any							,   UpperCAmelCase_			:				Union[str, Any]      )->	List[str]:







              """simple docstring"""
              UpperCamelCase       			=		" ".join(UpperCAmelCase_      ).replace("@@ "							,   ""      ).strip()
              return out_string






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Tuple							,   UpperCAmelCase_			:				str							,   UpperCAmelCase_			:				Optional[str] = None      )->	Tuple[str]:







              """simple docstring"""
              if not os.path.isdir(UpperCAmelCase_      ):
                     logger.error(f"Vocabulary path ({save_directory}) should be a directory"      )
                     return
              UpperCamelCase       			=		os.path.join(
                  UpperCAmelCase_							,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )
              UpperCamelCase       			=		os.path.join(
                  UpperCAmelCase_							,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]      )

              if os.path.abspath(self.vocab_file      ) != os.path.abspath(UpperCAmelCase_      ):
                     copyfile(self.vocab_file							,   UpperCAmelCase_      )

              if os.path.abspath(self.merges_file      ) != os.path.abspath(UpperCAmelCase_      ):
                     copyfile(self.merges_file							,   UpperCAmelCase_      )

              return out_vocab_file, out_merge_file






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[str]							,   UpperCAmelCase_			:				int      )->	Optional[int]:







              """simple docstring"""
              if isinstance(UpperCAmelCase_							,   UpperCAmelCase_      ):
                     try:
                            with open(UpperCAmelCase_							,   "r"							,   encoding="utf-8"      ) as fd:
                                   self.add_from_file(UpperCAmelCase_      )
                     except FileNotFoundError as fnfe:
                            raise fnfe
                     except UnicodeError:
                            raise Exception(f"Incorrect encoding detected in {f}, please rebuild the dataset"      )
                     return

              UpperCamelCase       			=		f.readlines()
              for lineTmp in lines:
                     UpperCamelCase       			=		lineTmp.strip()
                     UpperCamelCase       			=		line.rfind(" "      )
                     if idx == -1:
                            raise ValueError("Incorrect dictionary format, expected '<token> <cnt>'"      )
                     UpperCamelCase       			=		line[:idx]
                     UpperCamelCase       			=		len(self.encoder      )







"""simple docstring"""


import itertools
import json
import os
import unittest

from transformers import AddedToken, LongformerTokenizer, LongformerTokenizerFast
from transformers.models.longformer.tokenization_longformer import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin



@require_tokenizers
class 	__a	(   _lowerCAmelCase     ,  unittest.TestCase						):
       UpperCamelCase_		:       Optional[int]				=      LongformerTokenizer
       UpperCamelCase_		:       str				=      True
       UpperCamelCase_		:       Tuple				=      LongformerTokenizerFast
       UpperCamelCase_		:       List[str]				=      True






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Dict      )->	Any:







              """simple docstring"""
              super().setUp()

              # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
              UpperCamelCase       			=		[
                  "l",
                  "o",
                  "w",
                  "e",
                  "r",
                  "s",
                  "t",
                  "i",
                  "d",
                  "n",
                  "\u0120",
                  "\u0120l",
                  "\u0120n",
                  "\u0120lo",
                  "\u0120low",
                  "er",
                  "\u0120lowest",
                  "\u0120newer",
                  "\u0120wider",
                  "<unk>",
              ]
              UpperCamelCase       			=		dict(zip(UpperCAmelCase_							,   range(len(UpperCAmelCase_      )      )      )      )
              UpperCamelCase       			=		["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
              UpperCamelCase       			=		{"unk_token": "<unk>"}

              UpperCamelCase       			=		os.path.join(self.tmpdirname							,   VOCAB_FILES_NAMES["vocab_file"]      )
              UpperCamelCase       			=		os.path.join(self.tmpdirname							,   VOCAB_FILES_NAMES["merges_file"]      )
              with open(self.vocab_file							,   "w"							,   encoding="utf-8"      ) as fp:
                     fp.write(json.dumps(UpperCAmelCase_      ) + "\n"      )
              with open(self.merges_file							,   "w"							,   encoding="utf-8"      ) as fp:
                     fp.write("\n".join(UpperCAmelCase_      )      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[Any]							,   **UpperCAmelCase_			:				Dict      )->	int:







              """simple docstring"""
              kwargs.update(self.special_tokens_map      )
              return self.tokenizer_class.from_pretrained(self.tmpdirname							,   **UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				str							,   **UpperCAmelCase_			:				Optional[Any]      )->	Optional[int]:







              """simple docstring"""
              kwargs.update(self.special_tokens_map      )
              return self.rust_tokenizer_class.from_pretrained(self.tmpdirname							,   **UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[int]							,   UpperCAmelCase_			:				Dict      )->	List[Any]:







              """simple docstring"""
              UpperCamelCase       			=		"lower newer"
              UpperCamelCase       			=		"lower newer"
              return input_text, output_text






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Tuple      )->	Optional[Any]:







              """simple docstring"""
              UpperCamelCase       			=		self.tokenizer_class(self.vocab_file							,   self.merges_file							,   **self.special_tokens_map      )
              UpperCamelCase       			=		"lower newer"
              UpperCamelCase       			=		["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
              UpperCamelCase       			=		tokenizer.tokenize(UpperCAmelCase_      )  # , add_prefix_space=True)
              self.assertListEqual(UpperCAmelCase_							,   UpperCAmelCase_      )

              UpperCamelCase       			=		tokens + [tokenizer.unk_token]
              UpperCamelCase       			=		[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
              self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )							,   UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[int]      )->	Tuple:







              """simple docstring"""
              UpperCamelCase       			=		self.get_tokenizer()

              self.assertListEqual(tokenizer.encode("Hello world!"							,   add_special_tokens=UpperCAmelCase_      )							,   [0, 31_414, 232, 328, 2]      )
              self.assertListEqual(
                  tokenizer.encode("Hello world! cécé herlolip 418"							,   add_special_tokens=UpperCAmelCase_      )							,   [0, 31_414, 232, 328, 740, 1_140, 12_695, 69, 46_078, 1_588, 2]							,   )






       @slow
       def 						_SCREAMING_SNAKE_CASE							(	self			:				int      )->	int:







              """simple docstring"""
              UpperCamelCase       			=		self.tokenizer_class.from_pretrained("allenai/longformer-base-4096"      )

              UpperCamelCase       			=		tokenizer.encode("sequence builders"							,   add_special_tokens=UpperCAmelCase_      )
              UpperCamelCase       			=		tokenizer.encode("multi-sequence build"							,   add_special_tokens=UpperCAmelCase_      )

              UpperCamelCase       			=		tokenizer.encode(
                  "sequence builders"							,   add_special_tokens=UpperCAmelCase_							,   add_prefix_space=UpperCAmelCase_      )
              UpperCamelCase       			=		tokenizer.encode(
                  "sequence builders"							,   "multi-sequence build"							,   add_special_tokens=UpperCAmelCase_							,   add_prefix_space=UpperCAmelCase_      )

              UpperCamelCase       			=		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_      )
              UpperCamelCase       			=		tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_							,   UpperCAmelCase_      )

              assert encoded_sentence == encoded_text_from_decode
              assert encoded_pair == encoded_pair_from_decode






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[int]      )->	Optional[Any]:







              """simple docstring"""
              UpperCamelCase       			=		self.get_tokenizer()

              UpperCamelCase       			=		"Encode this sequence."
              UpperCamelCase       			=		tokenizer.byte_encoder[" ".encode("utf-8"      )[0]]

              # Testing encoder arguments
              UpperCamelCase       			=		tokenizer.encode(UpperCAmelCase_							,   add_special_tokens=UpperCAmelCase_							,   add_prefix_space=UpperCAmelCase_      )
              UpperCamelCase       			=		tokenizer.convert_ids_to_tokens(encoded[0]      )[0]
              self.assertNotEqual(UpperCAmelCase_							,   UpperCAmelCase_      )

              UpperCamelCase       			=		tokenizer.encode(UpperCAmelCase_							,   add_special_tokens=UpperCAmelCase_							,   add_prefix_space=UpperCAmelCase_      )
              UpperCamelCase       			=		tokenizer.convert_ids_to_tokens(encoded[0]      )[0]
              self.assertEqual(UpperCAmelCase_							,   UpperCAmelCase_      )

              tokenizer.add_special_tokens({"bos_token": "<s>"}      )
              UpperCamelCase       			=		tokenizer.encode(UpperCAmelCase_							,   add_special_tokens=UpperCAmelCase_      )
              UpperCamelCase       			=		tokenizer.convert_ids_to_tokens(encoded[1]      )[0]
              self.assertNotEqual(UpperCAmelCase_							,   UpperCAmelCase_      )

              # Testing spaces after special tokens
              UpperCamelCase       			=		"<mask>"
              tokenizer.add_special_tokens(
                  {"mask_token": AddedToken(UpperCAmelCase_							,   lstrip=UpperCAmelCase_							,   rstrip=UpperCAmelCase_      )}      )  # mask token has a left space
              UpperCamelCase       			=		tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )

              UpperCamelCase       			=		"Encode <mask> sequence"
              UpperCamelCase       			=		"Encode <mask>sequence"

              UpperCamelCase       			=		tokenizer.encode(UpperCAmelCase_      )
              UpperCamelCase       			=		encoded.index(UpperCAmelCase_      )
              UpperCamelCase       			=		tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]      )[0]
              self.assertEqual(UpperCAmelCase_							,   UpperCAmelCase_      )

              UpperCamelCase       			=		tokenizer.encode(UpperCAmelCase_      )
              UpperCamelCase       			=		encoded.index(UpperCAmelCase_      )
              UpperCamelCase       			=		tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]      )[0]
              self.assertNotEqual(UpperCAmelCase_							,   UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[Any]      )->	List[str]:







              """simple docstring"""
              pass






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[Any]      )->	Tuple:







              """simple docstring"""
              for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                     with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
                            UpperCamelCase       			=		self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_							,   **UpperCAmelCase_      )
                            UpperCamelCase       			=		self.tokenizer_class.from_pretrained(UpperCAmelCase_							,   **UpperCAmelCase_      )
                            UpperCamelCase       			=		"A, <mask> AllenNLP sentence."
                            UpperCamelCase       			=		tokenizer_r.encode_plus(UpperCAmelCase_							,   add_special_tokens=UpperCAmelCase_							,   return_token_type_ids=UpperCAmelCase_      )
                            UpperCamelCase       			=		tokenizer_p.encode_plus(UpperCAmelCase_							,   add_special_tokens=UpperCAmelCase_							,   return_token_type_ids=UpperCAmelCase_      )

                            # token_type_ids should put 0 everywhere
                            self.assertEqual(sum(tokens_r["token_type_ids"]      )							,   sum(tokens_p["token_type_ids"]      )      )

                            # attention_mask should put 1 everywhere, so sum over length should be 1
                            self.assertEqual(
                                sum(tokens_r["attention_mask"]      ) / len(tokens_r["attention_mask"]      )							,   sum(tokens_p["attention_mask"]      ) / len(tokens_p["attention_mask"]      )							,   )

                            UpperCamelCase       			=		tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]      )
                            UpperCamelCase       			=		tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]      )

                            # Rust correctly handles the space before the mask while python doesnt
                            self.assertSequenceEqual(tokens_p["input_ids"]							,   [0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]      )
                            self.assertSequenceEqual(tokens_r["input_ids"]							,   [0, 250, 6, 50_264, 3_823, 487, 21_992, 3_645, 4, 2]      )

                            self.assertSequenceEqual(
                                UpperCAmelCase_							,   ["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]      )
                            self.assertSequenceEqual(
                                UpperCAmelCase_							,   ["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Any      )->	Dict:







              """simple docstring"""
              for trim_offsets, add_prefix_space in itertools.product([True, False]							,   repeat=2      ):
                     UpperCamelCase       			=		self.rust_tokenizer_class.from_pretrained(
                         self.tmpdirname							,   use_fast=UpperCAmelCase_							,   add_prefix_space=UpperCAmelCase_							,   trim_offsets=UpperCAmelCase_      )

                     UpperCamelCase       			=		json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()      )
                     UpperCamelCase       			=		json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()      )

                     self.assertEqual(pre_tokenizer_state["add_prefix_space"]							,   UpperCAmelCase_      )

                     self.assertEqual(post_processor_state["add_prefix_space"]							,   UpperCAmelCase_      )
                     self.assertEqual(post_processor_state["trim_offsets"]							,   UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[int]      )->	Union[str, Any]:







              """simple docstring"""
              # Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
              # `trim_offsets`
              for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                     with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"      ):
                            UpperCamelCase       			=		"hello"  # `hello` is a token in the vocabulary of `pretrained_name`
                            UpperCamelCase       			=		f"{text_of_1_token} {text_of_1_token}"

                            UpperCamelCase       			=		self.rust_tokenizer_class.from_pretrained(
                                UpperCAmelCase_							,   use_fast=UpperCAmelCase_							,   add_prefix_space=UpperCAmelCase_							,   trim_offsets=UpperCAmelCase_      )
                            UpperCamelCase       			=		tokenizer_r(UpperCAmelCase_							,   return_offsets_mapping=UpperCAmelCase_							,   add_special_tokens=UpperCAmelCase_      )
                            self.assertEqual(encoding.offset_mapping[0]							,   (0, len(UpperCAmelCase_      ))      )
                            self.assertEqual(
                                encoding.offset_mapping[1]							,   (len(UpperCAmelCase_      ) + 1, len(UpperCAmelCase_      ) + 1 + len(UpperCAmelCase_      ))							,   )

                            UpperCamelCase       			=		self.rust_tokenizer_class.from_pretrained(
                                UpperCAmelCase_							,   use_fast=UpperCAmelCase_							,   add_prefix_space=UpperCAmelCase_							,   trim_offsets=UpperCAmelCase_      )
                            UpperCamelCase       			=		tokenizer_r(UpperCAmelCase_							,   return_offsets_mapping=UpperCAmelCase_							,   add_special_tokens=UpperCAmelCase_      )
                            self.assertEqual(encoding.offset_mapping[0]							,   (0, len(UpperCAmelCase_      ))      )
                            self.assertEqual(
                                encoding.offset_mapping[1]							,   (len(UpperCAmelCase_      ) + 1, len(UpperCAmelCase_      ) + 1 + len(UpperCAmelCase_      ))							,   )

                            UpperCamelCase       			=		self.rust_tokenizer_class.from_pretrained(
                                UpperCAmelCase_							,   use_fast=UpperCAmelCase_							,   add_prefix_space=UpperCAmelCase_							,   trim_offsets=UpperCAmelCase_      )
                            UpperCamelCase       			=		tokenizer_r(UpperCAmelCase_							,   return_offsets_mapping=UpperCAmelCase_							,   add_special_tokens=UpperCAmelCase_      )
                            self.assertEqual(encoding.offset_mapping[0]							,   (0, len(UpperCAmelCase_      ))      )
                            self.assertEqual(
                                encoding.offset_mapping[1]							,   (len(UpperCAmelCase_      ), len(UpperCAmelCase_      ) + 1 + len(UpperCAmelCase_      ))							,   )

                            UpperCamelCase       			=		self.rust_tokenizer_class.from_pretrained(
                                UpperCAmelCase_							,   use_fast=UpperCAmelCase_							,   add_prefix_space=UpperCAmelCase_							,   trim_offsets=UpperCAmelCase_      )
                            UpperCamelCase       			=		tokenizer_r(UpperCAmelCase_							,   return_offsets_mapping=UpperCAmelCase_							,   add_special_tokens=UpperCAmelCase_      )
                            self.assertEqual(encoding.offset_mapping[0]							,   (0, len(UpperCAmelCase_      ))      )
                            self.assertEqual(
                                encoding.offset_mapping[1]							,   (len(UpperCAmelCase_      ), len(UpperCAmelCase_      ) + 1 + len(UpperCAmelCase_      ))							,   )

                            UpperCamelCase       			=		f" {text}"

                            # tokenizer_r = self.rust_tokenizer_class.from_pretrained(
                            #     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
                            # )
                            # encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
                            # self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
                            # self.assertEqual(
                            #     encoding.offset_mapping[1],
                            #     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
                            # )

                            UpperCamelCase       			=		self.rust_tokenizer_class.from_pretrained(
                                UpperCAmelCase_							,   use_fast=UpperCAmelCase_							,   add_prefix_space=UpperCAmelCase_							,   trim_offsets=UpperCAmelCase_      )
                            UpperCamelCase       			=		tokenizer_r(UpperCAmelCase_							,   return_offsets_mapping=UpperCAmelCase_							,   add_special_tokens=UpperCAmelCase_      )
                            self.assertEqual(encoding.offset_mapping[0]							,   (1, 1 + len(UpperCAmelCase_      ))      )
                            self.assertEqual(
                                encoding.offset_mapping[1]							,   (1 + len(UpperCAmelCase_      ) + 1, 1 + len(UpperCAmelCase_      ) + 1 + len(UpperCAmelCase_      ))							,   )

                            UpperCamelCase       			=		self.rust_tokenizer_class.from_pretrained(
                                UpperCAmelCase_							,   use_fast=UpperCAmelCase_							,   add_prefix_space=UpperCAmelCase_							,   trim_offsets=UpperCAmelCase_      )
                            UpperCamelCase       			=		tokenizer_r(UpperCAmelCase_							,   return_offsets_mapping=UpperCAmelCase_							,   add_special_tokens=UpperCAmelCase_      )
                            self.assertEqual(encoding.offset_mapping[0]							,   (0, 1 + len(UpperCAmelCase_      ))      )
                            self.assertEqual(
                                encoding.offset_mapping[1]							,   (1 + len(UpperCAmelCase_      ), 1 + len(UpperCAmelCase_      ) + 1 + len(UpperCAmelCase_      ))							,   )

                            UpperCamelCase       			=		self.rust_tokenizer_class.from_pretrained(
                                UpperCAmelCase_							,   use_fast=UpperCAmelCase_							,   add_prefix_space=UpperCAmelCase_							,   trim_offsets=UpperCAmelCase_      )
                            UpperCamelCase       			=		tokenizer_r(UpperCAmelCase_							,   return_offsets_mapping=UpperCAmelCase_							,   add_special_tokens=UpperCAmelCase_      )
                            self.assertEqual(encoding.offset_mapping[0]							,   (0, 1 + len(UpperCAmelCase_      ))      )
                            self.assertEqual(
                                encoding.offset_mapping[1]							,   (1 + len(UpperCAmelCase_      ), 1 + len(UpperCAmelCase_      ) + 1 + len(UpperCAmelCase_      ))							,   )





