



"""simple docstring"""







import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import BeitConfig, BeitForImageClassification, BeitForMaskedImageModeling, BeitImageProcessor
from transformers.image_utils import PILImageResampling
from transformers.utils import logging


logging.set_verbosity_info()
A		=						logging.get_logger(__name__)

def  __SCREAMING_SNAKE_CASE		(			lowerCamelCase_:							Dict     ,		lowerCamelCase_:							int=False     ,		lowerCamelCase_:							str=False ):
						"""simple docstring"""






						snake_case   :			str						    = "backbone." if is_semantic else ""

						snake_case   :			List[str]						    = []
						for i in range(config.num_hidden_layers ):
												# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
												rename_keys.append((f'''{prefix}blocks.{i}.norm1.weight''', f'''beit.encoder.layer.{i}.layernorm_before.weight''') )
												rename_keys.append((f'''{prefix}blocks.{i}.norm1.bias''', f'''beit.encoder.layer.{i}.layernorm_before.bias''') )
												rename_keys.append(
												    (f'''{prefix}blocks.{i}.attn.proj.weight''', f'''beit.encoder.layer.{i}.attention.output.dense.weight''') )
												rename_keys.append(
												    (f'''{prefix}blocks.{i}.attn.proj.bias''', f'''beit.encoder.layer.{i}.attention.output.dense.bias''') )
												rename_keys.append((f'''{prefix}blocks.{i}.norm2.weight''', f'''beit.encoder.layer.{i}.layernorm_after.weight''') )
												rename_keys.append((f'''{prefix}blocks.{i}.norm2.bias''', f'''beit.encoder.layer.{i}.layernorm_after.bias''') )
												rename_keys.append((f'''{prefix}blocks.{i}.mlp.fc1.weight''', f'''beit.encoder.layer.{i}.intermediate.dense.weight''') )
												rename_keys.append((f'''{prefix}blocks.{i}.mlp.fc1.bias''', f'''beit.encoder.layer.{i}.intermediate.dense.bias''') )
												rename_keys.append((f'''{prefix}blocks.{i}.mlp.fc2.weight''', f'''beit.encoder.layer.{i}.output.dense.weight''') )
												rename_keys.append((f'''{prefix}blocks.{i}.mlp.fc2.bias''', f'''beit.encoder.layer.{i}.output.dense.bias''') )

						# projection layer + position embeddings
						rename_keys.extend(
						    [
						        (f'''{prefix}cls_token''', "beit.embeddings.cls_token"),
						        (f'''{prefix}patch_embed.proj.weight''', "beit.embeddings.patch_embeddings.projection.weight"),
						        (f'''{prefix}patch_embed.proj.bias''', "beit.embeddings.patch_embeddings.projection.bias"),
						        (f'''{prefix}pos_embed''', "beit.embeddings.position_embeddings"),
						    ] )

						if has_lm_head:
												# mask token + layernorm
												rename_keys.extend(
												    [
												        ("mask_token", "beit.embeddings.mask_token"),
												        ("norm.weight", "layernorm.weight"),
												        ("norm.bias", "layernorm.bias"),
												    ] )
						else:
												# layernorm + classification head
												rename_keys.extend(
												    [
												        ("fc_norm.weight", "beit.pooler.layernorm.weight"),
												        ("fc_norm.bias", "beit.pooler.layernorm.bias"),
												        ("head.weight", "classifier.weight"),
												        ("head.bias", "classifier.bias"),
												    ] )

						return rename_keys

def  __SCREAMING_SNAKE_CASE		(			lowerCamelCase_:							Tuple     ,		lowerCamelCase_:							Optional[Any]     ,		lowerCamelCase_:							str=False     ,		lowerCamelCase_:							Union[str, Any]=False ):
						"""simple docstring"""






						for i in range(config.num_hidden_layers ):
												snake_case   :			Union[str, Any]						    = "backbone." if is_semantic else ""
												# queries, keys and values
												snake_case   :			List[str]						    = state_dict.pop(f'''{prefix}blocks.{i}.attn.qkv.weight''' )
												snake_case   :			Union[str, Any]						    = state_dict.pop(f'''{prefix}blocks.{i}.attn.q_bias''' )
												snake_case   :			Any						    = state_dict.pop(f'''{prefix}blocks.{i}.attn.v_bias''' )

												snake_case   :			List[Any]						    = in_proj_weight[
												    : config.hidden_size, :
												]
												snake_case   :			int						    = q_bias
												snake_case   :			Union[str, Any]						    = in_proj_weight[
												    config.hidden_size : config.hidden_size * 2, :
												]
												snake_case   :			Optional[int]						    = in_proj_weight[
												    -config.hidden_size :, :
												]
												snake_case   :			List[str]						    = v_bias

												# gamma_1 and gamma_2
												# we call them lambda because otherwise they are renamed when using .from_pretrained
												snake_case   :			Tuple						    = state_dict.pop(f'''{prefix}blocks.{i}.gamma_1''' )
												snake_case   :			Optional[int]						    = state_dict.pop(f'''{prefix}blocks.{i}.gamma_2''' )

												snake_case   :			Tuple						    = gamma_a
												snake_case   :			int						    = gamma_a

def  __SCREAMING_SNAKE_CASE		(			lowerCamelCase_:							Any     ,		lowerCamelCase_:							List[Any]     ,		lowerCamelCase_:							Tuple ):
						"""simple docstring"""






						snake_case   :			Union[str, Any]						    = dct.pop(lowerCamelCase_ )
						snake_case   :			Any						    = val

def  __SCREAMING_SNAKE_CASE		(			):
						"""simple docstring"""






						snake_case   :			Dict						    = "http://images.cocodataset.org/val2017/000000039769.jpg"
						snake_case   :			Optional[Any]						    = Image.open(requests.get(lowerCamelCase_     ,		stream=lowerCamelCase_ ).raw )
						return im

@torch.no_grad()
def  __SCREAMING_SNAKE_CASE		(			lowerCamelCase_:							int     ,		lowerCamelCase_:							Optional[Any]     ,		lowerCamelCase_:							str=False ):
						"""simple docstring"""






						snake_case   :			Optional[int]						    = False if "rvlcdip" in checkpoint_url else True
						snake_case   :			Tuple						    = BeitConfig(use_absolute_position_embeddings=lowerCamelCase_     ,		use_mask_token=lowerCamelCase_ )

						# size of the architecture
						if "large" in checkpoint_url or "dit-l" in checkpoint_url:
												snake_case   :			Any						    = 1_0_2_4
												snake_case   :			List[str]						    = 4_0_9_6
												snake_case   :			Dict						    = 2_4
												snake_case   :			Union[str, Any]						    = 1_6

						# labels
						if "rvlcdip" in checkpoint_url:
												snake_case   :			Dict						    = 1_6
												snake_case   :			Tuple						    = "huggingface/label-files"
												snake_case   :			List[Any]						    = "rvlcdip-id2label.json"
												snake_case   :			Tuple						    = json.load(open(hf_hub_download(lowerCamelCase_     ,		lowerCamelCase_     ,		repo_type="dataset" )     ,		"r" ) )
												snake_case   :			Optional[Any]						    = {int(lowerCamelCase_ ): v for k, v in idalabel.items()}
												snake_case   :			Dict						    = idalabel
												snake_case   :			Any						    = {v: k for k, v in idalabel.items()}

						# load state_dict of original model, remove and rename some keys
						snake_case   :			List[str]						    = torch.hub.load_state_dict_from_url(lowerCamelCase_     ,		map_location="cpu" )["model"]

						snake_case   :			int						    = create_rename_keys(lowerCamelCase_     ,		has_lm_head=lowerCamelCase_ )
						for src, dest in rename_keys:
												rename_key(lowerCamelCase_     ,		lowerCamelCase_     ,		lowerCamelCase_ )
						read_in_q_k_v(lowerCamelCase_     ,		lowerCamelCase_     ,		has_lm_head=lowerCamelCase_ )

						# load HuggingFace model
						snake_case   :			Optional[int]						    = BeitForMaskedImageModeling(lowerCamelCase_ ) if has_lm_head else BeitForImageClassification(lowerCamelCase_ )
						model.eval()
						model.load_state_dict(lowerCamelCase_ )

						# Check outputs on an image
						snake_case   :			Tuple						    = BeitImageProcessor(
						    size=config.image_size     ,		resample=PILImageResampling.BILINEAR     ,		do_center_crop=lowerCamelCase_ )
						snake_case   :			Optional[int]						    = prepare_img()

						snake_case   :			List[str]						    = image_processor(images=lowerCamelCase_     ,		return_tensors="pt" )
						snake_case   :			Optional[int]						    = encoding["pixel_values"]

						snake_case   :			List[Any]						    = model(lowerCamelCase_ )
						snake_case   :			str						    = outputs.logits

						# verify logits
						snake_case   :			Any						    = [1, 1_6] if "rvlcdip" in checkpoint_url else [1, 1_9_6, 8_1_9_2]
						assert logits.shape == torch.Size(lowerCamelCase_ ), "Shape of logits not as expected"

						Path(lowerCamelCase_ ).mkdir(exist_ok=lowerCamelCase_ )
						print(f'''Saving model to {pytorch_dump_folder_path}''' )
						model.save_pretrained(lowerCamelCase_ )
						print(f'''Saving image processor to {pytorch_dump_folder_path}''' )
						image_processor.save_pretrained(lowerCamelCase_ )

						if push_to_hub:
												if has_lm_head:
																		snake_case   :			Dict						    = "dit-base" if "base" in checkpoint_url else "dit-large"
												else:
																		snake_case   :			List[str]						    = "dit-base-finetuned-rvlcdip" if "dit-b" in checkpoint_url else "dit-large-finetuned-rvlcdip"
												image_processor.push_to_hub(
												    repo_path_or_name=Path(lowerCamelCase_     ,		lowerCamelCase_ )     ,		organization="nielsr"     ,		commit_message="Add image processor"     ,		use_temp_dir=lowerCamelCase_     ,		)
												model.push_to_hub(
												    repo_path_or_name=Path(lowerCamelCase_     ,		lowerCamelCase_ )     ,		organization="nielsr"     ,		commit_message="Add model"     ,		use_temp_dir=lowerCamelCase_     ,		)


if __name__ == "__main__":
			A		=						argparse.ArgumentParser()

			parser.add_argument(
			    '--checkpoint_url',
			    default='https://layoutlm.blob.core.windows.net/dit/dit-pts/dit-base-224-p16-500k-62d53a.pth',
			    type=str,
			    help='URL to the original PyTorch checkpoint (.pth file).',
			)
			parser.add_argument(
			    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the folder to output PyTorch model.'
			)
			parser.add_argument(
			    '--push_to_hub',
			    action='store_true',
			)
			A		=						parser.parse_args()
			convert_dit_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path, args.push_to_hub)








"""simple docstring"""







import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class       _a  ( SCREAMING_SNAKE_CASE__     ,	unittest.TestCase):
			__magic_name__     					=     RobertaTokenizer
			__magic_name__     					=     RobertaTokenizerFast
			__magic_name__     					=     True
			__magic_name__     					=     {"""cls_token""": """<s>"""}



			def 				__lowercase					(			self							:				Any		)							->					int:
									super().setUp()

									# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
									snake_case   :			int						    = [
									    "l",
									    "o",
									    "w",
									    "e",
									    "r",
									    "s",
									    "t",
									    "i",
									    "d",
									    "n",
									    "\u0120",
									    "\u0120l",
									    "\u0120n",
									    "\u0120lo",
									    "\u0120low",
									    "er",
									    "\u0120lowest",
									    "\u0120newer",
									    "\u0120wider",
									    "<unk>",
									]
									snake_case   :			Optional[int]						    = dict(zip(_lowercase   ,	range(len(_lowercase		)		)		)		)
									snake_case   :			int						    = ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
									snake_case   :			Union[str, Any]						    = {"unk_token": "<unk>"}

									snake_case   :			Optional[Any]						    = os.path.join(self.tmpdirname   ,	VOCAB_FILES_NAMES["vocab_file"]		)
									snake_case   :			str						    = os.path.join(self.tmpdirname   ,	VOCAB_FILES_NAMES["merges_file"]		)
									with open(self.vocab_file   ,	"w"   ,	encoding="utf-8"		) as fp:
															fp.write(json.dumps(_lowercase		) + "\n"		)
									with open(self.merges_file   ,	"w"   ,	encoding="utf-8"		) as fp:
															fp.write("\n".join(_lowercase		)		)



			def 				__lowercase					(			self							:				Optional[Any]   ,	**_lowercase							:				Union[str, Any]		)							->					Dict:
									kwargs.update(self.special_tokens_map		)
									return self.tokenizer_class.from_pretrained(self.tmpdirname   ,	**_lowercase		)



			def 				__lowercase					(			self							:				List[str]   ,	**_lowercase							:				Union[str, Any]		)							->					str:
									kwargs.update(self.special_tokens_map		)
									return RobertaTokenizerFast.from_pretrained(self.tmpdirname   ,	**_lowercase		)



			def 				__lowercase					(			self							:				Union[str, Any]   ,	_lowercase							:				str		)							->					Optional[Any]:
									snake_case   :			Any						    = "lower newer"
									snake_case   :			Union[str, Any]						    = "lower newer"
									return input_text, output_text



			def 				__lowercase					(			self							:				Optional[int]		)							->					Any:
									snake_case   :			Any						    = self.tokenizer_class(self.vocab_file   ,	self.merges_file   ,	**self.special_tokens_map		)
									snake_case   :			Dict						    = "lower newer"
									snake_case   :			Dict						    = ["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
									snake_case   :			Any						    = tokenizer.tokenize(_lowercase		)  # , add_prefix_space=True)
									self.assertListEqual(_lowercase   ,	_lowercase		)

									snake_case   :			List[str]						    = tokens + [tokenizer.unk_token]
									snake_case   :			List[Any]						    = [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase		)   ,	_lowercase		)



			def 				__lowercase					(			self							:				int		)							->					str:
									snake_case   :			List[str]						    = self.get_tokenizer()

									self.assertListEqual(tokenizer.encode("Hello world!"   ,	add_special_tokens=_lowercase		)   ,	[0, 31414, 232, 328, 2]		)
									self.assertListEqual(
									    tokenizer.encode("Hello world! cécé herlolip 418"   ,	add_special_tokens=_lowercase		)   ,	[0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]   ,	)



			@slow
			def 				__lowercase					(			self							:				Dict		)							->					Optional[int]:
									snake_case   :			Tuple						    = self.tokenizer_class.from_pretrained("roberta-base"		)

									snake_case   :			Tuple						    = tokenizer.encode("sequence builders"   ,	add_special_tokens=_lowercase		)
									snake_case   :			Optional[int]						    = tokenizer.encode("multi-sequence build"   ,	add_special_tokens=_lowercase		)

									snake_case   :			Optional[Any]						    = tokenizer.encode(
									    "sequence builders"   ,	add_special_tokens=_lowercase   ,	add_prefix_space=_lowercase		)
									snake_case   :			Optional[int]						    = tokenizer.encode(
									    "sequence builders"   ,	"multi-sequence build"   ,	add_special_tokens=_lowercase   ,	add_prefix_space=_lowercase		)

									snake_case   :			Optional[Any]						    = tokenizer.build_inputs_with_special_tokens(_lowercase		)
									snake_case   :			Any						    = tokenizer.build_inputs_with_special_tokens(_lowercase   ,	_lowercase		)

									assert encoded_sentence == encoded_text_from_decode
									assert encoded_pair == encoded_pair_from_decode



			def 				__lowercase					(			self							:				int		)							->					Any:
									snake_case   :			Any						    = self.get_tokenizer()

									snake_case   :			Optional[Any]						    = "Encode this sequence."
									snake_case   :			List[Any]						    = tokenizer.byte_encoder[" ".encode("utf-8"		)[0]]

									# Testing encoder arguments
									snake_case   :			Any						    = tokenizer.encode(_lowercase   ,	add_special_tokens=_lowercase   ,	add_prefix_space=_lowercase		)
									snake_case   :			Any						    = tokenizer.convert_ids_to_tokens(encoded[0]		)[0]
									self.assertNotEqual(_lowercase   ,	_lowercase		)

									snake_case   :			Dict						    = tokenizer.encode(_lowercase   ,	add_special_tokens=_lowercase   ,	add_prefix_space=_lowercase		)
									snake_case   :			Any						    = tokenizer.convert_ids_to_tokens(encoded[0]		)[0]
									self.assertEqual(_lowercase   ,	_lowercase		)

									tokenizer.add_special_tokens({"bos_token": "<s>"}		)
									snake_case   :			Union[str, Any]						    = tokenizer.encode(_lowercase   ,	add_special_tokens=_lowercase		)
									snake_case   :			List[str]						    = tokenizer.convert_ids_to_tokens(encoded[1]		)[0]
									self.assertNotEqual(_lowercase   ,	_lowercase		)

									# Testing spaces after special tokens
									snake_case   :			Optional[int]						    = "<mask>"
									tokenizer.add_special_tokens(
									    {"mask_token": AddedToken(_lowercase   ,	lstrip=_lowercase   ,	rstrip=_lowercase		)}		)  # mask token has a left space
									snake_case   :			Optional[int]						    = tokenizer.convert_tokens_to_ids(_lowercase		)

									snake_case   :			Tuple						    = "Encode <mask> sequence"
									snake_case   :			Union[str, Any]						    = "Encode <mask>sequence"

									snake_case   :			Union[str, Any]						    = tokenizer.encode(_lowercase		)
									snake_case   :			Optional[int]						    = encoded.index(_lowercase		)
									snake_case   :			int						    = tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]		)[0]
									self.assertEqual(_lowercase   ,	_lowercase		)

									snake_case   :			int						    = tokenizer.encode(_lowercase		)
									snake_case   :			Optional[int]						    = encoded.index(_lowercase		)
									snake_case   :			List[Any]						    = tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]		)[0]
									self.assertNotEqual(_lowercase   ,	_lowercase		)



			def 				__lowercase					(			self							:				Dict		)							->					Union[str, Any]:
									pass



			def 				__lowercase					(			self							:				List[str]		)							->					str:
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
																					snake_case   :			List[Any]						    = self.rust_tokenizer_class.from_pretrained(_lowercase   ,	**_lowercase		)
																					snake_case   :			Optional[int]						    = self.tokenizer_class.from_pretrained(_lowercase   ,	**_lowercase		)
																					snake_case   :			Optional[int]						    = "A, <mask> AllenNLP sentence."
																					snake_case   :			Tuple						    = tokenizer_r.encode_plus(_lowercase   ,	add_special_tokens=_lowercase   ,	return_token_type_ids=_lowercase		)
																					snake_case   :			Optional[Any]						    = tokenizer_p.encode_plus(_lowercase   ,	add_special_tokens=_lowercase   ,	return_token_type_ids=_lowercase		)

																					# token_type_ids should put 0 everywhere
																					self.assertEqual(sum(tokens_r["token_type_ids"]		)   ,	sum(tokens_p["token_type_ids"]		)		)

																					# attention_mask should put 1 everywhere, so sum over length should be 1
																					self.assertEqual(
																					    sum(tokens_r["attention_mask"]		) / len(tokens_r["attention_mask"]		)   ,	sum(tokens_p["attention_mask"]		) / len(tokens_p["attention_mask"]		)   ,	)

																					snake_case   :			List[str]						    = tokenizer_r.convert_ids_to_tokens(tokens_r["input_ids"]		)
																					snake_case   :			List[Any]						    = tokenizer_p.convert_ids_to_tokens(tokens_p["input_ids"]		)

																					# Rust correctly handles the space before the mask while python doesnt
																					self.assertSequenceEqual(tokens_p["input_ids"]   ,	[0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]		)
																					self.assertSequenceEqual(tokens_r["input_ids"]   ,	[0, 250, 6, 50264, 3823, 487, 21992, 3645, 4, 2]		)

																					self.assertSequenceEqual(
																					    _lowercase   ,	["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]		)
																					self.assertSequenceEqual(
																					    _lowercase   ,	["<s>", "A", ",", "<mask>", "ĠAllen", "N", "LP", "Ġsentence", ".", "</s>"]		)



			def 				__lowercase					(			self							:				Tuple		)							->					List[Any]:
									for trim_offsets, add_prefix_space in itertools.product([True, False]   ,	repeat=2		):
															snake_case   :			List[Any]						    = self.rust_tokenizer_class.from_pretrained(
															    self.tmpdirname   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)

															snake_case   :			Optional[int]						    = json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()		)
															snake_case   :			Union[str, Any]						    = json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()		)

															self.assertEqual(pre_tokenizer_state["add_prefix_space"]   ,	_lowercase		)

															self.assertEqual(post_processor_state["add_prefix_space"]   ,	_lowercase		)
															self.assertEqual(post_processor_state["trim_offsets"]   ,	_lowercase		)



			def 				__lowercase					(			self							:				int		)							->					List[str]:
									# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
									# `trim_offsets`
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
																					snake_case   :			List[Any]						    = "hello"  # `hello` is a token in the vocabulary of `pretrained_name`
																					snake_case   :			List[str]						    = F'''{text_of_1_token} {text_of_1_token}'''

																					snake_case   :			Optional[Any]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Union[str, Any]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(len(_lowercase		) + 1, len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			List[Any]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Tuple						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(len(_lowercase		) + 1, len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Dict						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			int						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(len(_lowercase		), len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Optional[int]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Optional[int]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(len(_lowercase		), len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Dict						    = F''' {text}'''

																					# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																					#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																					# )
																					# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																					# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																					# self.assertEqual(
																					#     encoding.offset_mapping[1],
																					#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																					# )

																					snake_case   :			List[str]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			List[str]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(1, 1 + len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(1 + len(_lowercase		) + 1, 1 + len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Union[str, Any]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Optional[int]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, 1 + len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(1 + len(_lowercase		), 1 + len(_lowercase		) + 1 + len(_lowercase		))   ,	)

																					snake_case   :			Optional[Any]						    = self.rust_tokenizer_class.from_pretrained(
																					    _lowercase   ,	use_fast=_lowercase   ,	add_prefix_space=_lowercase   ,	trim_offsets=_lowercase		)
																					snake_case   :			Union[str, Any]						    = tokenizer_r(_lowercase   ,	return_offsets_mapping=_lowercase   ,	add_special_tokens=_lowercase		)
																					self.assertEqual(encoding.offset_mapping[0]   ,	(0, 1 + len(_lowercase		))		)
																					self.assertEqual(
																					    encoding.offset_mapping[1]   ,	(1 + len(_lowercase		), 1 + len(_lowercase		) + 1 + len(_lowercase		))   ,	)



