
"""simple docstring"""




from collections import Counter
import datasets
import transformers
from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS

from transformers.utils import logging

logging.set_verbosity_info()

snake_case    =		{
    name: (getattr(transformers, name), getattr(transformers, name + 'Fast')) for name in SLOW_TO_FAST_CONVERTERS
}

snake_case    =		datasets.load_dataset('xnli', split='test+validation')

snake_case    =		0
snake_case    =		0
snake_case    =		0
snake_case    =		0

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			):
					if spm_diff == list(reversed(SCREAMING_SNAKE_CASE_			)			):
										# AAA -> AA+A vs A+AA case.
										return True
					elif len(SCREAMING_SNAKE_CASE_			) == len(SCREAMING_SNAKE_CASE_			) and fast.decode(SCREAMING_SNAKE_CASE_			) == fast.decode(SCREAMING_SNAKE_CASE_			):
										# Second order OK
										# Barrich -> Barr + ich vs Bar + rich
										return True
					SCREAMING_SNAKE_CASE				=	slow.encode(slow.decode(SCREAMING_SNAKE_CASE_			)			)
					SCREAMING_SNAKE_CASE				=	fast.encode(fast.decode(SCREAMING_SNAKE_CASE_			)			)
					if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
										# Type 3 error.
										# Snehagatha ->
										#       Sne, h, aga, th, a
										#       Sne, ha, gat, ha
										# Encoding the wrong with sp does not even recover what spm gave us
										# It fits tokenizer however...
										return True
					return False

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			):
					SCREAMING_SNAKE_CASE				=	fast.encode_plus(SCREAMING_SNAKE_CASE_			)[0]
					SCREAMING_SNAKE_CASE				=	enc.offsets
					SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE				=	offsets[idx], offsets[idx - 1]
					if curr is not None and line[curr[0] : curr[1]] == "\u200f":
										return True
					if prev is not None and line[prev[0] : prev[1]] == "\u200f":
										return True

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			):
					# Encoding can be the same with same result AAA -> A + AA vs AA + A
					# We can check that we use at least exactly the same number of tokens.
					for i, (spm_id, tok_id) in enumerate(zip(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			)			):
										if spm_id != tok_id:
															break
					SCREAMING_SNAKE_CASE				=	i
					for i, (spm_id, tok_id) in enumerate(zip(reversed(SCREAMING_SNAKE_CASE_			),     reversed(SCREAMING_SNAKE_CASE_			)			)			):
										if spm_id != tok_id:
															break
					SCREAMING_SNAKE_CASE				=	len(SCREAMING_SNAKE_CASE_			) - i

					SCREAMING_SNAKE_CASE				=	spm_ids[first:last]
					SCREAMING_SNAKE_CASE				=	tok_ids[first:last]

					if check_diff(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			):
										return True

					if check_LTR_mark(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			):
										return True

					if last - first > 5:
										# We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems
										SCREAMING_SNAKE_CASE				=	Counter(spm_ids[first:last]			)
										SCREAMING_SNAKE_CASE				=	Counter(tok_ids[first:last]			)

										SCREAMING_SNAKE_CASE				=	{spm_ for (spm_, si) in spms.items() if toks.get(spm_,     0			) == si}
										SCREAMING_SNAKE_CASE				=	3
										for i in range(last - first - min_width			):
															if all(spm_ids[first + i + j] in removable_tokens for j in range(SCREAMING_SNAKE_CASE_			)			):
																				SCREAMING_SNAKE_CASE				=	[
																				    k
																				    for k in range(last - first - min_width			)
																				    if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
																				]
																				for j in possible_matches:
																									if check_diff(spm_ids[first : first + i],     tok_ids[first : first + j],     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			) and check_details(
																									    SCREAMING_SNAKE_CASE_,     spm_ids[first + i : last],     tok_ids[first + j : last],     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     ):
																														return True

					print(f'''Spm: {[fast.decode([spm_ids[i]]			) for i in range(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			)]}'''			)
					try:
										print(f'''Tok: {[fast.decode([tok_ids[i]]			) for i in range(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			)]}'''			)
					except Exception:
										pass

					SCREAMING_SNAKE_CASE				=	fast.decode(spm_ids[:first]			)
					SCREAMING_SNAKE_CASE				=	fast.decode(spm_ids[last:]			)
					SCREAMING_SNAKE_CASE				=	fast.decode(spm_ids[first:last]			)
					print()
					print(SCREAMING_SNAKE_CASE_			)
					return False

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			):
					global perfect
					global imperfect
					global wrong
					global total

					SCREAMING_SNAKE_CASE				=	slow.encode(SCREAMING_SNAKE_CASE_			)
					SCREAMING_SNAKE_CASE				=	fast.encode(SCREAMING_SNAKE_CASE_			)

					SCREAMING_SNAKE_CASE				=	False
					total += 1

					if slow_ids != fast_ids:
										if check_details(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			):
															SCREAMING_SNAKE_CASE				=	True
															imperfect += 1
										else:
															wrong += 1
					else:
										perfect += 1

					if total % 1_0_0_0_0 == 0:
										print(f'''({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})'''			)

					if skip_assert:
										return

					assert (
					    slow_ids == fast_ids
					), f'''line {text} : \n\n{slow_ids}\n{fast_ids}\n\n{slow.tokenize(SCREAMING_SNAKE_CASE_			)}\n{fast.tokenize(SCREAMING_SNAKE_CASE_			)}'''

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			):
					global batch_total
					for i in range(len(SCREAMING_SNAKE_CASE_			)			):
										# premise, all languages
										for text in dataset[i]["premise"].values():
															test_string(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			)

										# hypothesis, all languages
										for text in dataset[i]["hypothesis"]["translation"]:
															test_string(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			)


if __name__ == "__main__":
	for name, (slow_class, fast_class) in TOKENIZER_CLASSES.items():
		snake_case    =		list(slow_class.max_model_input_sizes.keys())
		for checkpoint in checkpoint_names:
			snake_case    =		0
			snake_case    =		0
			snake_case    =		0
			snake_case    =		0

			print(f'========================== Checking {name}: {checkpoint} ==========================')
			snake_case    =		slow_class.from_pretrained(checkpoint, force_download=True)
			snake_case    =		fast_class.from_pretrained(checkpoint, force_download=True)
			test_tokenizer(slow, fast)
			print(f'Accuracy {perfect * 1_0_0 / total:.2f}')


"""simple docstring"""




def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_			):
					if not isinstance(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			):
										SCREAMING_SNAKE_CASE				=	(
										    'operation can not be conducted on a object of type '
										    f'''{type(SCREAMING_SNAKE_CASE_			).__name__}'''
										)
										raise TypeError(SCREAMING_SNAKE_CASE_			)
					SCREAMING_SNAKE_CASE				=	''
					for _ in range(0,     3_2			):
										bit_string += str(number % 2			)
										SCREAMING_SNAKE_CASE				=	number >> 1
					return bit_string

def   UpperCamelCase_  (		SCREAMING_SNAKE_CASE_			):
					if number < 0:
										raise ValueError('the value of input must be positive'			)
					elif isinstance(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			):
										raise TypeError('Input value must be a \'int\' type'			)
					elif isinstance(SCREAMING_SNAKE_CASE_,     SCREAMING_SNAKE_CASE_			):
										raise TypeError('\'<\' not supported between instances of \'str\' and \'int\''			)
					SCREAMING_SNAKE_CASE				=	0
					# iterator over [1 to 32],since we are dealing with 32 bit integer
					for _ in range(1,     3_3			):
										# left shift the bits by unity
										SCREAMING_SNAKE_CASE				=	result << 1
										# get the end bit
										SCREAMING_SNAKE_CASE				=	number % 2
										# right shift the bits by unity
										SCREAMING_SNAKE_CASE				=	number >> 1
										# add that bit to our ans
										SCREAMING_SNAKE_CASE				=	result | end_bit
					return get_reverse_bit_string(SCREAMING_SNAKE_CASE_			)


if __name__ == "__main__":
	import doctest

	doctest.testmod()
