

"""simple docstring"""





from collections import Counter
import datasets
import transformers
from transformers.convert_slow_tokenizer import SLOW_TO_FAST_CONVERTERS

from transformers.utils import logging

logging.set_verbosity_info()

_SCREAMING_SNAKE_CASE				:     Any       				=       {
    name: (getattr(transformers, name), getattr(transformers, name + """Fast""")) for name in SLOW_TO_FAST_CONVERTERS
}

_SCREAMING_SNAKE_CASE				:     Tuple       				=       datasets.load_dataset("""xnli""", split="""test+validation""")

_SCREAMING_SNAKE_CASE				:     Union[str, Any]       				=       0
_SCREAMING_SNAKE_CASE				:     Tuple       				=       0
_SCREAMING_SNAKE_CASE				:     Optional[int]       				=       0
_SCREAMING_SNAKE_CASE				:     int       				=       0



def 				_lowerCAmelCase					(  UpperCAmelCase   :							Optional[int]					,							UpperCAmelCase   :							Optional[Any]					,							UpperCAmelCase   :							str					,							UpperCAmelCase   :							int ):


					'''simple docstring'''






					if spm_diff == list(reversed(UpperCAmelCase ) ):
										# AAA -> AA+A vs A+AA case.
										return True
					elif len(UpperCAmelCase ) == len(UpperCAmelCase ) and fast.decode(UpperCAmelCase ) == fast.decode(UpperCAmelCase ):
										# Second order OK
										# Barrich -> Barr + ich vs Bar + rich
										return True
					UpperCamelCase__						:			int				     =slow.encode(slow.decode(UpperCAmelCase ) )
					UpperCamelCase__						:			int				     =fast.encode(fast.decode(UpperCAmelCase ) )
					if spm_reencoded != spm_diff and spm_reencoded == tok_reencoded:
										# Type 3 error.
										# Snehagatha ->
										#       Sne, h, aga, th, a
										#       Sne, ha, gat, ha
										# Encoding the wrong with sp does not even recover what spm gave us
										# It fits tokenizer however...
										return True
					return False



def 				_lowerCAmelCase					(  UpperCAmelCase   :							Optional[int]					,							UpperCAmelCase   :							Dict					,							UpperCAmelCase   :							Any ):


					'''simple docstring'''






					UpperCamelCase__						:			Any				     =fast.encode_plus(UpperCAmelCase )[0]
					UpperCamelCase__						:			int				     =enc.offsets
					UpperCamelCase__						:			List[str]				     =offsets[idx], offsets[idx - 1]
					if curr is not None and line[curr[0] : curr[1]] == "\u200f":
										return True
					if prev is not None and line[prev[0] : prev[1]] == "\u200f":
										return True



def 				_lowerCAmelCase					(  UpperCAmelCase   :							Optional[int]					,							UpperCAmelCase   :							Optional[Any]					,							UpperCAmelCase   :							Tuple					,							UpperCAmelCase   :							Optional[int]					,							UpperCAmelCase   :							int ):


					'''simple docstring'''






					for i, (spm_id, tok_id) in enumerate(zip(UpperCAmelCase					,							UpperCAmelCase ) ):
										if spm_id != tok_id:
															break
					UpperCamelCase__						:			int				     =i
					for i, (spm_id, tok_id) in enumerate(zip(reversed(UpperCAmelCase )					,							reversed(UpperCAmelCase ) ) ):
										if spm_id != tok_id:
															break
					UpperCamelCase__						:			str				     =len(UpperCAmelCase ) - i

					UpperCamelCase__						:			List[str]				     =spm_ids[first:last]
					UpperCamelCase__						:			Optional[int]				     =tok_ids[first:last]

					if check_diff(UpperCAmelCase					,							UpperCAmelCase					,							UpperCAmelCase					,							UpperCAmelCase ):
										return True

					if check_LTR_mark(UpperCAmelCase					,							UpperCAmelCase					,							UpperCAmelCase ):
										return True

					if last - first > 5:
										# We might have twice a single problem, attempt to subdivide the disjointed tokens into smaller problems
										UpperCamelCase__						:			Dict				     =Counter(spm_ids[first:last] )
										UpperCamelCase__						:			Optional[Any]				     =Counter(tok_ids[first:last] )

										UpperCamelCase__						:			Any				     ={spm_ for (spm_, si) in spms.items() if toks.get(spm_					,							0 ) == si}
										UpperCamelCase__						:			str				     =3
										for i in range(last - first - min_width ):
															if all(spm_ids[first + i + j] in removable_tokens for j in range(UpperCAmelCase ) ):
																				UpperCamelCase__						:			List[str]				     =[
																				    k
																				    for k in range(last - first - min_width )
																				    if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
																				]
																				for j in possible_matches:
																									if check_diff(spm_ids[first : first + i]					,							tok_ids[first : first + j]					,							UpperCAmelCase					,							UpperCAmelCase ) and check_details(
																									    UpperCAmelCase					,							spm_ids[first + i : last]					,							tok_ids[first + j : last]					,							UpperCAmelCase					,							UpperCAmelCase					,							):
																														return True

					print(F'''Spm: {[fast.decode([spm_ids[i]] ) for i in range(UpperCAmelCase					,							UpperCAmelCase )]}''' )
					try:
										print(F'''Tok: {[fast.decode([tok_ids[i]] ) for i in range(UpperCAmelCase					,							UpperCAmelCase )]}''' )
					except Exception:
										pass

					UpperCamelCase__						:			Optional[Any]				     =fast.decode(spm_ids[:first] )
					UpperCamelCase__						:			Optional[int]				     =fast.decode(spm_ids[last:] )
					UpperCamelCase__						:			Optional[Any]				     =fast.decode(spm_ids[first:last] )
					print()
					print(UpperCAmelCase )
					return False



def 				_lowerCAmelCase					(  UpperCAmelCase   :							Tuple					,							UpperCAmelCase   :							Any					,							UpperCAmelCase   :							Union[str, Any] ):


					'''simple docstring'''






					global perfect
					global imperfect
					global wrong
					global total

					UpperCamelCase__						:			Optional[Any]				     =slow.encode(UpperCAmelCase )
					UpperCamelCase__						:			Optional[int]				     =fast.encode(UpperCAmelCase )

					UpperCamelCase__						:			Any				     =False
					total += 1

					if slow_ids != fast_ids:
										if check_details(UpperCAmelCase					,							UpperCAmelCase					,							UpperCAmelCase					,							UpperCAmelCase					,							UpperCAmelCase ):
															UpperCamelCase__						:			List[str]				     =True
															imperfect += 1
										else:
															wrong += 1
					else:
										perfect += 1

					if total % 10_000 == 0:
										print(F'''({perfect} / {imperfect} / {wrong} ----- {perfect + imperfect + wrong})''' )

					if skip_assert:
										return

					assert (
					    slow_ids == fast_ids
					), F'''line {text} : \n\n{slow_ids}\n{fast_ids}\n\n{slow.tokenize(UpperCAmelCase )}\n{fast.tokenize(UpperCAmelCase )}'''



def 				_lowerCAmelCase					(  UpperCAmelCase   :							List[str]					,							UpperCAmelCase   :							Any ):


					'''simple docstring'''






					global batch_total
					for i in range(len(UpperCAmelCase ) ):
										# premise, all languages
										for text in dataset[i]["premise"].values():
															test_string(UpperCAmelCase					,							UpperCAmelCase					,							UpperCAmelCase )

										# hypothesis, all languages
										for text in dataset[i]["hypothesis"]["translation"]:
															test_string(UpperCAmelCase					,							UpperCAmelCase					,							UpperCAmelCase )


if __name__ == "__main__":
				for name, (slow_class, fast_class) in TOKENIZER_CLASSES.items():
								_SCREAMING_SNAKE_CASE				:     Optional[int]       				=       list(slow_class.max_model_input_sizes.keys())
								for checkpoint in checkpoint_names:
												_SCREAMING_SNAKE_CASE				:     str       				=       0
												_SCREAMING_SNAKE_CASE				:     str       				=       0
												_SCREAMING_SNAKE_CASE				:     Union[str, Any]       				=       0
												_SCREAMING_SNAKE_CASE				:     Optional[Any]       				=       0

												print(F'''========================== Checking {name}: {checkpoint} ==========================''')
												_SCREAMING_SNAKE_CASE				:     Dict       				=       slow_class.from_pretrained(checkpoint, force_download=True)
												_SCREAMING_SNAKE_CASE				:     Optional[int]       				=       fast_class.from_pretrained(checkpoint, force_download=True)
												test_tokenizer(slow, fast)
												print(F'''Accuracy {perfect * 1_0_0 / total:.2f}''')








def  lowerCAmelCase  (	UpperCAmelCase       )  ->int:






					"""simple docstring"""






					return n * (2 * n - 1)







def  lowerCAmelCase  (	UpperCAmelCase       )  ->bool:






					"""simple docstring"""






					__magic_name__					:							Any		   =							(1 + 24 * n) ** 0.5
					return ((1 + root) / 6) % 1 == 0







def  lowerCAmelCase  (	UpperCAmelCase = 144       )  ->int:






					"""simple docstring"""






					__magic_name__					:							Optional[int]		   =							start
					__magic_name__					:							int		   =							hexagonal_num(UpperCAmelCase       )
					while not is_pentagonal(UpperCAmelCase       ):
										n += 1
										__magic_name__					:							Optional[int]		   =							hexagonal_num(UpperCAmelCase       )
					return num


if __name__ == "__main__":
				print(f"{solution()} = ")



