
"""simple docstring"""


import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__A       						=       logging.get_logger(__name__)

__A       						=       {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__A       						=       {
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

__A       						=       {
    'openai-gpt': 512,
}



def     _lowerCamelCase(__UpperCamelCase       )				-> Tuple:
	_lowerCAmelCase								=text.strip()
	if not text:
		return []
	_lowerCAmelCase								=text.split()
	return tokens







class 		lowerCamelCase__					(					__magic_name__				):





			'''simple docstring'''



			def __init__(							self     ,     __UpperCAmelCase=True     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase=True     ,     __UpperCAmelCase=None     ,     __UpperCAmelCase=True     ,     ) ->     List[str]:
				if never_split is None:
					_lowerCAmelCase								=[]
				_lowerCAmelCase								=do_lower_case
				_lowerCAmelCase								=set(__UpperCAmelCase  )
				_lowerCAmelCase								=tokenize_chinese_chars
				_lowerCAmelCase								=strip_accents
				_lowerCAmelCase								=do_split_on_punc



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase     ,     __UpperCAmelCase=None  ) ->     Any:
				_lowerCAmelCase								=self.never_split.union(set(__UpperCAmelCase  )  ) if never_split else self.never_split
				_lowerCAmelCase								=self._clean_text(__UpperCAmelCase  )

				# This was added on November 1st, 2018 for the multilingual and Chinese
				# models. This is also applied to the English models now, but it doesn't
				# matter since the English models were not trained on any Chinese data
				# and generally don't have any Chinese data in them (there are Chinese
				# characters in the vocabulary because Wikipedia does have some Chinese
				# words in the English Wikipedia.).
				if self.tokenize_chinese_chars:
					_lowerCAmelCase								=self._tokenize_chinese_chars(__UpperCAmelCase  )
				# prevents treating the same character with different unicode codepoints as different characters
				_lowerCAmelCase								=unicodedata.normalize("""NFC"""     ,     __UpperCAmelCase  )
				_lowerCAmelCase								=whitespace_tokenize(__UpperCAmelCase  )
				_lowerCAmelCase								=[]
				for token in orig_tokens:
					if token not in never_split:
						if self.do_lower_case:
							_lowerCAmelCase								=token.lower()
							if self.strip_accents is not False:
								_lowerCAmelCase								=self._run_strip_accents(__UpperCAmelCase  )
						elif self.strip_accents:
							_lowerCAmelCase								=self._run_strip_accents(__UpperCAmelCase  )
					split_tokens.extend(self._run_split_on_punc(__UpperCAmelCase     ,     __UpperCAmelCase  )  )

				_lowerCAmelCase								=whitespace_tokenize(""" """.join(__UpperCAmelCase  )  )
				return output_tokens



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase  ) ->     Tuple:
				_lowerCAmelCase								=unicodedata.normalize("""NFD"""     ,     __UpperCAmelCase  )
				_lowerCAmelCase								=[]
				for char in text:
					_lowerCAmelCase								=unicodedata.category(__UpperCAmelCase  )
					if cat == "Mn":
						continue
					output.append(__UpperCAmelCase  )
				return "".join(__UpperCAmelCase  )



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase     ,     __UpperCAmelCase=None  ) ->     Any:
				if not self.do_split_on_punc or (never_split is not None and text in never_split):
					return [text]
				_lowerCAmelCase								=list(__UpperCAmelCase  )
				_lowerCAmelCase								=0
				_lowerCAmelCase								=True
				_lowerCAmelCase								=[]
				while i < len(__UpperCAmelCase  ):
					_lowerCAmelCase								=chars[i]
					if _is_punctuation(__UpperCAmelCase  ):
						output.append([char]  )
						_lowerCAmelCase								=True
					else:
						if start_new_word:
							output.append([]  )
						_lowerCAmelCase								=False
						output[-1].append(__UpperCAmelCase  )
					i += 1

				return ["".join(__UpperCAmelCase  ) for x in output]



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase  ) ->     str:
				_lowerCAmelCase								=[]
				for char in text:
					_lowerCAmelCase								=ord(__UpperCAmelCase  )
					if self._is_chinese_char(__UpperCAmelCase  ):
						output.append(""" """  )
						output.append(__UpperCAmelCase  )
						output.append(""" """  )
					else:
						output.append(__UpperCAmelCase  )
				return "".join(__UpperCAmelCase  )



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase  ) ->     Tuple:
				# This defines a "chinese character" as anything in the CJK Unicode block:
				#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
				#
				# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
				# despite its name. The modern Korean Hangul alphabet is a different block,
				# as is Japanese Hiragana and Katakana. Those alphabets are used to write
				# space-separated words, so they are not treated specially and handled
				# like the all of the other languages.
				if (
				    (cp >= 0x4_e_0_0 and cp <= 0x9_f_f_f)
				    or (cp >= 0x3_4_0_0 and cp <= 0x4_d_b_f)  #
				    or (cp >= 0x2_0_0_0_0 and cp <= 0x2_a_6_d_f)  #
				    or (cp >= 0x2_a_7_0_0 and cp <= 0x2_b_7_3_f)  #
				    or (cp >= 0x2_b_7_4_0 and cp <= 0x2_b_8_1_f)  #
				    or (cp >= 0x2_b_8_2_0 and cp <= 0x2_c_e_a_f)  #
				    or (cp >= 0xf_9_0_0 and cp <= 0xf_a_f_f)
				    or (cp >= 0x2_f_8_0_0 and cp <= 0x2_f_a_1_f)  #
				):  #
					return True

				return False



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase  ) ->     Optional[Any]:
				_lowerCAmelCase								=[]
				for char in text:
					_lowerCAmelCase								=ord(__UpperCAmelCase  )
					if cp == 0 or cp == 0xf_f_f_d or _is_control(__UpperCAmelCase  ):
						continue
					if _is_whitespace(__UpperCAmelCase  ):
						output.append(""" """  )
					else:
						output.append(__UpperCAmelCase  )
				return "".join(__UpperCAmelCase  )







def     _lowerCamelCase(__UpperCamelCase       )				-> Union[str, Any]:
	_lowerCAmelCase								=set()
	_lowerCAmelCase								=word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)       )
		_lowerCAmelCase								=char
	return pairs



def     _lowerCamelCase(__UpperCamelCase       )				-> Dict:
	_lowerCAmelCase								=text.replace("""—"""    , """-"""       )
	_lowerCAmelCase								=text.replace("""–"""    , """-"""       )
	_lowerCAmelCase								=text.replace("""―"""    , """-"""       )
	_lowerCAmelCase								=text.replace("""…"""    , """..."""       )
	_lowerCAmelCase								=text.replace("""´"""    , """'"""       )
	_lowerCAmelCase								=re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""    , R""" \1 """    , __UpperCamelCase       )
	_lowerCAmelCase								=re.sub(R"""\s*\n\s*"""    , """ \n """    , __UpperCamelCase       )
	_lowerCAmelCase								=re.sub(R"""[^\S\n]+"""    , """ """    , __UpperCamelCase       )
	return text.strip()







class 		lowerCamelCase__					(					__magic_name__				):





			'''simple docstring'''
			lowerCamelCase  					=		VOCAB_FILES_NAMES
			lowerCamelCase  					=		PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase  					=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase  					=		['''input_ids''', '''attention_mask''']



			def __init__(							self     ,     __UpperCAmelCase     ,     __UpperCAmelCase     ,     __UpperCAmelCase="<unk>"     ,     **__UpperCAmelCase  ) ->     Union[str, Any]:
				super().__init__(unk_token=__UpperCAmelCase     ,     **__UpperCAmelCase  )

				try:
					import ftfy
					from spacy.lang.en import English

					_lowerCAmelCase								=English()
					_lowerCAmelCase								=_nlp.tokenizer
					_lowerCAmelCase								=ftfy.fix_text
				except ImportError:
					logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""  )
					_lowerCAmelCase								=BasicTokenizer(do_lower_case=__UpperCAmelCase  )
					_lowerCAmelCase								=None

				with open(__UpperCAmelCase     ,     encoding="""utf-8"""  ) as vocab_handle:
					_lowerCAmelCase								=json.load(__UpperCAmelCase  )
				_lowerCAmelCase								={v: k for k, v in self.encoder.items()}
				with open(__UpperCAmelCase     ,     encoding="""utf-8"""  ) as merges_handle:
					_lowerCAmelCase								=merges_handle.read().split("""\n"""  )[1:-1]
				_lowerCAmelCase								=[tuple(merge.split()  ) for merge in merges]
				_lowerCAmelCase								=dict(zip(__UpperCAmelCase     ,     range(len(__UpperCAmelCase  )  )  )  )
				_lowerCAmelCase								={}



			@property
			def 					_lowerCAmelCase		(							self  ) ->     Dict:
				return True



			@property
			def 					_lowerCAmelCase		(							self  ) ->     int:
				return len(self.encoder  )



			def 					_lowerCAmelCase		(							self  ) ->     int:
				return dict(self.encoder     ,     **self.added_tokens_encoder  )



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase  ) ->     Tuple:
				_lowerCAmelCase								=tuple(token[:-1]  ) + (token[-1] + """</w>""",)
				if token in self.cache:
					return self.cache[token]
				_lowerCAmelCase								=get_pairs(__UpperCAmelCase  )

				if not pairs:
					return token + "</w>"

				while True:
					_lowerCAmelCase								=min(__UpperCAmelCase     ,     key=lambda __UpperCAmelCase  : self.bpe_ranks.get(__UpperCAmelCase     ,     float("""inf"""  )  )  )
					if bigram not in self.bpe_ranks:
						break
					_lowerCAmelCase						,							_lowerCAmelCase								=bigram
					_lowerCAmelCase								=[]
					_lowerCAmelCase								=0
					while i < len(__UpperCAmelCase  ):
						try:
							_lowerCAmelCase								=word.index(__UpperCAmelCase     ,     __UpperCAmelCase  )
						except ValueError:
							new_word.extend(word[i:]  )
							break
						else:
							new_word.extend(word[i:j]  )
							_lowerCAmelCase								=j

						if word[i] == first and i < len(__UpperCAmelCase  ) - 1 and word[i + 1] == second:
							new_word.append(first + second  )
							i += 2
						else:
							new_word.append(word[i]  )
							i += 1
					_lowerCAmelCase								=tuple(__UpperCAmelCase  )
					_lowerCAmelCase								=new_word
					if len(__UpperCAmelCase  ) == 1:
						break
					else:
						_lowerCAmelCase								=get_pairs(__UpperCAmelCase  )
				_lowerCAmelCase								=""" """.join(__UpperCAmelCase  )
				if word == "\n  </w>":
					_lowerCAmelCase								="""\n</w>"""
				_lowerCAmelCase								=word
				return word



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase  ) ->     int:
				_lowerCAmelCase								=[]
				if self.fix_text is None:
					# Using BERT's BasicTokenizer
					_lowerCAmelCase								=self.nlp.tokenize(__UpperCAmelCase  )
					for token in text:
						split_tokens.extend(list(self.bpe(__UpperCAmelCase  ).split(""" """  )  )  )
				else:
					# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
					_lowerCAmelCase								=self.nlp(text_standardize(self.fix_text(__UpperCAmelCase  )  )  )
					for token in text:
						split_tokens.extend(list(self.bpe(token.text.lower()  ).split(""" """  )  )  )
				return split_tokens



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase  ) ->     Union[str, Any]:
				return self.encoder.get(__UpperCAmelCase     ,     self.encoder.get(self.unk_token  )  )



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase  ) ->     List[Any]:
				return self.decoder.get(__UpperCAmelCase     ,     self.unk_token  )



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase  ) ->     Dict:
				_lowerCAmelCase								="""""".join(__UpperCAmelCase  ).replace("""</w>"""     ,     """ """  ).strip()
				return out_string



			def 					_lowerCAmelCase		(							self     ,     __UpperCAmelCase     ,     __UpperCAmelCase = None  ) ->     Tuple[str]:
				if not os.path.isdir(__UpperCAmelCase  ):
					logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
					return
				_lowerCAmelCase								=os.path.join(
				    __UpperCAmelCase     ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
				_lowerCAmelCase								=os.path.join(
				    __UpperCAmelCase     ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]  )

				with open(__UpperCAmelCase     ,     """w"""     ,     encoding="""utf-8"""  ) as f:
					f.write(json.dumps(self.encoder     ,     indent=2     ,     sort_keys=__UpperCAmelCase     ,     ensure_ascii=__UpperCAmelCase  ) + """\n"""  )

				_lowerCAmelCase								=0
				with open(__UpperCAmelCase     ,     """w"""     ,     encoding="""utf-8"""  ) as writer:
					writer.write("""#version: 0.2\n"""  )
					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,     key=lambda __UpperCAmelCase  : kv[1]  ):
						if index != token_index:
							logger.warning(
							    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
							    """ Please check that the tokenizer is not corrupted!"""  )
							_lowerCAmelCase								=token_index
						writer.write(""" """.join(__UpperCAmelCase  ) + """\n"""  )
						index += 1

				return vocab_file, merge_file





"""simple docstring"""


import itertools



def     _lowerCamelCase(__UpperCamelCase       )				-> List[str]:
	return (
	    int("""""".join(combination[0:2]       )       ) * int("""""".join(combination[2:5]       )       )
	    == int("""""".join(combination[5:9]       )       )
	) or (
	    int("""""".join(combination[0]       )       ) * int("""""".join(combination[1:5]       )       )
	    == int("""""".join(combination[5:9]       )       )
	)



def     _lowerCamelCase()				-> Dict:
	return sum(
	    {
	        int("""""".join(pandigital[5:9]       )       )
	        for pandigital in itertools.permutations("""123456789"""       )
	        if is_combination_valid(__UpperCamelCase       )
	    }       )


if __name__ == "__main__":
			print(solution())



