




import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


lowercase_		=							logging.get_logger(__name__)

lowercase_		=							{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

lowercase_		=							{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

lowercase_		=							{
    '''openai-gpt''': 512,
}







def  lowerCAmelCase  (	UpperCAmelCase       )  ->List[str]:






					"""simple docstring"""






					__magic_name__					:							str		   =							text.strip()
					if not text:
										return []
					__magic_name__					:							Any		   =							text.split()
					return tokens





class 		A__				(   __SCREAMING_SNAKE_CASE ):






	def __init__(	self				,     lowerCamelCase=True				,     lowerCamelCase=None				,     lowerCamelCase=True				,     lowerCamelCase=None				,     lowerCamelCase=True				,     )      ->   Optional[int]:




						"""simple docstring"""






						if never_split is None:
											__magic_name__					:							Union[str, Any]		   =							[]
						__magic_name__					:							Any		   =							do_lower_case
						__magic_name__					:							Tuple		   =							set(lowerCamelCase						)
						__magic_name__					:							Optional[int]		   =							tokenize_chinese_chars
						__magic_name__					:							List[str]		   =							strip_accents
						__magic_name__					:							List[Any]		   =							do_split_on_punc






	def  lowercase      (	self				,     lowerCamelCase				,     lowerCamelCase=None						)      ->   Tuple:




						"""simple docstring"""






						__magic_name__					:							Optional[Any]		   =							self.never_split.union(set(lowerCamelCase						)						) if never_split else self.never_split
						__magic_name__					:							Tuple		   =							self._clean_text(lowerCamelCase						)

						# This was added on November 1st, 2018 for the multilingual and Chinese
						# models. This is also applied to the English models now, but it doesn't
						# matter since the English models were not trained on any Chinese data
						# and generally don't have any Chinese data in them (there are Chinese
						# characters in the vocabulary because Wikipedia does have some Chinese
						# words in the English Wikipedia.).
						if self.tokenize_chinese_chars:
											__magic_name__					:							Dict		   =							self._tokenize_chinese_chars(lowerCamelCase						)
						# prevents treating the same character with different unicode codepoints as different characters
						__magic_name__					:							str		   =							unicodedata.normalize('''NFC'''				,     lowerCamelCase						)
						__magic_name__					:							int		   =							whitespace_tokenize(lowerCamelCase						)
						__magic_name__					:							Dict		   =							[]
						for token in orig_tokens:
											if token not in never_split:
																if self.do_lower_case:
																					__magic_name__					:							int		   =							token.lower()
																					if self.strip_accents is not False:
																										__magic_name__					:							Dict		   =							self._run_strip_accents(lowerCamelCase						)
																elif self.strip_accents:
																					__magic_name__					:							List[str]		   =							self._run_strip_accents(lowerCamelCase						)
											split_tokens.extend(self._run_split_on_punc(lowerCamelCase				,     lowerCamelCase						)						)

						__magic_name__					:							Union[str, Any]		   =							whitespace_tokenize(''' '''.join(lowerCamelCase						)						)
						return output_tokens






	def  lowercase      (	self				,     lowerCamelCase						)      ->   List[Any]:




						"""simple docstring"""






						__magic_name__					:							Tuple		   =							unicodedata.normalize('''NFD'''				,     lowerCamelCase						)
						__magic_name__					:							Optional[int]		   =							[]
						for char in text:
											__magic_name__					:							Optional[int]		   =							unicodedata.category(lowerCamelCase						)
											if cat == "Mn":
																continue
											output.append(lowerCamelCase						)
						return "".join(lowerCamelCase						)






	def  lowercase      (	self				,     lowerCamelCase				,     lowerCamelCase=None						)      ->   Dict:




						"""simple docstring"""






						if not self.do_split_on_punc or (never_split is not None and text in never_split):
											return [text]
						__magic_name__					:							str		   =							list(lowerCamelCase						)
						__magic_name__					:							Union[str, Any]		   =							0
						__magic_name__					:							Tuple		   =							True
						__magic_name__					:							int		   =							[]
						while i < len(lowerCamelCase						):
											__magic_name__					:							List[str]		   =							chars[i]
											if _is_punctuation(lowerCamelCase						):
																output.append([char]						)
																__magic_name__					:							Tuple		   =							True
											else:
																if start_new_word:
																					output.append([]						)
																__magic_name__					:							Any		   =							False
																output[-1].append(lowerCamelCase						)
											i += 1

						return ["".join(lowerCamelCase						) for x in output]






	def  lowercase      (	self				,     lowerCamelCase						)      ->   Union[str, Any]:




						"""simple docstring"""






						__magic_name__					:							Union[str, Any]		   =							[]
						for char in text:
											__magic_name__					:							Tuple		   =							ord(lowerCamelCase						)
											if self._is_chinese_char(lowerCamelCase						):
																output.append(''' '''						)
																output.append(lowerCamelCase						)
																output.append(''' '''						)
											else:
																output.append(lowerCamelCase						)
						return "".join(lowerCamelCase						)






	def  lowercase      (	self				,     lowerCamelCase						)      ->   Dict:




						"""simple docstring"""






						if (
						    (cp >= 0X4e00 and cp <= 0X9fff)
						    or (cp >= 0X3400 and cp <= 0X4dbf)  #
						    or (cp >= 0X20000 and cp <= 0X2a6df)  #
						    or (cp >= 0X2a700 and cp <= 0X2b73f)  #
						    or (cp >= 0X2b740 and cp <= 0X2b81f)  #
						    or (cp >= 0X2b820 and cp <= 0X2ceaf)  #
						    or (cp >= 0Xf900 and cp <= 0Xfaff)
						    or (cp >= 0X2f800 and cp <= 0X2fa1f)  #
						):  #
											return True

						return False






	def  lowercase      (	self				,     lowerCamelCase						)      ->   str:




						"""simple docstring"""






						__magic_name__					:							Any		   =							[]
						for char in text:
											__magic_name__					:							List[str]		   =							ord(lowerCamelCase						)
											if cp == 0 or cp == 0Xfffd or _is_control(lowerCamelCase						):
																continue
											if _is_whitespace(lowerCamelCase						):
																output.append(''' '''						)
											else:
																output.append(lowerCamelCase						)
						return "".join(lowerCamelCase						)







def  lowerCAmelCase  (	UpperCAmelCase       )  ->List[Any]:






					"""simple docstring"""






					__magic_name__					:							Optional[Any]		   =							set()
					__magic_name__					:							Optional[int]		   =							word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)       )
										__magic_name__					:							Optional[int]		   =							char
					return pairs







def  lowerCAmelCase  (	UpperCAmelCase       )  ->Union[str, Any]:






					"""simple docstring"""






					__magic_name__					:							Dict		   =							text.replace('''—''',	'''-'''       )
					__magic_name__					:							int		   =							text.replace('''–''',	'''-'''       )
					__magic_name__					:							Optional[int]		   =							text.replace('''―''',	'''-'''       )
					__magic_name__					:							Tuple		   =							text.replace('''…''',	'''...'''       )
					__magic_name__					:							List[str]		   =							text.replace('''´''',	'''\''''       )
					__magic_name__					:							str		   =							re.sub(r'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)''',	r''' \1 ''',	UpperCAmelCase       )
					__magic_name__					:							List[Any]		   =							re.sub(r'''\s*\n\s*''',	''' \n ''',	UpperCAmelCase       )
					__magic_name__					:							List[Any]		   =							re.sub(r'''[^\S\n]+''',	''' ''',	UpperCAmelCase       )
					return text.strip()





class 		A__				(   __SCREAMING_SNAKE_CASE ):
	lowerCamelCase__		: Tuple				    =VOCAB_FILES_NAMES
	lowerCamelCase__		: Optional[int]				    =PRETRAINED_VOCAB_FILES_MAP
	lowerCamelCase__		: List[Any]				    =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	lowerCamelCase__		: List[str]				    =["input_ids", "attention_mask"]






	def __init__(	self				,     lowerCamelCase				,     lowerCamelCase				,     lowerCamelCase="<unk>"				,     **lowerCamelCase						)      ->   Optional[int]:




						"""simple docstring"""






						super().__init__(unk_token=lowerCamelCase				,     **lowerCamelCase						)

						try:
											import ftfy
											from spacy.lang.en import English

											__magic_name__					:							str		   =							English()
											__magic_name__					:							Union[str, Any]		   =							_nlp.tokenizer
											__magic_name__					:							Optional[Any]		   =							ftfy.fix_text
						except ImportError:
											logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''						)
											__magic_name__					:							Any		   =							BasicTokenizer(do_lower_case=lowerCamelCase						)
											__magic_name__					:							Union[str, Any]		   =							None

						with open(lowerCamelCase				,     encoding='''utf-8'''						) as vocab_handle:
											__magic_name__					:							Optional[Any]		   =							json.load(lowerCamelCase						)
						__magic_name__					:							List[str]		   =							{v: k for k, v in self.encoder.items()}
						with open(lowerCamelCase				,     encoding='''utf-8'''						) as merges_handle:
											__magic_name__					:							Tuple		   =							merges_handle.read().split('''\n'''						)[1:-1]
						__magic_name__					:							Tuple		   =							[tuple(merge.split()						) for merge in merges]
						__magic_name__					:							Optional[Any]		   =							dict(zip(lowerCamelCase				,     range(len(lowerCamelCase						)						)						)						)
						__magic_name__					:							Dict		   =							{}






	@property
	def  lowercase      (	self						)      ->   str:




						"""simple docstring"""






						return True






	@property
	def  lowercase      (	self						)      ->   int:




						"""simple docstring"""






						return len(self.encoder						)






	def  lowercase      (	self						)      ->   List[Any]:




						"""simple docstring"""






						return dict(self.encoder				,     **self.added_tokens_encoder						)






	def  lowercase      (	self				,     lowerCamelCase						)      ->   List[Any]:




						"""simple docstring"""






						__magic_name__					:							Optional[int]		   =							tuple(token[:-1]						) + (token[-1] + '''</w>''',)
						if token in self.cache:
											return self.cache[token]
						__magic_name__					:							List[str]		   =							get_pairs(lowerCamelCase						)

						if not pairs:
											return token + "</w>"

						while True:
											__magic_name__					:							Dict		   =							min(lowerCamelCase				,     key=lambda lowerCamelCase						: self.bpe_ranks.get(lowerCamelCase				,     float('''inf'''						)						)						)
											if bigram not in self.bpe_ranks:
																break
											__magic_name__   , __magic_name__					:							Optional[int]		   =							bigram
											__magic_name__					:							Any		   =							[]
											__magic_name__					:							Optional[int]		   =							0
											while i < len(lowerCamelCase						):
																try:
																					__magic_name__					:							List[str]		   =							word.index(lowerCamelCase				,     lowerCamelCase						)
																except ValueError:
																					new_word.extend(word[i:]						)
																					break
																else:
																					new_word.extend(word[i:j]						)
																					__magic_name__					:							int		   =							j

																if word[i] == first and i < len(lowerCamelCase						) - 1 and word[i + 1] == second:
																					new_word.append(first + second						)
																					i += 2
																else:
																					new_word.append(word[i]						)
																					i += 1
											__magic_name__					:							Union[str, Any]		   =							tuple(lowerCamelCase						)
											__magic_name__					:							List[Any]		   =							new_word
											if len(lowerCamelCase						) == 1:
																break
											else:
																__magic_name__					:							Union[str, Any]		   =							get_pairs(lowerCamelCase						)
						__magic_name__					:							str		   =							''' '''.join(lowerCamelCase						)
						if word == "\n  </w>":
											__magic_name__					:							List[Any]		   =							'''\n</w>'''
						__magic_name__					:							List[Any]		   =							word
						return word






	def  lowercase      (	self				,     lowerCamelCase						)      ->   int:




						"""simple docstring"""






						__magic_name__					:							Dict		   =							[]
						if self.fix_text is None:
											# Using BERT's BasicTokenizer
											__magic_name__					:							List[Any]		   =							self.nlp.tokenize(lowerCamelCase						)
											for token in text:
																split_tokens.extend(list(self.bpe(lowerCamelCase						).split(''' '''						)						)						)
						else:
											# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
											__magic_name__					:							Optional[int]		   =							self.nlp(text_standardize(self.fix_text(lowerCamelCase						)						)						)
											for token in text:
																split_tokens.extend(list(self.bpe(token.text.lower()						).split(''' '''						)						)						)
						return split_tokens






	def  lowercase      (	self				,     lowerCamelCase						)      ->   Any:




						"""simple docstring"""






						return self.encoder.get(lowerCamelCase				,     self.encoder.get(self.unk_token						)						)






	def  lowercase      (	self				,     lowerCamelCase						)      ->   int:




						"""simple docstring"""






						return self.decoder.get(lowerCamelCase				,     self.unk_token						)






	def  lowercase      (	self				,     lowerCamelCase						)      ->   Optional[Any]:




						"""simple docstring"""






						__magic_name__					:							Tuple		   =							''''''.join(lowerCamelCase						).replace('''</w>'''				,     ''' '''						).strip()
						return out_string






	def  lowercase      (	self				,     lowerCamelCase				,     lowerCamelCase = None						)      ->   Tuple[str]:




						"""simple docstring"""






						if not os.path.isdir(lowerCamelCase						):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''						)
											return
						__magic_name__					:							int		   =							os.path.join(
						    lowerCamelCase				,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)
						__magic_name__					:							int		   =							os.path.join(
						    lowerCamelCase				,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']						)

						with open(lowerCamelCase				,     '''w'''				,     encoding='''utf-8'''						) as f:
											f.write(json.dumps(self.encoder				,     indent=2				,     sort_keys=lowerCamelCase				,     ensure_ascii=lowerCamelCase						) + '''\n'''						)

						__magic_name__					:							Optional[Any]		   =							0
						with open(lowerCamelCase				,     '''w'''				,     encoding='''utf-8'''						) as writer:
											writer.write('''#version: 0.2\n'''						)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()				,     key=lambda lowerCamelCase						: kv[1]						):
																if index != token_index:
																					logger.warning(
																					    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																					    ''' Please check that the tokenizer is not corrupted!'''						)
																					__magic_name__					:							Any		   =							token_index
																writer.write(''' '''.join(lowerCamelCase						) + '''\n'''						)
																index += 1

						return vocab_file, merge_file









import random
import sys

from . import cryptomath_module as cryptomath

lowercase_		=							(
    r''' !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`'''
    r'''abcdefghijklmnopqrstuvwxyz{|}~'''
)







def  lowerCAmelCase  (	UpperCAmelCase,	UpperCAmelCase,	UpperCAmelCase       )  ->None:






					"""simple docstring"""






					if mode == "encrypt":
										if key_a == 1:
															sys.exit(
															    '''The affine cipher becomes weak when key '''
															    '''A is set to 1. Choose different key'''       )
										if key_b == 0:
															sys.exit(
															    '''The affine cipher becomes weak when key '''
															    '''B is set to 0. Choose different key'''       )
					if key_a < 0 or key_b < 0 or key_b > len(UpperCAmelCase       ) - 1:
										sys.exit(
										    '''Key A must be greater than 0 and key B must '''
										    F'''be between 0 and {len(UpperCAmelCase       ) - 1}.'''       )
					if cryptomath.gcd(UpperCAmelCase,	len(UpperCAmelCase       )       ) != 1:
										sys.exit(
										    F'''Key A {key_a} and the symbol set size {len(UpperCAmelCase       )} '''
										    '''are not relatively prime. Choose a different key.'''       )







def  lowerCAmelCase  (	UpperCAmelCase,	UpperCAmelCase       )  ->str:






					"""simple docstring"""






					__magic_name__   , __magic_name__					:							str		   =							divmod(UpperCAmelCase,	len(UpperCAmelCase       )       )
					check_keys(UpperCAmelCase,	UpperCAmelCase,	'''encrypt'''       )
					__magic_name__					:							Dict		   =							''''''
					for symbol in message:
										if symbol in SYMBOLS:
															__magic_name__					:							Union[str, Any]		   =							SYMBOLS.find(UpperCAmelCase       )
															cipher_text += SYMBOLS[(sym_index * key_a + key_b) % len(UpperCAmelCase       )]
										else:
															cipher_text += symbol
					return cipher_text







def  lowerCAmelCase  (	UpperCAmelCase,	UpperCAmelCase       )  ->str:






					"""simple docstring"""






					__magic_name__   , __magic_name__					:							List[str]		   =							divmod(UpperCAmelCase,	len(UpperCAmelCase       )       )
					check_keys(UpperCAmelCase,	UpperCAmelCase,	'''decrypt'''       )
					__magic_name__					:							Union[str, Any]		   =							''''''
					__magic_name__					:							int		   =							cryptomath.find_mod_inverse(UpperCAmelCase,	len(UpperCAmelCase       )       )
					for symbol in message:
										if symbol in SYMBOLS:
															__magic_name__					:							Optional[Any]		   =							SYMBOLS.find(UpperCAmelCase       )
															plain_text += SYMBOLS[
															    (sym_index - key_b) * mod_inverse_of_key_a % len(UpperCAmelCase       )
															]
										else:
															plain_text += symbol
					return plain_text







def  lowerCAmelCase  (	)  ->int:






					"""simple docstring"""






					while True:
										__magic_name__					:							List[Any]		   =							random.randint(2,	len(UpperCAmelCase       )       )
										__magic_name__					:							int		   =							random.randint(2,	len(UpperCAmelCase       )       )
										if cryptomath.gcd(UpperCAmelCase,	len(UpperCAmelCase       )       ) == 1 and key_b % len(UpperCAmelCase       ) != 0:
															return key_b * len(UpperCAmelCase       ) + key_b







def  lowerCAmelCase  (	)  ->None:






					"""simple docstring"""






					__magic_name__					:							List[Any]		   =							input('''Enter message: '''       ).strip()
					__magic_name__					:							Optional[int]		   =							int(input('''Enter key [2000 - 9000]: '''       ).strip()       )
					__magic_name__					:							List[Any]		   =							input('''Encrypt/Decrypt [E/D]: '''       ).strip().lower()

					if mode.startswith('''e'''       ):
										__magic_name__					:							Any		   =							'''encrypt'''
										__magic_name__					:							Any		   =							encrypt_message(UpperCAmelCase,	UpperCAmelCase       )
					elif mode.startswith('''d'''       ):
										__magic_name__					:							Optional[Any]		   =							'''decrypt'''
										__magic_name__					:							Union[str, Any]		   =							decrypt_message(UpperCAmelCase,	UpperCAmelCase       )
					print(F'''\n{mode.title()}ed text: \n{translated}'''       )


if __name__ == "__main__":
				import doctest

				doctest.testmod()
				# main()



