





"""simple docstring"""





import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase__     			=			logging.get_logger(__name__)

lowerCAmelCase__     			=			{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

lowerCAmelCase__     			=			{
    'vocab_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'},
    'merges_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'},
}

lowerCAmelCase__     			=			{
    'ctrl': 256,
}

lowerCAmelCase__     			=			{
    'Pregnancy': 168_629,
    'Christianity': 7_675,
    'Explain': 106_423,
    'Fitness': 63_440,
    'Saving': 63_163,
    'Ask': 27_171,
    'Ass': 95_985,
    'Joke': 163_509,
    'Questions': 45_622,
    'Thoughts': 49_605,
    'Retail': 52_342,
    'Feminism': 164_338,
    'Writing': 11_992,
    'Atheism': 192_263,
    'Netflix': 48_616,
    'Computing': 39_639,
    'Opinion': 43_213,
    'Alone': 44_967,
    'Funny': 58_917,
    'Gaming': 40_358,
    'Human': 4_088,
    'India': 1_331,
    'Joker': 77_138,
    'Diet': 36_206,
    'Legal': 11_859,
    'Norman': 4_939,
    'Tip': 72_689,
    'Weight': 52_343,
    'Movies': 46_273,
    'Running': 23_425,
    'Science': 2_090,
    'Horror': 37_793,
    'Confession': 60_572,
    'Finance': 12_250,
    'Politics': 16_360,
    'Scary': 191_985,
    'Support': 12_654,
    'Technologies': 32_516,
    'Teenage': 66_160,
    'Event': 32_769,
    'Learned': 67_460,
    'Notion': 182_770,
    'Wikipedia': 37_583,
    'Books': 6_665,
    'Extract': 76_050,
    'Confessions': 102_701,
    'Conspiracy': 75_932,
    'Links': 63_674,
    'Narcissus': 150_425,
    'Relationship': 54_766,
    'Relationships': 134_796,
    'Reviews': 41_671,
    'News': 4_256,
    'Translation': 26_820,
    'multilingual': 128_406,
}


def       a__					(					_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		UpperCamelCase   =	set()
		UpperCamelCase   =	word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)    )
				UpperCamelCase   =	char

		UpperCamelCase   =	set(A_    )
		return pairs






class        _lowerCamelCase      (       _A ):
						UpperCAmelCase_ =			VOCAB_FILES_NAMES
						UpperCAmelCase_ =			PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase_ =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase_ =			CONTROL_CODES


						def __init__(self   ,							__a   ,							__a   ,							__a="<unk>"   ,							**__a		) ->   List[Any]:
								super().__init__(unk_token=UpperCamelCase__   ,							**UpperCamelCase__		)

								with open(UpperCamelCase__   ,							encoding="utf-8"		) as vocab_handle:
										UpperCamelCase   =	json.load(UpperCamelCase__		)
								UpperCamelCase   =	{v: k for k, v in self.encoder.items()}
								with open(UpperCamelCase__   ,							encoding="utf-8"		) as merges_handle:
										UpperCamelCase   =	merges_handle.read().split("\n"		)[1:-1]
								UpperCamelCase   =	[tuple(merge.split()		) for merge in merges]
								UpperCamelCase   =	dict(zip(UpperCamelCase__   ,							range(len(UpperCamelCase__		)		)		)		)
								UpperCamelCase   =	{}


						@property
						def     snake_case_   (self		) ->   int:
								return len(self.encoder		)


						def     snake_case_   (self		) ->   Optional[int]:
								return dict(self.encoder   ,							**self.added_tokens_encoder		)


						def     snake_case_   (self   ,							__a		) ->   Any:






								if token in self.cache:
										return self.cache[token]
								UpperCamelCase   =	tuple(UpperCamelCase__		)
								UpperCamelCase   =	tuple(list(word[:-1]		) + [word[-1] + "</w>"]		)
								UpperCamelCase   =	get_pairs(UpperCamelCase__		)

								if not pairs:
										return token

								while True:
										UpperCamelCase   =	min(UpperCamelCase__   ,							key=lambda __a		: self.bpe_ranks.get(UpperCamelCase__   ,							float("inf"		)		)		)
										if bigram not in self.bpe_ranks:
												break
										UpperCamelCase   ,				UpperCamelCase   =	bigram
										UpperCamelCase   =	[]
										UpperCamelCase   =	0
										while i < len(UpperCamelCase__		):
												try:
														UpperCamelCase   =	word.index(UpperCamelCase__   ,							UpperCamelCase__		)
												except ValueError:
														new_word.extend(word[i:]		)
														break
												else:
														new_word.extend(word[i:j]		)
														UpperCamelCase   =	j

												if word[i] == first and i < len(UpperCamelCase__		) - 1 and word[i + 1] == second:
														new_word.append(first + second		)
														i += 2
												else:
														new_word.append(word[i]		)
														i += 1
										UpperCamelCase   =	tuple(UpperCamelCase__		)
										UpperCamelCase   =	new_word
										if len(UpperCamelCase__		) == 1:
												break
										else:
												UpperCamelCase   =	get_pairs(UpperCamelCase__		)
								UpperCamelCase   =	"@@ ".join(UpperCamelCase__		)
								UpperCamelCase   =	word[:-4]
								UpperCamelCase   =	word
								return word


						def     snake_case_   (self   ,							__a		) ->   Any:
								UpperCamelCase   =	[]

								UpperCamelCase   =	re.findall(r"\S+\n?"   ,							UpperCamelCase__		)

								for token in words:
										split_tokens.extend(list(self.bpe(UpperCamelCase__		).split(" "		)		)		)
								return split_tokens


						def     snake_case_   (self   ,							__a		) ->   Dict:
								return self.encoder.get(UpperCamelCase__   ,							self.encoder.get(self.unk_token		)		)


						def     snake_case_   (self   ,							__a		) ->   Optional[Any]:
								return self.decoder.get(UpperCamelCase__   ,							self.unk_token		)


						def     snake_case_   (self   ,							__a		) ->   Dict:
								UpperCamelCase   =	" ".join(UpperCamelCase__		).replace("@@ "   ,							""		).strip()
								return out_string




						def     snake_case_   (self   ,							__a   ,							__a = None		) ->   Tuple[str]:






								if not os.path.isdir(UpperCamelCase__		):
										logger.error(F"Vocabulary path ({save_directory}) should be a directory"		)
										return
								UpperCamelCase   =	os.path.join(
								    UpperCamelCase__   ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
								UpperCamelCase   =	os.path.join(
								    UpperCamelCase__   ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]		)

								with open(UpperCamelCase__   ,							"w"   ,							encoding="utf-8"		) as f:
										f.write(json.dumps(self.encoder   ,							indent=2   ,							sort_keys=UpperCamelCase__   ,							ensure_ascii=UpperCamelCase__		) + "\n"		)

								UpperCamelCase   =	0
								with open(UpperCamelCase__   ,							"w"   ,							encoding="utf-8"		) as writer:
										writer.write("#version: 0.2\n"		)
										for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,							key=lambda __a		: kv[1]		):
												if index != token_index:
														logger.warning(
														    F"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
														    " Please check that the tokenizer is not corrupted!"		)
														UpperCamelCase   =	token_index
												writer.write(" ".join(UpperCamelCase__		) + "\n"		)
												index += 1

								return vocab_file, merge_file

						# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
						#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
						#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
						#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
						#     return ''.join(tokens_generated_so_far)













"""simple docstring"""





from typing import List

from .keymap import KEYMAP, get_character


def       a__					(					_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		def decorator(_SCREAMING_SNAKE_CASE    ):
				UpperCamelCase   =	getattr(_SCREAMING_SNAKE_CASE  ,	"handle_key"  ,	[]    )
				handle += [key]
				setattr(_SCREAMING_SNAKE_CASE  ,	"handle_key"  ,	_SCREAMING_SNAKE_CASE    )
				return func

		return decorator


def       a__					(					*_SCREAMING_SNAKE_CASE    ):




		"""simple docstring"""


		def decorator(_SCREAMING_SNAKE_CASE    ):
				UpperCamelCase   =	getattr(_SCREAMING_SNAKE_CASE  ,	"handle_key"  ,	[]    )
				handle += keys
				setattr(_SCREAMING_SNAKE_CASE  ,	"handle_key"  ,	_SCREAMING_SNAKE_CASE    )
				return func

		return decorator





class        _lowerCamelCase      (       _lowercase ):


						def __new__(cls   ,							__a   ,							__a   ,							__a		) ->   Any:
								UpperCamelCase   =	super().__new__(cls   ,							__a   ,							__a   ,							__a		)
								if not hasattr(__a   ,							"key_handler"		):
										setattr(__a   ,							"key_handler"   ,							{}		)
								setattr(__a   ,							"handle_input"   ,							KeyHandler.handle_input		)

								for value in attrs.values():
										UpperCamelCase   =	getattr(__a   ,							"handle_key"   ,							[]		)
										for key in handled_keys:
												UpperCamelCase   =	value
								return new_cls




						@staticmethod
						def     snake_case_   (cls		) ->   Optional[Any]:
								UpperCamelCase   =	get_character()
								if char != KEYMAP["undefined"]:
										UpperCamelCase   =	ord(__a		)
								UpperCamelCase   =	cls.key_handler.get(__a		)
								if handler:
										UpperCamelCase   =	char
										return handler(cls		)
								else:
										return None



def       a__					(					cls    ):




		"""simple docstring"""


		return KeyHandler(cls.__name__  ,	cls.__bases__  ,	cls.__dict__.copy()    )






