


'''simple docstring'''






import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__UpperCAmelCase					       =							logging.get_logger(__name__)

__UpperCAmelCase					       =							{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__UpperCAmelCase					       =							{
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

__UpperCAmelCase					       =							{
    '''ctrl''': 256,
}

__UpperCAmelCase					       =							{
    '''Pregnancy''': 168_629,
    '''Christianity''': 7_675,
    '''Explain''': 106_423,
    '''Fitness''': 63_440,
    '''Saving''': 63_163,
    '''Ask''': 27_171,
    '''Ass''': 95_985,
    '''Joke''': 163_509,
    '''Questions''': 45_622,
    '''Thoughts''': 49_605,
    '''Retail''': 52_342,
    '''Feminism''': 164_338,
    '''Writing''': 11_992,
    '''Atheism''': 192_263,
    '''Netflix''': 48_616,
    '''Computing''': 39_639,
    '''Opinion''': 43_213,
    '''Alone''': 44_967,
    '''Funny''': 58_917,
    '''Gaming''': 40_358,
    '''Human''': 4_088,
    '''India''': 1_331,
    '''Joker''': 77_138,
    '''Diet''': 36_206,
    '''Legal''': 11_859,
    '''Norman''': 4_939,
    '''Tip''': 72_689,
    '''Weight''': 52_343,
    '''Movies''': 46_273,
    '''Running''': 23_425,
    '''Science''': 2_090,
    '''Horror''': 37_793,
    '''Confession''': 60_572,
    '''Finance''': 12_250,
    '''Politics''': 16_360,
    '''Scary''': 191_985,
    '''Support''': 12_654,
    '''Technologies''': 32_516,
    '''Teenage''': 66_160,
    '''Event''': 32_769,
    '''Learned''': 67_460,
    '''Notion''': 182_770,
    '''Wikipedia''': 37_583,
    '''Books''': 6_665,
    '''Extract''': 76_050,
    '''Confessions''': 102_701,
    '''Conspiracy''': 75_932,
    '''Links''': 63_674,
    '''Narcissus''': 150_425,
    '''Relationship''': 54_766,
    '''Relationships''': 134_796,
    '''Reviews''': 41_671,
    '''News''': 4_256,
    '''Translation''': 26_820,
    '''multilingual''': 128_406,
}


def       _snake_case     (  A )		->					Tuple:
							lowerCAmelCase__      =			set()
							lowerCAmelCase__      =			word[0]
							for char in word[1:]:
														pairs.add((prev_char, char) )
														lowerCAmelCase__      =			char

							lowerCAmelCase__      =			set(A )
							return pairs






class 	a__       (							a__						):





							'''simple docstring'''






							lowercase__    :     List[Any]					=   VOCAB_FILES_NAMES
							lowercase__    :     Dict					=   PRETRAINED_VOCAB_FILES_MAP
							lowercase__    :     Tuple					=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase__    :     Union[str, Any]					=   CONTROL_CODES







							def __init__(	self      ,					lowerCamelCase_      ,					lowerCamelCase_      ,					lowerCamelCase_="<unk>"      ,					**lowerCamelCase_     ) ->							List[str]:
														super().__init__(unk_token=lowerCamelCase_      ,					**lowerCamelCase_     )

														with open(lowerCamelCase_      ,					encoding='''utf-8'''     ) as vocab_handle:
																					lowerCAmelCase__      =			json.load(lowerCamelCase_     )
														lowerCAmelCase__      =			{v: k for k, v in self.encoder.items()}
														with open(lowerCamelCase_      ,					encoding='''utf-8'''     ) as merges_handle:
																					lowerCAmelCase__      =			merges_handle.read().split('''\n'''     )[1:-1]
														lowerCAmelCase__      =			[tuple(merge.split()     ) for merge in merges]
														lowerCAmelCase__      =			dict(zip(lowerCamelCase_      ,					range(len(lowerCamelCase_     )     )     )     )
														lowerCAmelCase__      =			{}







							@property
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							int:
														return len(self.encoder     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							List[Any]:
														return dict(self.encoder      ,					**self.added_tokens_encoder     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							List[str]:
														if token in self.cache:
																					return self.cache[token]
														lowerCAmelCase__      =			tuple(lowerCamelCase_     )
														lowerCAmelCase__      =			tuple(list(word[:-1]     ) + [word[-1] + '''</w>''']     )
														lowerCAmelCase__      =			get_pairs(lowerCamelCase_     )

														if not pairs:
																					return token

														while True:
																					lowerCAmelCase__      =			min(lowerCamelCase_      ,					key=lambda lowerCamelCase_     : self.bpe_ranks.get(lowerCamelCase_      ,					float('''inf'''     )     )     )
																					if bigram not in self.bpe_ranks:
																												break
																					lowerCAmelCase__     ,      lowerCAmelCase__      =			bigram
																					lowerCAmelCase__      =			[]
																					lowerCAmelCase__      =			0
																					while i < len(lowerCamelCase_     ):
																												try:
																																			lowerCAmelCase__      =			word.index(lowerCamelCase_      ,					lowerCamelCase_     )
																												except ValueError:
																																			new_word.extend(word[i:]     )
																																			break
																												else:
																																			new_word.extend(word[i:j]     )
																																			lowerCAmelCase__      =			j

																												if word[i] == first and i < len(lowerCamelCase_     ) - 1 and word[i + 1] == second:
																																			new_word.append(first + second     )
																																			i += 2
																												else:
																																			new_word.append(word[i]     )
																																			i += 1
																					lowerCAmelCase__      =			tuple(lowerCamelCase_     )
																					lowerCAmelCase__      =			new_word
																					if len(lowerCamelCase_     ) == 1:
																												break
																					else:
																												lowerCAmelCase__      =			get_pairs(lowerCamelCase_     )
														lowerCAmelCase__      =			'''@@ '''.join(lowerCamelCase_     )
														lowerCAmelCase__      =			word[:-4]
														lowerCAmelCase__      =			word
														return word







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							List[Any]:
														lowerCAmelCase__      =			[]

														lowerCAmelCase__      =			re.findall(r'''\S+\n?'''      ,					lowerCamelCase_     )

														for token in words:
																					split_tokens.extend(list(self.bpe(lowerCamelCase_     ).split(''' '''     )     )     )
														return split_tokens







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							str:
														return self.encoder.get(lowerCamelCase_      ,					self.encoder.get(self.unk_token     )     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							int:
														return self.decoder.get(lowerCamelCase_      ,					self.unk_token     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_     ) ->							Any:
														lowerCAmelCase__      =			''' '''.join(lowerCamelCase_     ).replace('''@@ '''      ,					''''''     ).strip()
														return out_string







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_ = None     ) ->							Tuple[str]:
														if not os.path.isdir(lowerCamelCase_     ):
																					logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""     )
																					return
														lowerCAmelCase__      =			os.path.join(
														    lowerCamelCase_      ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )
														lowerCAmelCase__      =			os.path.join(
														    lowerCamelCase_      ,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']     )

														with open(lowerCamelCase_      ,					'''w'''      ,					encoding='''utf-8'''     ) as f:
																					f.write(json.dumps(self.encoder      ,					indent=2      ,					sort_keys=lowerCamelCase_      ,					ensure_ascii=lowerCamelCase_     ) + '''\n'''     )

														lowerCAmelCase__      =			0
														with open(lowerCamelCase_      ,					'''w'''      ,					encoding='''utf-8'''     ) as writer:
																					writer.write('''#version: 0.2\n'''     )
																					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,					key=lambda lowerCamelCase_     : kv[1]     ):
																												if index != token_index:
																																			logger.warning(
																																			    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																																			    ''' Please check that the tokenizer is not corrupted!'''     )
																																			lowerCAmelCase__      =			token_index
																												writer.write(''' '''.join(lowerCamelCase_     ) + '''\n'''     )
																												index += 1

														return vocab_file, merge_file

							# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
							#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
							#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
							#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
							#     return ''.join(tokens_generated_so_far)



'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase					       =							logging.get_logger(__name__)

__UpperCAmelCase					       =							{'''ctrl''': '''https://huggingface.co/ctrl/resolve/main/config.json'''}




class 	a__       (							a__						):





							'''simple docstring'''






							lowercase__    :     Dict					=   "ctrl"
							lowercase__    :     str					=   ["past_key_values"]
							lowercase__    :     Optional[int]					=   {
							    "max_position_embeddings": "n_positions",
							    "hidden_size": "n_embd",
							    "num_attention_heads": "n_head",
							    "num_hidden_layers": "n_layer",
							}







							def __init__(	self      ,					lowerCamelCase_=24_65_34      ,					lowerCamelCase_=2_56      ,					lowerCamelCase_=12_80      ,					lowerCamelCase_=81_92      ,					lowerCamelCase_=48      ,					lowerCamelCase_=16      ,					lowerCamelCase_=0.1      ,					lowerCamelCase_=0.1      ,					lowerCamelCase_=1e-6      ,					lowerCamelCase_=0.02      ,					lowerCamelCase_=True      ,					**lowerCamelCase_      ,					) ->							Union[str, Any]:
														lowerCAmelCase__      =			vocab_size
														lowerCAmelCase__      =			n_positions
														lowerCAmelCase__      =			n_embd
														lowerCAmelCase__      =			n_layer
														lowerCAmelCase__      =			n_head
														lowerCAmelCase__      =			dff
														lowerCAmelCase__      =			resid_pdrop
														lowerCAmelCase__      =			embd_pdrop
														lowerCAmelCase__      =			layer_norm_epsilon
														lowerCAmelCase__      =			initializer_range

														lowerCAmelCase__      =			use_cache

														super().__init__(**lowerCamelCase_     )