






import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__   :      List[str]				=		logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__   :      str				=		{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

SCREAMING_SNAKE_CASE__   :      Union[str, Any]				=		{
    """vocab_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"""},
    """merges_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"""},
}

SCREAMING_SNAKE_CASE__   :      int				=		{
    """ctrl""": 2_56,
}

SCREAMING_SNAKE_CASE__   :      Tuple				=		{
    """Pregnancy""": 16_86_29,
    """Christianity""": 76_75,
    """Explain""": 10_64_23,
    """Fitness""": 6_34_40,
    """Saving""": 6_31_63,
    """Ask""": 2_71_71,
    """Ass""": 9_59_85,
    """Joke""": 16_35_09,
    """Questions""": 4_56_22,
    """Thoughts""": 4_96_05,
    """Retail""": 5_23_42,
    """Feminism""": 16_43_38,
    """Writing""": 1_19_92,
    """Atheism""": 19_22_63,
    """Netflix""": 4_86_16,
    """Computing""": 3_96_39,
    """Opinion""": 4_32_13,
    """Alone""": 4_49_67,
    """Funny""": 5_89_17,
    """Gaming""": 4_03_58,
    """Human""": 40_88,
    """India""": 13_31,
    """Joker""": 7_71_38,
    """Diet""": 3_62_06,
    """Legal""": 1_18_59,
    """Norman""": 49_39,
    """Tip""": 7_26_89,
    """Weight""": 5_23_43,
    """Movies""": 4_62_73,
    """Running""": 2_34_25,
    """Science""": 20_90,
    """Horror""": 3_77_93,
    """Confession""": 6_05_72,
    """Finance""": 1_22_50,
    """Politics""": 1_63_60,
    """Scary""": 19_19_85,
    """Support""": 1_26_54,
    """Technologies""": 3_25_16,
    """Teenage""": 6_61_60,
    """Event""": 3_27_69,
    """Learned""": 6_74_60,
    """Notion""": 18_27_70,
    """Wikipedia""": 3_75_83,
    """Books""": 66_65,
    """Extract""": 7_60_50,
    """Confessions""": 10_27_01,
    """Conspiracy""": 7_59_32,
    """Links""": 6_36_74,
    """Narcissus""": 15_04_25,
    """Relationship""": 5_47_66,
    """Relationships""": 13_47_96,
    """Reviews""": 4_16_71,
    """News""": 42_56,
    """Translation""": 2_68_20,
    """multilingual""": 12_84_06,
}

def   __lowercase       (			snake_case			):







				"""simple docstring"""






				__magic_name__		:Union[str, Any]   =   set()
				__magic_name__		:Optional[Any]   =   word[0]
				for char in word[1:]:
								pairs.add((prev_char, char)			)
								__magic_name__		:int   =   char

				__magic_name__		:List[str]   =   set(snake_case			)
				return pairs

class     lowerCamelCase_					(   lowerCamelCase			):
				a__       							=	VOCAB_FILES_NAMES
				a__       							=	PRETRAINED_VOCAB_FILES_MAP
				a__       							=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				a__       							=	CONTROL_CODES







				def __init__(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase="<unk>"							,       **__lowerCAmelCase  ):
								"""simple docstring"""
								super().__init__(unk_token=__lowerCAmelCase							,       **__lowerCAmelCase  )

								with open(__lowerCAmelCase							,       encoding='''utf-8'''  ) as vocab_handle:
												__magic_name__		:List[Any]   =   json.load(__lowerCAmelCase  )
								__magic_name__		:Union[str, Any]   =   {v: k for k, v in self.encoder.items()}
								with open(__lowerCAmelCase							,       encoding='''utf-8'''  ) as merges_handle:
												__magic_name__		:Any   =   merges_handle.read().split('''\n'''  )[1:-1]
								__magic_name__		:Tuple   =   [tuple(merge.split()  ) for merge in merges]
								__magic_name__		:Optional[int]   =   dict(zip(__lowerCAmelCase							,       range(len(__lowerCAmelCase  )  )  )  )
								__magic_name__		:List[Any]   =   {}







				@property
				def       A						(    self  ):
								"""simple docstring"""
								return len(self.encoder  )







				def       A						(    self  ):
								"""simple docstring"""
								return dict(self.encoder							,       **self.added_tokens_encoder  )







				def       A						(    self							,       __lowerCAmelCase  ):
								"""simple docstring"""
								if token in self.cache:
												return self.cache[token]
								__magic_name__		:str   =   tuple(__lowerCAmelCase  )
								__magic_name__		:List[Any]   =   tuple(list(word[:-1]  ) + [word[-1] + '''</w>''']  )
								__magic_name__		:Optional[int]   =   get_pairs(__lowerCAmelCase  )

								if not pairs:
												return token

								while True:
												__magic_name__		:List[Any]   =   min(__lowerCAmelCase							,       key=lambda __lowerCAmelCase  : self.bpe_ranks.get(__lowerCAmelCase							,       float('''inf'''  )  )  )
												if bigram not in self.bpe_ranks:
																break
												__magic_name__  ,					__magic_name__		:List[Any]   =   bigram
												__magic_name__		:Tuple   =   []
												__magic_name__		:Any   =   0
												while i < len(__lowerCAmelCase  ):
																try:
																				__magic_name__		:Dict   =   word.index(__lowerCAmelCase							,       __lowerCAmelCase  )
																except ValueError:
																				new_word.extend(word[i:]  )
																				break
																else:
																				new_word.extend(word[i:j]  )
																				__magic_name__		:Optional[Any]   =   j

																if word[i] == first and i < len(__lowerCAmelCase  ) - 1 and word[i + 1] == second:
																				new_word.append(first + second  )
																				i += 2
																else:
																				new_word.append(word[i]  )
																				i += 1
												__magic_name__		:Dict   =   tuple(__lowerCAmelCase  )
												__magic_name__		:Union[str, Any]   =   new_word
												if len(__lowerCAmelCase  ) == 1:
																break
												else:
																__magic_name__		:str   =   get_pairs(__lowerCAmelCase  )
								__magic_name__		:List[Any]   =   '''@@ '''.join(__lowerCAmelCase  )
								__magic_name__		:Union[str, Any]   =   word[:-4]
								__magic_name__		:Dict   =   word
								return word







				def       A						(    self							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:Any   =   []

								__magic_name__		:List[str]   =   re.findall(R'''\S+\n?'''							,       __lowerCAmelCase  )

								for token in words:
												split_tokens.extend(list(self.bpe(__lowerCAmelCase  ).split(''' '''  )  )  )
								return split_tokens







				def       A						(    self							,       __lowerCAmelCase  ):
								"""simple docstring"""
								return self.encoder.get(__lowerCAmelCase							,       self.encoder.get(self.unk_token  )  )







				def       A						(    self							,       __lowerCAmelCase  ):
								"""simple docstring"""
								return self.decoder.get(__lowerCAmelCase							,       self.unk_token  )







				def       A						(    self							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   ''' '''.join(__lowerCAmelCase  ).replace('''@@ '''							,       ''''''  ).strip()
								return out_string







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase = None  ):
								"""simple docstring"""
								if not os.path.isdir(__lowerCAmelCase  ):
												logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
												return
								__magic_name__		:int   =   os.path.join(
								    __lowerCAmelCase							,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )
								__magic_name__		:Dict   =   os.path.join(
								    __lowerCAmelCase							,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']  )

								with open(__lowerCAmelCase							,       '''w'''							,       encoding='''utf-8'''  ) as f:
												f.write(json.dumps(self.encoder							,       indent=2							,       sort_keys=__lowerCAmelCase							,       ensure_ascii=__lowerCAmelCase  ) + '''\n'''  )

								__magic_name__		:int   =   0
								with open(__lowerCAmelCase							,       '''w'''							,       encoding='''utf-8'''  ) as writer:
												writer.write('''#version: 0.2\n'''  )
												for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,       key=lambda __lowerCAmelCase  : kv[1]  ):
																if index != token_index:
																				logger.warning(
																				    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																				    ''' Please check that the tokenizer is not corrupted!'''  )
																				__magic_name__		:Optional[Any]   =   token_index
																writer.write(''' '''.join(__lowerCAmelCase  ) + '''\n'''  )
																index += 1

								return vocab_file, merge_file

				# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
				#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
				#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
				#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
				#     return ''.join(tokens_generated_so_far)








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


UpperCamelCase_						:     int	       =							{
    """configuration_transfo_xl""": ["""TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP""", """TransfoXLConfig"""],
    """tokenization_transfo_xl""": ["""TransfoXLCorpus""", """TransfoXLTokenizer"""],
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCamelCase_						:     Any	       =							[
							    """TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """AdaptiveEmbedding""",
							    """TransfoXLForSequenceClassification""",
							    """TransfoXLLMHeadModel""",
							    """TransfoXLModel""",
							    """TransfoXLPreTrainedModel""",
							    """load_tf_weights_in_transfo_xl""",
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							UpperCamelCase_						:     Tuple	       =							[
							    """TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST""",
							    """TFAdaptiveEmbedding""",
							    """TFTransfoXLForSequenceClassification""",
							    """TFTransfoXLLMHeadModel""",
							    """TFTransfoXLMainLayer""",
							    """TFTransfoXLModel""",
							    """TFTransfoXLPreTrainedModel""",
							]


if TYPE_CHECKING:
							from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, TransfoXLConfig
							from .tokenization_transfo_xl import TransfoXLCorpus, TransfoXLTokenizer

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_transfo_xl import (
														    TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
														    AdaptiveEmbedding,
														    TransfoXLForSequenceClassification,
														    TransfoXLLMHeadModel,
														    TransfoXLModel,
														    TransfoXLPreTrainedModel,
														    load_tf_weights_in_transfo_xl,
														)

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_transfo_xl import (
														    TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFAdaptiveEmbedding,
														    TFTransfoXLForSequenceClassification,
														    TFTransfoXLLMHeadModel,
														    TFTransfoXLMainLayer,
														    TFTransfoXLModel,
														    TFTransfoXLPreTrainedModel,
														)

else:
							import sys

							UpperCamelCase_						:     Optional[int]	       =							_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




