


'''simple docstring'''



import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__A		=logging.get_logger(__name__)

__A		={
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__A		={
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

__A		={
    '''ctrl''': 2_56,
}

__A		={
    '''Pregnancy''': 16_86_29,
    '''Christianity''': 76_75,
    '''Explain''': 10_64_23,
    '''Fitness''': 6_34_40,
    '''Saving''': 6_31_63,
    '''Ask''': 2_71_71,
    '''Ass''': 9_59_85,
    '''Joke''': 16_35_09,
    '''Questions''': 4_56_22,
    '''Thoughts''': 4_96_05,
    '''Retail''': 5_23_42,
    '''Feminism''': 16_43_38,
    '''Writing''': 1_19_92,
    '''Atheism''': 19_22_63,
    '''Netflix''': 4_86_16,
    '''Computing''': 3_96_39,
    '''Opinion''': 4_32_13,
    '''Alone''': 4_49_67,
    '''Funny''': 5_89_17,
    '''Gaming''': 4_03_58,
    '''Human''': 40_88,
    '''India''': 13_31,
    '''Joker''': 7_71_38,
    '''Diet''': 3_62_06,
    '''Legal''': 1_18_59,
    '''Norman''': 49_39,
    '''Tip''': 7_26_89,
    '''Weight''': 5_23_43,
    '''Movies''': 4_62_73,
    '''Running''': 2_34_25,
    '''Science''': 20_90,
    '''Horror''': 3_77_93,
    '''Confession''': 6_05_72,
    '''Finance''': 1_22_50,
    '''Politics''': 1_63_60,
    '''Scary''': 19_19_85,
    '''Support''': 1_26_54,
    '''Technologies''': 3_25_16,
    '''Teenage''': 6_61_60,
    '''Event''': 3_27_69,
    '''Learned''': 6_74_60,
    '''Notion''': 18_27_70,
    '''Wikipedia''': 3_75_83,
    '''Books''': 66_65,
    '''Extract''': 7_60_50,
    '''Confessions''': 10_27_01,
    '''Conspiracy''': 7_59_32,
    '''Links''': 6_36_74,
    '''Narcissus''': 15_04_25,
    '''Relationship''': 5_47_66,
    '''Relationships''': 13_47_96,
    '''Reviews''': 4_16_71,
    '''News''': 42_56,
    '''Translation''': 2_68_20,
    '''multilingual''': 12_84_06,
}







def       _UpperCamelCase (						UpperCamelCase__   ):
			UpperCAmelCase__					:   Optional[Any]       				=       set()
			UpperCAmelCase__					:   List[Any]       				=       word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)   )
						UpperCAmelCase__					:   Optional[int]       				=       char

			UpperCAmelCase__					:   int       				=       set(__lowerCAmelCase   )
			return pairs


class 						_snake_case   (       lowerCamelCase_		):
					lowerCAmelCase :Optional[Any]						=   VOCAB_FILES_NAMES
					lowerCAmelCase :str						=   PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase :str						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase :int						=   CONTROL_CODES




					def __init__( self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase="<unk>"							,							**_lowerCamelCase):
								super().__init__(unk_token=__snake_case							,							**__snake_case)

								with open(__snake_case							,							encoding="""utf-8""") as vocab_handle:
											UpperCAmelCase__					:   int       				=       json.load(__snake_case)
								UpperCAmelCase__					:   List[Any]       				=       {v: k for k, v in self.encoder.items()}
								with open(__snake_case							,							encoding="""utf-8""") as merges_handle:
											UpperCAmelCase__					:   List[Any]       				=       merges_handle.read().split("""\n""")[1:-1]
								UpperCAmelCase__					:   Optional[Any]       				=       [tuple(merge.split()) for merge in merges]
								UpperCAmelCase__					:   str       				=       dict(zip(__snake_case							,							range(len(__snake_case))))
								UpperCAmelCase__					:   Tuple       				=       {}




					@property
					def 			snake_case__  ( self):
								return len(self.encoder)




					def 			snake_case__  ( self):
								return dict(self.encoder							,							**self.added_tokens_encoder)




					def 			snake_case__  ( self							,							_lowerCamelCase):
								if token in self.cache:
											return self.cache[token]
								UpperCAmelCase__					:   Union[str, Any]       				=       tuple(__snake_case)
								UpperCAmelCase__					:   str       				=       tuple(list(word[:-1]) + [word[-1] + """</w>"""])
								UpperCAmelCase__					:   str       				=       get_pairs(__snake_case)

								if not pairs:
											return token

								while True:
											UpperCAmelCase__					:   Union[str, Any]       				=       min(__snake_case							,							key=lambda _lowerCamelCase: self.bpe_ranks.get(__snake_case							,							float("""inf""")))
											if bigram not in self.bpe_ranks:
														break
											UpperCAmelCase__				,    UpperCAmelCase__					:   Optional[Any]       				=       bigram
											UpperCAmelCase__					:   str       				=       []
											UpperCAmelCase__					:   List[str]       				=       0
											while i < len(__snake_case):
														try:
																	UpperCAmelCase__					:   List[Any]       				=       word.index(__snake_case							,							__snake_case)
														except ValueError:
																	new_word.extend(word[i:])
																	break
														else:
																	new_word.extend(word[i:j])
																	UpperCAmelCase__					:   int       				=       j

														if word[i] == first and i < len(__snake_case) - 1 and word[i + 1] == second:
																	new_word.append(first + second)
																	i += 2
														else:
																	new_word.append(word[i])
																	i += 1
											UpperCAmelCase__					:   str       				=       tuple(__snake_case)
											UpperCAmelCase__					:   List[Any]       				=       new_word
											if len(__snake_case) == 1:
														break
											else:
														UpperCAmelCase__					:   Optional[int]       				=       get_pairs(__snake_case)
								UpperCAmelCase__					:   Union[str, Any]       				=       """@@ """.join(__snake_case)
								UpperCAmelCase__					:   Tuple       				=       word[:-4]
								UpperCAmelCase__					:   Union[str, Any]       				=       word
								return word




					def 			snake_case__  ( self							,							_lowerCamelCase):
								UpperCAmelCase__					:   Optional[int]       				=       []

								UpperCAmelCase__					:   Optional[Any]       				=       re.findall(r"""\S+\n?"""							,							__snake_case)

								for token in words:
											split_tokens.extend(list(self.bpe(__snake_case).split(""" """)))
								return split_tokens




					def 			snake_case__  ( self							,							_lowerCamelCase):
								return self.encoder.get(__snake_case							,							self.encoder.get(self.unk_token))




					def 			snake_case__  ( self							,							_lowerCamelCase):
								return self.decoder.get(__snake_case							,							self.unk_token)




					def 			snake_case__  ( self							,							_lowerCamelCase):
								UpperCAmelCase__					:   str       				=       """ """.join(__snake_case).replace("""@@ """							,							"""""").strip()
								return out_string




					def 			snake_case__  ( self							,							_lowerCamelCase							,							_lowerCamelCase = None):
								if not os.path.isdir(__snake_case):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
											return
								UpperCAmelCase__					:   Optional[int]       				=       os.path.join(
								    __snake_case							,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])
								UpperCAmelCase__					:   int       				=       os.path.join(
								    __snake_case							,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""])

								with open(__snake_case							,							"""w"""							,							encoding="""utf-8""") as f:
											f.write(json.dumps(self.encoder							,							indent=2							,							sort_keys=__snake_case							,							ensure_ascii=__snake_case) + """\n""")

								UpperCAmelCase__					:   Any       				=       0
								with open(__snake_case							,							"""w"""							,							encoding="""utf-8""") as writer:
											writer.write("""#version: 0.2\n""")
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							,							key=lambda _lowerCamelCase: kv[1]):
														if index != token_index:
																	logger.warning(
																	    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																	    """ Please check that the tokenizer is not corrupted!""")
																	UpperCAmelCase__					:   List[Any]       				=       token_index
														writer.write(""" """.join(__snake_case) + """\n""")
														index += 1

								return vocab_file, merge_file

					# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
					#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
					#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
					#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
					#     return ''.join(tokens_generated_so_far)



'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


__A		={
    'configuration_table_transformer': [
        'TABLE_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'TableTransformerConfig',
        'TableTransformerOnnxConfig',
    ]
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__A		=[
		    'TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'TableTransformerForObjectDetection',
		    'TableTransformerModel',
		    'TableTransformerPreTrainedModel',
		]


if TYPE_CHECKING:
		from .configuration_table_transformer import (
		    TABLE_TRANSFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
		    TableTransformerConfig,
		    TableTransformerOnnxConfig,
		)

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_table_transformer import (
				    TABLE_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    TableTransformerForObjectDetection,
				    TableTransformerModel,
				    TableTransformerPreTrainedModel,
				)

else:
		import sys

		__A		=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)