


'''simple docstring'''






import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__A	:	Optional[int]     =     logging.get_logger(__name__)

__A	:	List[str]     =     {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__A	:	Dict     =     {
    'vocab_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'},
    'merges_file': {'ctrl': 'https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'},
}

__A	:	Tuple     =     {
    'ctrl': 256,
}

__A	:	Any     =     {
    'Pregnancy': 168_629,
    'Christianity': 7_675,
    'Explain': 106_423,
    'Fitness': 63_440,
    'Saving': 63_163,
    'Ask': 27_171,
    'Ass': 95_985,
    'Joke': 163_509,
    'Questions': 45_622,
    'Thoughts': 49_605,
    'Retail': 52_342,
    'Feminism': 164_338,
    'Writing': 11_992,
    'Atheism': 192_263,
    'Netflix': 48_616,
    'Computing': 39_639,
    'Opinion': 43_213,
    'Alone': 44_967,
    'Funny': 58_917,
    'Gaming': 40_358,
    'Human': 4_088,
    'India': 1_331,
    'Joker': 77_138,
    'Diet': 36_206,
    'Legal': 11_859,
    'Norman': 4_939,
    'Tip': 72_689,
    'Weight': 52_343,
    'Movies': 46_273,
    'Running': 23_425,
    'Science': 2_090,
    'Horror': 37_793,
    'Confession': 60_572,
    'Finance': 12_250,
    'Politics': 16_360,
    'Scary': 191_985,
    'Support': 12_654,
    'Technologies': 32_516,
    'Teenage': 66_160,
    'Event': 32_769,
    'Learned': 67_460,
    'Notion': 182_770,
    'Wikipedia': 37_583,
    'Books': 6_665,
    'Extract': 76_050,
    'Confessions': 102_701,
    'Conspiracy': 75_932,
    'Links': 63_674,
    'Narcissus': 150_425,
    'Relationship': 54_766,
    'Relationships': 134_796,
    'Reviews': 41_671,
    'News': 4_256,
    'Translation': 26_820,
    'multilingual': 128_406,
}





def      UpperCAmelCase			(		lowerCamelCase_  :Dict ):







		'''simple docstring'''
		snake_case_		:							int			       =						set()
		snake_case_		:							List[Any]			       =						word[0]
		for char in word[1:]:
				pairs.add((prev_char, char) )
				snake_case_		:							Union[str, Any]			       =						char

		snake_case_		:							List[Any]			       =						set(A_ )
		return pairs






class 						__UpperCamelCase	(						_A							):
							lowercase			:		List[Any]   		=  VOCAB_FILES_NAMES
							lowercase			:		Union[str, Any]   		=  PRETRAINED_VOCAB_FILES_MAP
							lowercase			:		Tuple   		=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase			:		Optional[int]   		=  CONTROL_CODES





							def __init__(  self		:int  ,_UpperCamelCase		:Union[str, Any]  ,_UpperCamelCase		:Union[str, Any]  ,_UpperCamelCase		:Any="<unk>"  ,**_UpperCamelCase		:Dict			):
									super().__init__(unk_token=UpperCamelCase__  ,**UpperCamelCase__			)

									with open(UpperCamelCase__  ,encoding="""utf-8"""			) as vocab_handle:
											snake_case_		:							Optional[Any]			       =						json.load(UpperCamelCase__			)
									snake_case_		:							Optional[Any]			       =						{v: k for k, v in self.encoder.items()}
									with open(UpperCamelCase__  ,encoding="""utf-8"""			) as merges_handle:
											snake_case_		:							Any			       =						merges_handle.read().split("""\n"""			)[1:-1]
									snake_case_		:							Optional[Any]			       =						[tuple(merge.split()			) for merge in merges]
									snake_case_		:							Optional[int]			       =						dict(zip(UpperCamelCase__  ,range(len(UpperCamelCase__			)			)			)			)
									snake_case_		:							Tuple			       =						{}





							@property
							def 		a__   (  self		:Optional[int]			):
									return len(self.encoder			)





							def 		a__   (  self		:Tuple			):
									return dict(self.encoder  ,**self.added_tokens_encoder			)





							def 		a__   (  self		:str  ,_UpperCamelCase		:List[Any]			):






									if token in self.cache:
											return self.cache[token]
									snake_case_		:							Dict			       =						tuple(UpperCamelCase__			)
									snake_case_		:							List[Any]			       =						tuple(list(word[:-1]			) + [word[-1] + """</w>"""]			)
									snake_case_		:							List[Any]			       =						get_pairs(UpperCamelCase__			)

									if not pairs:
											return token

									while True:
											snake_case_		:							Any			       =						min(UpperCamelCase__  ,key=lambda _UpperCamelCase			: self.bpe_ranks.get(UpperCamelCase__  ,float("""inf"""			)			)			)
											if bigram not in self.bpe_ranks:
													break
											snake_case_       ,      snake_case_		:							Dict			       =						bigram
											snake_case_		:							Optional[Any]			       =						[]
											snake_case_		:							int			       =						0
											while i < len(UpperCamelCase__			):
													try:
															snake_case_		:							Any			       =						word.index(UpperCamelCase__  ,UpperCamelCase__			)
													except ValueError:
															new_word.extend(word[i:]			)
															break
													else:
															new_word.extend(word[i:j]			)
															snake_case_		:							List[str]			       =						j

													if word[i] == first and i < len(UpperCamelCase__			) - 1 and word[i + 1] == second:
															new_word.append(first + second			)
															i += 2
													else:
															new_word.append(word[i]			)
															i += 1
											snake_case_		:							Optional[int]			       =						tuple(UpperCamelCase__			)
											snake_case_		:							Optional[Any]			       =						new_word
											if len(UpperCamelCase__			) == 1:
													break
											else:
													snake_case_		:							Union[str, Any]			       =						get_pairs(UpperCamelCase__			)
									snake_case_		:							str			       =						"""@@ """.join(UpperCamelCase__			)
									snake_case_		:							Union[str, Any]			       =						word[:-4]
									snake_case_		:							Any			       =						word
									return word





							def 		a__   (  self		:Union[str, Any]  ,_UpperCamelCase		:Tuple			):
									snake_case_		:							List[str]			       =						[]

									snake_case_		:							Optional[Any]			       =						re.findall(R"""\S+\n?"""  ,UpperCamelCase__			)

									for token in words:
											split_tokens.extend(list(self.bpe(UpperCamelCase__			).split(""" """			)			)			)
									return split_tokens





							def 		a__   (  self		:int  ,_UpperCamelCase		:str			):
									return self.encoder.get(UpperCamelCase__  ,self.encoder.get(self.unk_token			)			)





							def 		a__   (  self		:Optional[int]  ,_UpperCamelCase		:List[Any]			):
									return self.decoder.get(UpperCamelCase__  ,self.unk_token			)





							def 		a__   (  self		:Optional[Any]  ,_UpperCamelCase		:Tuple			):
									snake_case_		:							Optional[Any]			       =						""" """.join(UpperCamelCase__			).replace("""@@ """  ,""""""			).strip()
									return out_string





							def 		a__   (  self		:Optional[int]  ,_UpperCamelCase		:str  ,_UpperCamelCase		:Optional[str] = None			):






									if not os.path.isdir(UpperCamelCase__			):
											logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''			)
											return
									snake_case_		:							Tuple			       =						os.path.join(
									    UpperCamelCase__  ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)
									snake_case_		:							Optional[int]			       =						os.path.join(
									    UpperCamelCase__  ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]			)

									with open(UpperCamelCase__  ,"""w"""  ,encoding="""utf-8"""			) as f:
											f.write(json.dumps(self.encoder  ,indent=2  ,sort_keys=UpperCamelCase__  ,ensure_ascii=UpperCamelCase__			) + """\n"""			)

									snake_case_		:							str			       =						0
									with open(UpperCamelCase__  ,"""w"""  ,encoding="""utf-8"""			) as writer:
											writer.write("""#version: 0.2\n"""			)
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,key=lambda _UpperCamelCase			: kv[1]			):
													if index != token_index:
															logger.warning(
															    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
															    """ Please check that the tokenizer is not corrupted!"""			)
															snake_case_		:							Optional[Any]			       =						token_index
													writer.write(""" """.join(UpperCamelCase__			) + """\n"""			)
													index += 1

									return vocab_file, merge_file

							# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
							#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
							#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
							#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
							#     return ''.join(tokens_generated_so_far)



'''simple docstring'''






import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A	:	str     =     logging.get_logger(__name__)

__A	:	Dict     =     {
    'google/pix2struct-textcaps-base': (
        'https://huggingface.co/google/pix2struct-textcaps-base/resolve/main/config.json'
    ),
}
class 						__UpperCamelCase	(						lowercase__							):
							lowercase			:		Tuple   		=  'pix2struct_text_model'
							lowercase			:		int   		=  ['past_key_values']
							lowercase			:		int   		=  {
							    'hidden_size': 'hidden_size',
							    'num_attention_heads': 'num_heads',
							    'num_hidden_layers': 'num_layers',
							}





							def __init__(  self		:int  ,_UpperCamelCase		:List[str]=5_0_2_4_4  ,_UpperCamelCase		:Optional[Any]=7_6_8  ,_UpperCamelCase		:Dict=6_4  ,_UpperCamelCase		:Dict=2_0_4_8  ,_UpperCamelCase		:Optional[int]=1_2  ,_UpperCamelCase		:Union[str, Any]=1_2  ,_UpperCamelCase		:List[str]=3_2  ,_UpperCamelCase		:Union[str, Any]=1_2_8  ,_UpperCamelCase		:Tuple=0.1  ,_UpperCamelCase		:List[str]=1E-6  ,_UpperCamelCase		:List[Any]=1.0  ,_UpperCamelCase		:Optional[int]="gelu_new"  ,_UpperCamelCase		:Dict=0  ,_UpperCamelCase		:List[str]=False  ,_UpperCamelCase		:Optional[int]=0  ,_UpperCamelCase		:Dict=1  ,_UpperCamelCase		:List[Any]=False  ,_UpperCamelCase		:Tuple=True  ,**_UpperCamelCase		:List[Any]  ,):
									snake_case_		:							List[str]			       =						vocab_size
									snake_case_		:							Any			       =						hidden_size
									snake_case_		:							Any			       =						d_kv
									snake_case_		:							List[Any]			       =						d_ff
									snake_case_		:							Union[str, Any]			       =						num_layers
									snake_case_		:							Union[str, Any]			       =						num_heads
									snake_case_		:							str			       =						relative_attention_num_buckets
									snake_case_		:							Optional[int]			       =						relative_attention_max_distance
									snake_case_		:							Tuple			       =						dropout_rate
									snake_case_		:							Tuple			       =						layer_norm_epsilon
									snake_case_		:							Any			       =						initializer_factor
									snake_case_		:							List[Any]			       =						use_cache

									snake_case_		:							Optional[int]			       =						eos_token_id
									snake_case_		:							List[Any]			       =						decoder_start_token_id

									# for backwards compatibility
									snake_case_		:							List[str]			       =						dense_act_fn

									super().__init__(
									    pad_token_id=_UpperCamelCase  ,eos_token_id=_UpperCamelCase  ,decoder_start_token_id=_UpperCamelCase  ,tie_word_embeddings=_UpperCamelCase  ,is_decoder=_UpperCamelCase  ,**_UpperCamelCase  ,)





							@classmethod
							def 		a__   (  cls		:Optional[int]  ,_UpperCamelCase		:Union[str, os.PathLike]  ,**_UpperCamelCase		:Optional[int]			):
									cls._set_token_in_kwargs(_UpperCamelCase			)

									snake_case_       ,      snake_case_		:							Optional[Any]			       =						cls.get_config_dict(_UpperCamelCase  ,**_UpperCamelCase			)

									# get the text config dict if we are loading from Pix2StructConfig
									if config_dict.get("""model_type"""			) == "pix2struct":
											snake_case_		:							Optional[int]			       =						config_dict["""text_config"""]

									if "model_type" in config_dict and hasattr(cls  ,"""model_type"""			) and config_dict["model_type"] != cls.model_type:
											logger.warning(
											    F'''You are using a model of type {config_dict["model_type"]} to instantiate a model of type '''
											    F'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.'''			)

									return cls.from_dict(_UpperCamelCase  ,**_UpperCamelCase			)





class 						__UpperCamelCase	(						lowercase__							):
							lowercase			:		Dict   		=  'pix2struct_vision_model'





							def __init__(  self		:List[str]  ,_UpperCamelCase		:Any=7_6_8  ,_UpperCamelCase		:List[str]=7_6_8  ,_UpperCamelCase		:List[Any]=2_0_4_8  ,_UpperCamelCase		:Union[str, Any]=6_4  ,_UpperCamelCase		:int=1_2  ,_UpperCamelCase		:int=1_2  ,_UpperCamelCase		:Any="gelu_new"  ,_UpperCamelCase		:Optional[int]=1E-6  ,_UpperCamelCase		:List[Any]=0.0  ,_UpperCamelCase		:Union[str, Any]=0.0  ,_UpperCamelCase		:int=1E-1_0  ,_UpperCamelCase		:str=1.0  ,_UpperCamelCase		:Optional[int]=4_0_9_6  ,_UpperCamelCase		:str=3_2  ,_UpperCamelCase		:Union[str, Any]=1_2_8  ,**_UpperCamelCase		:Union[str, Any]  ,):
									super().__init__(**_UpperCamelCase			)

									snake_case_		:							str			       =						hidden_size
									snake_case_		:							Tuple			       =						patch_embed_hidden_size
									snake_case_		:							Optional[int]			       =						d_ff
									snake_case_		:							Dict			       =						dropout_rate
									snake_case_		:							Optional[int]			       =						num_hidden_layers
									snake_case_		:							Dict			       =						num_attention_heads
									snake_case_		:							Union[str, Any]			       =						initializer_range
									snake_case_		:							Optional[Any]			       =						initializer_factor
									snake_case_		:							List[str]			       =						attention_dropout
									snake_case_		:							List[str]			       =						layer_norm_eps
									snake_case_		:							List[str]			       =						dense_act_fn
									snake_case_		:							int			       =						seq_len
									snake_case_		:							str			       =						relative_attention_num_buckets
									snake_case_		:							Tuple			       =						relative_attention_max_distance
									snake_case_		:							List[str]			       =						d_kv





							@classmethod
							def 		a__   (  cls		:Optional[Any]  ,_UpperCamelCase		:Union[str, os.PathLike]  ,**_UpperCamelCase		:List[str]			):
									cls._set_token_in_kwargs(_UpperCamelCase			)

									snake_case_       ,      snake_case_		:							Tuple			       =						cls.get_config_dict(_UpperCamelCase  ,**_UpperCamelCase			)

									# get the vision config dict if we are loading from Pix2StructConfig
									if config_dict.get("""model_type"""			) == "pix2struct":
											snake_case_		:							Dict			       =						config_dict["""vision_config"""]

									if "model_type" in config_dict and hasattr(cls  ,"""model_type"""			) and config_dict["model_type"] != cls.model_type:
											logger.warning(
											    F'''You are using a model of type {config_dict["model_type"]} to instantiate a model of type '''
											    F'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.'''			)

									return cls.from_dict(_UpperCamelCase  ,**_UpperCamelCase			)





class 						__UpperCamelCase	(						lowercase__							):
							lowercase			:		Union[str, Any]   		=  'pix2struct'
							lowercase			:		int   		=  True





							def __init__(  self		:int  ,_UpperCamelCase		:Tuple=None  ,_UpperCamelCase		:Optional[int]=None  ,_UpperCamelCase		:Dict=1.0  ,_UpperCamelCase		:Optional[int]=0.02  ,_UpperCamelCase		:Tuple=False  ,_UpperCamelCase		:List[str]=False  ,_UpperCamelCase		:Dict=True  ,**_UpperCamelCase		:Union[str, Any]  ,):
									super().__init__(tie_word_embeddings=_UpperCamelCase  ,is_encoder_decoder=_UpperCamelCase  ,**_UpperCamelCase			)

									if text_config is None:
											snake_case_		:							Optional[int]			       =						{}
											logger.info("""text_config is None. Initializing the Pix2StructTextConfig with default values."""			)

									if vision_config is None:
											snake_case_		:							List[str]			       =						{}
											logger.info("""vision_config is None. Initializing the Pix2StructVisionConfig with default values."""			)

									snake_case_		:							Optional[int]			       =						PixaStructTextConfig(**_UpperCamelCase			)
									snake_case_		:							List[str]			       =						PixaStructVisionConfig(**_UpperCamelCase			)

									snake_case_		:							Any			       =						self.text_config.decoder_start_token_id
									snake_case_		:							List[Any]			       =						self.text_config.pad_token_id
									snake_case_		:							Optional[int]			       =						self.text_config.eos_token_id

									snake_case_		:							Optional[int]			       =						initializer_factor
									snake_case_		:							List[str]			       =						initializer_range

									snake_case_		:							Union[str, Any]			       =						self.initializer_range
									snake_case_		:							str			       =						self.initializer_range

									snake_case_		:							Dict			       =						is_vqa





							@classmethod
							def 		a__   (  cls		:Any  ,_UpperCamelCase		:PixaStructTextConfig  ,_UpperCamelCase		:PixaStructVisionConfig  ,**_UpperCamelCase		:Any			):
									return cls(text_config=text_config.to_dict()  ,vision_config=vision_config.to_dict()  ,**_UpperCamelCase			)





							def 		a__   (  self		:Optional[int]			):
									snake_case_		:							int			       =						copy.deepcopy(self.__dict__			)
									snake_case_		:							str			       =						self.text_config.to_dict()
									snake_case_		:							Tuple			       =						self.vision_config.to_dict()
									snake_case_		:							Any			       =						self.__class__.model_type
									return output