

'''simple docstring'''







import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase									=     logging.get_logger(__name__)


__lowerCamelCase									=     {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all BART models at https://huggingface.co/models?filter=bart
__lowerCamelCase									=     {
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
}

__lowerCamelCase									=     {
    '''facebook/bart-base''': 1024,
    '''facebook/bart-large''': 1024,
    '''facebook/bart-large-mnli''': 1024,
    '''facebook/bart-large-cnn''': 1024,
    '''facebook/bart-large-xsum''': 1024,
    '''yjernite/bart_eli5''': 1024,
}
@lru_cache()
def    UpperCAmelCase__     (   )      ->      Union[str, Any]:
       A_						       =   (
           list(range(ord("""!"""					),							ord("""~"""					) + 1					)					) + list(range(ord("""¡"""					),							ord("""¬"""					) + 1					)					) + list(range(ord("""®"""					),							ord("""ÿ"""					) + 1					)					)
       )
       A_						       =   bs[:]
       A_						       =   0
       for b in range(2**8					):
              if b not in bs:
                     bs.append(UpperCAmelCase__					)
                     cs.append(2**8 + n					)
                     n += 1
       A_						       =   [chr(UpperCAmelCase__					) for n in cs]
       return dict(zip(UpperCAmelCase__,							UpperCAmelCase__					)					)
def    UpperCAmelCase__     (   UpperCAmelCase__					)      ->      int:
       A_						       =   set()
       A_						       =   word[0]
       for char in word[1:]:
              pairs.add((prev_char, char)					)
              A_						       =   char
       return pairs







class        A__   (   _snake_case      ):
    lowercase						       =		VOCAB_FILES_NAMES
    lowercase						       =		PRETRAINED_VOCAB_FILES_MAP
    lowercase						       =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase						       =		["input_ids", "attention_mask"]



    def __init__(       self		,					UpperCamelCase__		,					UpperCamelCase__		,					UpperCamelCase__="replace"		,					UpperCamelCase__="<s>"		,					UpperCamelCase__="</s>"		,					UpperCamelCase__="</s>"		,					UpperCamelCase__="<s>"		,					UpperCamelCase__="<unk>"		,					UpperCamelCase__="<pad>"		,					UpperCamelCase__="<mask>"		,					UpperCamelCase__=False		,					**UpperCamelCase__		,					)    ->   List[str]:





           '''simple docstring'''
           A_						       =   AddedToken(UpperCamelCase__		,					lstrip=UpperCamelCase__		,					rstrip=UpperCamelCase__							) if isinstance(UpperCamelCase__		,					UpperCamelCase__							) else bos_token
           A_						       =   AddedToken(UpperCamelCase__		,					lstrip=UpperCamelCase__		,					rstrip=UpperCamelCase__							) if isinstance(UpperCamelCase__		,					UpperCamelCase__							) else eos_token
           A_						       =   AddedToken(UpperCamelCase__		,					lstrip=UpperCamelCase__		,					rstrip=UpperCamelCase__							) if isinstance(UpperCamelCase__		,					UpperCamelCase__							) else sep_token
           A_						       =   AddedToken(UpperCamelCase__		,					lstrip=UpperCamelCase__		,					rstrip=UpperCamelCase__							) if isinstance(UpperCamelCase__		,					UpperCamelCase__							) else cls_token
           A_						       =   AddedToken(UpperCamelCase__		,					lstrip=UpperCamelCase__		,					rstrip=UpperCamelCase__							) if isinstance(UpperCamelCase__		,					UpperCamelCase__							) else unk_token
           A_						       =   AddedToken(UpperCamelCase__		,					lstrip=UpperCamelCase__		,					rstrip=UpperCamelCase__							) if isinstance(UpperCamelCase__		,					UpperCamelCase__							) else pad_token

           # Mask token behave like a normal word, i.e. include the space before it
           A_						       =   AddedToken(UpperCamelCase__		,					lstrip=UpperCamelCase__		,					rstrip=UpperCamelCase__							) if isinstance(UpperCamelCase__		,					UpperCamelCase__							) else mask_token

           super().__init__(
               errors=UpperCamelCase__		,					bos_token=UpperCamelCase__		,					eos_token=UpperCamelCase__		,					unk_token=UpperCamelCase__		,					sep_token=UpperCamelCase__		,					cls_token=UpperCamelCase__		,					pad_token=UpperCamelCase__		,					mask_token=UpperCamelCase__		,					add_prefix_space=UpperCamelCase__		,					**UpperCamelCase__		,					)

           with open(UpperCamelCase__		,					encoding="""utf-8"""							) as vocab_handle:
                  A_						       =   json.load(UpperCamelCase__							)
           A_						       =   {v: k for k, v in self.encoder.items()}
           A_						       =   errors  # how to handle errors in decoding
           A_						       =   bytes_to_unicode()
           A_						       =   {v: k for k, v in self.byte_encoder.items()}
           with open(UpperCamelCase__		,					encoding="""utf-8"""							) as merges_handle:
                  A_						       =   merges_handle.read().split("""\n"""							)[1:-1]
           A_						       =   [tuple(merge.split()							) for merge in bpe_merges]
           A_						       =   dict(zip(UpperCamelCase__		,					range(len(UpperCamelCase__							)							)							)							)
           A_						       =   {}
           A_						       =   add_prefix_space

           # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
           A_						       =   re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""							)



    @property
    def        snake_case_      (       self							)    ->   str:





           '''simple docstring'''
           return len(self.encoder							)



    def        snake_case_      (       self							)    ->   Tuple:





           '''simple docstring'''
           return dict(self.encoder		,					**self.added_tokens_encoder							)



    def        snake_case_      (       self		,					UpperCamelCase__							)    ->   int:





           '''simple docstring'''
           if token in self.cache:
                  return self.cache[token]
           A_						       =   tuple(UpperCamelCase__							)
           A_						       =   get_pairs(UpperCamelCase__							)

           if not pairs:
                  return token

           while True:
                  A_						       =   min(UpperCamelCase__		,					key=lambda UpperCamelCase__							: self.bpe_ranks.get(UpperCamelCase__		,					float("""inf"""							)							)							)
                  if bigram not in self.bpe_ranks:
                         break
                  A_       ,      A_						       =   bigram
                  A_						       =   []
                  A_						       =   0
                  while i < len(UpperCamelCase__							):
                         try:
                                A_						       =   word.index(UpperCamelCase__		,					UpperCamelCase__							)
                         except ValueError:
                                new_word.extend(word[i:]							)
                                break
                         else:
                                new_word.extend(word[i:j]							)
                                A_						       =   j

                         if word[i] == first and i < len(UpperCamelCase__							) - 1 and word[i + 1] == second:
                                new_word.append(first + second							)
                                i += 2
                         else:
                                new_word.append(word[i]							)
                                i += 1
                  A_						       =   tuple(UpperCamelCase__							)
                  A_						       =   new_word
                  if len(UpperCamelCase__							) == 1:
                         break
                  else:
                         A_						       =   get_pairs(UpperCamelCase__							)
           A_						       =   """ """.join(UpperCamelCase__							)
           A_						       =   word
           return word



    def        snake_case_      (       self		,					UpperCamelCase__							)    ->   str:





           '''simple docstring'''
           A_						       =   []
           for token in re.findall(self.pat		,					UpperCamelCase__							):
                  A_						       =   """""".join(
                      self.byte_encoder[b] for b in token.encode("""utf-8"""							)							)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
                  bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCamelCase__							).split(""" """							)							)
           return bpe_tokens



    def        snake_case_      (       self		,					UpperCamelCase__							)    ->   Optional[int]:





           '''simple docstring'''
           return self.encoder.get(UpperCamelCase__		,					self.encoder.get(self.unk_token							)							)



    def        snake_case_      (       self		,					UpperCamelCase__							)    ->   Union[str, Any]:





           '''simple docstring'''
           return self.decoder.get(UpperCamelCase__							)



    def        snake_case_      (       self		,					UpperCamelCase__							)    ->   List[Any]:





           '''simple docstring'''
           A_						       =   """""".join(UpperCamelCase__							)
           A_						       =   bytearray([self.byte_decoder[c] for c in text]							).decode("""utf-8"""		,					errors=self.errors							)
           return text



    def        snake_case_      (       self		,					UpperCamelCase__		,					UpperCamelCase__ = None							)    ->   Tuple[str]:





           '''simple docstring'''
           if not os.path.isdir(UpperCamelCase__							):
                  logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''							)
                  return
           A_						       =   os.path.join(
               UpperCamelCase__		,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)
           A_						       =   os.path.join(
               UpperCamelCase__		,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]							)

           with open(UpperCamelCase__		,					"""w"""		,					encoding="""utf-8"""							) as f:
                  f.write(json.dumps(self.encoder		,					indent=2		,					sort_keys=UpperCamelCase__		,					ensure_ascii=UpperCamelCase__							) + """\n"""							)

           A_						       =   0
           with open(UpperCamelCase__		,					"""w"""		,					encoding="""utf-8"""							) as writer:
                  writer.write("""#version: 0.2\n"""							)
                  for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,					key=lambda UpperCamelCase__							: kv[1]							):
                         if index != token_index:
                                logger.warning(
                                    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                                    """ Please check that the tokenizer is not corrupted!"""							)
                                A_						       =   token_index
                         writer.write(""" """.join(UpperCamelCase__							) + """\n"""							)
                         index += 1

           return vocab_file, merge_file



    def        snake_case_      (       self		,					UpperCamelCase__		,					UpperCamelCase__ = None							)    ->   List[int]:





           '''simple docstring'''
           if token_ids_a is None:
                  return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
           A_						       =   [self.cls_token_id]
           A_						       =   [self.sep_token_id]
           return cls + token_ids_a + sep + sep + token_ids_a + sep



    def        snake_case_      (       self		,					UpperCamelCase__		,					UpperCamelCase__ = None		,					UpperCamelCase__ = False							)    ->   List[int]:





           '''simple docstring'''
           if already_has_special_tokens:
                  return super().get_special_tokens_mask(
                      token_ids_a=UpperCamelCase__		,					token_ids_a=UpperCamelCase__		,					already_has_special_tokens=UpperCamelCase__							)

           if token_ids_a is None:
                  return [1] + ([0] * len(UpperCamelCase__							)) + [1]
           return [1] + ([0] * len(UpperCamelCase__							)) + [1, 1] + ([0] * len(UpperCamelCase__							)) + [1]



    def        snake_case_      (       self		,					UpperCamelCase__		,					UpperCamelCase__ = None							)    ->   List[int]:





           '''simple docstring'''
           A_						       =   [self.sep_token_id]
           A_						       =   [self.cls_token_id]

           if token_ids_a is None:
                  return len(cls + token_ids_a + sep							) * [0]
           return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]







    def        snake_case_      (       self		,					UpperCamelCase__		,					UpperCamelCase__=False		,					**UpperCamelCase__							)    ->   Any:





           '''simple docstring'''
           A_						       =   kwargs.pop("""add_prefix_space"""		,					self.add_prefix_space							)
           if (is_split_into_words or add_prefix_space) and (len(UpperCamelCase__							) > 0 and not text[0].isspace()):
                  A_						       =   """ """ + text
           return (text, kwargs)



'''simple docstring'''







import copy
import os
from typing import Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCamelCase									=     logging.get_logger(__name__)

__lowerCamelCase									=     {
    '''google/pix2struct-textcaps-base''': (
        '''https://huggingface.co/google/pix2struct-textcaps-base/resolve/main/config.json'''
    ),
}


class        A__   (   _snake_case      ):
    lowercase						       =		"pix2struct_text_model"
    lowercase						       =		["past_key_values"]
    lowercase						       =		{
        "hidden_size": "hidden_size",
        "num_attention_heads": "num_heads",
        "num_hidden_layers": "num_layers",
    }



    def __init__(       self		,					UpperCamelCase__=50244		,					UpperCamelCase__=768		,					UpperCamelCase__=64		,					UpperCamelCase__=2048		,					UpperCamelCase__=12		,					UpperCamelCase__=12		,					UpperCamelCase__=32		,					UpperCamelCase__=128		,					UpperCamelCase__=0.1		,					UpperCamelCase__=1e-6		,					UpperCamelCase__=1.0		,					UpperCamelCase__="gelu_new"		,					UpperCamelCase__=0		,					UpperCamelCase__=False		,					UpperCamelCase__=0		,					UpperCamelCase__=1		,					UpperCamelCase__=False		,					UpperCamelCase__=True		,					**UpperCamelCase__		,					)    ->   List[str]:





           '''simple docstring'''
           A_						       =   vocab_size
           A_						       =   hidden_size
           A_						       =   d_kv
           A_						       =   d_ff
           A_						       =   num_layers
           A_						       =   num_heads
           A_						       =   relative_attention_num_buckets
           A_						       =   relative_attention_max_distance
           A_						       =   dropout_rate
           A_						       =   layer_norm_epsilon
           A_						       =   initializer_factor
           A_						       =   use_cache

           A_						       =   eos_token_id
           A_						       =   decoder_start_token_id

           # for backwards compatibility
           A_						       =   dense_act_fn

           super().__init__(
               pad_token_id=UpperCamelCase__		,					eos_token_id=UpperCamelCase__		,					decoder_start_token_id=UpperCamelCase__		,					tie_word_embeddings=UpperCamelCase__		,					is_decoder=UpperCamelCase__		,					**UpperCamelCase__		,					)







    @classmethod
    def        snake_case_      (       cls		,					UpperCamelCase__		,					**UpperCamelCase__							)    ->   "PretrainedConfig":





           '''simple docstring'''
           cls._set_token_in_kwargs(UpperCamelCase__							)

           A_       ,      A_						       =   cls.get_config_dict(UpperCamelCase__		,					**UpperCamelCase__							)

           # get the text config dict if we are loading from Pix2StructConfig
           if config_dict.get("""model_type"""							) == "pix2struct":
                  A_						       =   config_dict["""text_config"""]

           if "model_type" in config_dict and hasattr(cls		,					"""model_type"""							) and config_dict["model_type"] != cls.model_type:
                  logger.warning(
                      f'''You are using a model of type {config_dict["model_type"]} to instantiate a model of type '''
                      f'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.'''							)

           return cls.from_dict(UpperCamelCase__		,					**UpperCamelCase__							)


class        A__   (   _snake_case      ):
    lowercase						       =		"pix2struct_vision_model"



    def __init__(       self		,					UpperCamelCase__=768		,					UpperCamelCase__=768		,					UpperCamelCase__=2048		,					UpperCamelCase__=64		,					UpperCamelCase__=12		,					UpperCamelCase__=12		,					UpperCamelCase__="gelu_new"		,					UpperCamelCase__=1e-6		,					UpperCamelCase__=0.0		,					UpperCamelCase__=0.0		,					UpperCamelCase__=1e-1_0		,					UpperCamelCase__=1.0		,					UpperCamelCase__=4096		,					UpperCamelCase__=32		,					UpperCamelCase__=128		,					**UpperCamelCase__		,					)    ->   int:





           '''simple docstring'''
           super().__init__(**UpperCamelCase__							)

           A_						       =   hidden_size
           A_						       =   patch_embed_hidden_size
           A_						       =   d_ff
           A_						       =   dropout_rate
           A_						       =   num_hidden_layers
           A_						       =   num_attention_heads
           A_						       =   initializer_range
           A_						       =   initializer_factor
           A_						       =   attention_dropout
           A_						       =   layer_norm_eps
           A_						       =   dense_act_fn
           A_						       =   seq_len
           A_						       =   relative_attention_num_buckets
           A_						       =   relative_attention_max_distance
           A_						       =   d_kv







    @classmethod
    def        snake_case_      (       cls		,					UpperCamelCase__		,					**UpperCamelCase__							)    ->   "PretrainedConfig":





           '''simple docstring'''
           cls._set_token_in_kwargs(UpperCamelCase__							)

           A_       ,      A_						       =   cls.get_config_dict(UpperCamelCase__		,					**UpperCamelCase__							)

           # get the vision config dict if we are loading from Pix2StructConfig
           if config_dict.get("""model_type"""							) == "pix2struct":
                  A_						       =   config_dict["""vision_config"""]

           if "model_type" in config_dict and hasattr(cls		,					"""model_type"""							) and config_dict["model_type"] != cls.model_type:
                  logger.warning(
                      f'''You are using a model of type {config_dict["model_type"]} to instantiate a model of type '''
                      f'''{cls.model_type}. This is not supported for all configurations of models and can yield errors.'''							)

           return cls.from_dict(UpperCamelCase__		,					**UpperCamelCase__							)


class        A__   (   _snake_case      ):
    lowercase						       =		"pix2struct"
    lowercase						       =		True



    def __init__(       self		,					UpperCamelCase__=None		,					UpperCamelCase__=None		,					UpperCamelCase__=1.0		,					UpperCamelCase__=0.02		,					UpperCamelCase__=False		,					UpperCamelCase__=False		,					UpperCamelCase__=True		,					**UpperCamelCase__		,					)    ->   str:





           '''simple docstring'''
           super().__init__(tie_word_embeddings=UpperCamelCase__		,					is_encoder_decoder=UpperCamelCase__		,					**UpperCamelCase__							)

           if text_config is None:
                  A_						       =   {}
                  logger.info("""text_config is None. Initializing the Pix2StructTextConfig with default values."""							)

           if vision_config is None:
                  A_						       =   {}
                  logger.info("""vision_config is None. Initializing the Pix2StructVisionConfig with default values."""							)

           A_						       =   PixaStructTextConfig(**UpperCamelCase__							)
           A_						       =   PixaStructVisionConfig(**UpperCamelCase__							)

           A_						       =   self.text_config.decoder_start_token_id
           A_						       =   self.text_config.pad_token_id
           A_						       =   self.text_config.eos_token_id

           A_						       =   initializer_factor
           A_						       =   initializer_range

           A_						       =   self.initializer_range
           A_						       =   self.initializer_range

           A_						       =   is_vqa



    @classmethod
    def        snake_case_      (       cls		,					UpperCamelCase__		,					UpperCamelCase__		,					**UpperCamelCase__							)    ->   Optional[int]:





           '''simple docstring'''
           return cls(text_config=text_config.to_dict()		,					vision_config=vision_config.to_dict()		,					**UpperCamelCase__							)







    def        snake_case_      (       self							)    ->   List[Any]:





           '''simple docstring'''
           A_						       =   copy.deepcopy(self.__dict__							)
           A_						       =   self.text_config.to_dict()
           A_						       =   self.vision_config.to_dict()
           A_						       =   self.__class__.model_type
           return output
