
"""simple docstring"""




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


A__   :					str							=   logging.get_logger(__name__)

A__   :					List[str]							=   {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

A__   :					Dict							=   {
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

A__   :					int							=   {
    'google/realm-cc-news-pretrained-embedder': 5_1_2,
    'google/realm-cc-news-pretrained-encoder': 5_1_2,
    'google/realm-cc-news-pretrained-scorer': 5_1_2,
    'google/realm-cc-news-pretrained-openqa': 5_1_2,
    'google/realm-orqa-nq-openqa': 5_1_2,
    'google/realm-orqa-nq-reader': 5_1_2,
    'google/realm-orqa-wq-openqa': 5_1_2,
    'google/realm-orqa-wq-reader': 5_1_2,
}

A__   :					Dict							=   {
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}


class 			__magic_name__       (       SCREAMING_SNAKE_CASE__ ):
       UpperCamelCase_   =		VOCAB_FILES_NAMES
       UpperCamelCase_   =		PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase_   =		PRETRAINED_INIT_CONFIGURATION
       UpperCamelCase_   =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase_   =		RealmTokenizer
       def __init__(     self ,				A_=None ,				A_=None ,				A_=True ,				A_="[UNK]" ,				A_="[SEP]" ,				A_="[PAD]" ,				A_="[CLS]" ,				A_="[MASK]" ,				A_=True ,				A_=None ,				**A_ ,				)     ->	Dict:





         """simple docstring"""




         super().__init__(
             A_ ,				tokenizer_file=A_ ,				do_lower_case=A_ ,				unk_token=A_ ,				sep_token=A_ ,				pad_token=A_ ,				cls_token=A_ ,				mask_token=A_ ,				tokenize_chinese_chars=A_ ,				strip_accents=A_ ,				**A_ ,				)

         _lowercase:			str       =     json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
         if (
             normalizer_state.get('''lowercase''' ,				A_       ) != do_lower_case
             or normalizer_state.get('''strip_accents''' ,				A_       ) != strip_accents
             or normalizer_state.get('''handle_chinese_chars''' ,				A_       ) != tokenize_chinese_chars
         ):
           _lowercase:			List[Any]       =     getattr(A_ ,				normalizer_state.pop('''type'''       )       )
           _lowercase:			int       =     do_lower_case
           _lowercase:			Optional[Any]       =     strip_accents
           _lowercase:			Tuple       =     tokenize_chinese_chars
           _lowercase:			List[Any]       =     normalizer_class(**A_       )

         _lowercase:			Any       =     do_lower_case
       def 							lowercase_							(     self ,				A_ ,				**A_       )     ->	Any:





         """simple docstring"""




         _lowercase:			int       =     PaddingStrategy.MAX_LENGTH

         _lowercase:			Optional[Any]       =     text
         _lowercase:			str       =     kwargs.pop('''text_pair''' ,				A_       )
         _lowercase:			str       =     kwargs.pop('''return_tensors''' ,				A_       )

         _lowercase:			Dict       =     {
             '''input_ids''': [],
             '''attention_mask''': [],
             '''token_type_ids''': [],
         }

         for idx, candidate_text in enumerate(A_       ):
           if batch_text_pair is not None:
             _lowercase:			Dict       =     batch_text_pair[idx]
           else:
             _lowercase:			List[Any]       =     None

           _lowercase:			str       =     super().__call__(A_ ,				A_ ,				return_tensors=A_ ,				**A_       )

           _lowercase:			List[str]       =     encoded_candidates.get('''input_ids'''       )
           _lowercase:			str       =     encoded_candidates.get('''attention_mask'''       )
           _lowercase:			Tuple       =     encoded_candidates.get('''token_type_ids'''       )

           if encoded_input_ids is not None:
             output_data["input_ids"].append(A_       )
           if encoded_attention_mask is not None:
             output_data["attention_mask"].append(A_       )
           if encoded_token_type_ids is not None:
             output_data["token_type_ids"].append(A_       )

         _lowercase:			Optional[Any]       =     {key: item for key, item in output_data.items() if len(A_       ) != 0}

         return BatchEncoding(A_ ,				tensor_type=A_       )
       def 							lowercase_							(     self ,				A_ ,				A_=None       )     ->	str:





         """simple docstring"""




         _lowercase:			Optional[Any]       =     [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

         return output
       def 							lowercase_							(     self ,				A_ ,				A_ = None       )     ->	List[int]:





         """simple docstring"""




         _lowercase:			List[Any]       =     [self.sep_token_id]
         _lowercase:			Dict       =     [self.cls_token_id]
         if token_ids_a is None:
           return len(cls + token_ids_a + sep       ) * [0]
         return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]



       def 							lowercase_							(     self ,				A_ ,				A_ = None       )     ->	Tuple[str]:





         """simple docstring"""




         _lowercase:			int       =     self._tokenizer.model.save(A_ ,				name=A_       )
         return tuple(A_       )


"""simple docstring"""




import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..bit import BitConfig


A__   :					Dict							=   logging.get_logger(__name__)

A__   :					int							=   {
    'Intel/dpt-large': 'https://huggingface.co/Intel/dpt-large/resolve/main/config.json',
    # See all DPT models at https://huggingface.co/models?filter=dpt
}


class 			__magic_name__       (       SCREAMING_SNAKE_CASE__ ):
       UpperCamelCase_   =		'''dpt'''
       def __init__(     self ,				A_=768 ,				A_=12 ,				A_=12 ,				A_=3072 ,				A_="gelu" ,				A_=0.0 ,				A_=0.0 ,				A_=0.02 ,				A_=1E-12 ,				A_=384 ,				A_=16 ,				A_=3 ,				A_=False ,				A_=True ,				A_=[2, 5, 8, 11] ,				A_="project" ,				A_=[4, 2, 1, 0.5] ,				A_=[96, 192, 384, 768] ,				A_=256 ,				A_=-1 ,				A_=False ,				A_=True ,				A_=0.4 ,				A_=255 ,				A_=0.1 ,				A_=[1, 1024, 24, 24] ,				A_=[0, 1] ,				A_=None ,				**A_ ,				)     ->	int:





         """simple docstring"""




         super().__init__(**A_       )

         _lowercase:			Union[str, Any]       =     hidden_size
         _lowercase:			str       =     is_hybrid

         if self.is_hybrid:
           if backbone_config is None:
             logger.info('''Initializing the config with a `BiT` backbone.'''       )
             _lowercase:			Dict       =     {
                 '''global_padding''': '''same''',
                 '''layer_type''': '''bottleneck''',
                 '''depths''': [3, 4, 9],
                 '''out_features''': ['''stage1''', '''stage2''', '''stage3'''],
                 '''embedding_dynamic_padding''': True,
             }
             _lowercase:			List[str]       =     BitConfig(**A_       )
           elif isinstance(A_ ,				A_       ):
             logger.info('''Initializing the config with a `BiT` backbone.'''       )
             _lowercase:			Dict       =     BitConfig(**A_       )
           elif isinstance(A_ ,				A_       ):
             _lowercase:			str       =     backbone_config
           else:
             raise ValueError(
                 f'''backbone_config must be a dictionary or a `PretrainedConfig`, got {backbone_config.__class__}.'''       )

           _lowercase:			Dict       =     backbone_featmap_shape
           _lowercase:			Optional[int]       =     neck_ignore_stages

           if readout_type != "project":
             raise ValueError('''Readout type must be \'project\' when using `DPT-hybrid` mode.'''       )
         else:
           _lowercase:			Any       =     None
           _lowercase:			str       =     None
           _lowercase:			Optional[Any]       =     []

         _lowercase:			Tuple       =     num_hidden_layers
         _lowercase:			Optional[int]       =     num_attention_heads
         _lowercase:			Any       =     intermediate_size
         _lowercase:			List[Any]       =     hidden_act
         _lowercase:			Union[str, Any]       =     hidden_dropout_prob
         _lowercase:			Tuple       =     attention_probs_dropout_prob
         _lowercase:			Dict       =     initializer_range
         _lowercase:			Dict       =     layer_norm_eps
         _lowercase:			Any       =     image_size
         _lowercase:			Optional[int]       =     patch_size
         _lowercase:			Optional[int]       =     num_channels
         _lowercase:			int       =     qkv_bias
         _lowercase:			List[str]       =     backbone_out_indices
         if readout_type not in ["ignore", "add", "project"]:
           raise ValueError('''Readout_type must be one of [\'ignore\', \'add\', \'project\']'''       )
         _lowercase:			Union[str, Any]       =     readout_type
         _lowercase:			Optional[Any]       =     reassemble_factors
         _lowercase:			List[str]       =     neck_hidden_sizes
         _lowercase:			Tuple       =     fusion_hidden_size
         _lowercase:			int       =     head_in_index
         _lowercase:			Optional[int]       =     use_batch_norm_in_fusion_residual
         # auxiliary head attributes (semantic segmentation)
         _lowercase:			int       =     use_auxiliary_head
         _lowercase:			Dict       =     auxiliary_loss_weight
         _lowercase:			List[Any]       =     semantic_loss_ignore_index
         _lowercase:			Dict       =     semantic_classifier_dropout



       def 							lowercase_							(     self       )     ->	str:





         """simple docstring"""




         _lowercase:			Optional[int]       =     copy.deepcopy(self.__dict__       )

         if output["backbone_config"] is not None:
           _lowercase:			List[str]       =     self.backbone_config.to_dict()

         _lowercase:			List[Any]       =     self.__class__.model_type
         return output
