


'''simple docstring'''




import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing




class 			_lowercase (	__lowercase							):
   def __init__(						self       :      Optional[int]       ,   SCREAMING_SNAKE_CASE_       :      str = "▁"       ,   SCREAMING_SNAKE_CASE_       :      bool = True       ,   SCREAMING_SNAKE_CASE_       :      Union[str, AddedToken] = "<unk>"       ,   SCREAMING_SNAKE_CASE_       :      Union[str, AddedToken] = "</s>"       ,   SCREAMING_SNAKE_CASE_       :      Union[str, AddedToken] = "<pad>"       ,   ) ->							str:
        __snake_case           =						{
            'pad': {'id': 0, 'token': pad_token},
            'eos': {'id': 1, 'token': eos_token},
            'unk': {'id': 2, 'token': unk_token},
        }

        __snake_case           =						[None] * len(self.special_tokens      )
        for token_dict in self.special_tokens.values():
             __snake_case           =						token_dict['token']

        __snake_case           =						Tokenizer(Unigram()      )

        __snake_case           =						normalizers.Sequence(
            [
                normalizers.Nmt(),
                normalizers.NFKC(),
                normalizers.Replace(Regex(' {2,}'      )       ,   ' '      ),
                normalizers.Lowercase(),
            ]      )
        __snake_case           =						pre_tokenizers.Sequence(
            [
                pre_tokenizers.Metaspace(replacement=SCREAMING_SNAKE_CASE_       ,   add_prefix_space=SCREAMING_SNAKE_CASE_      ),
                pre_tokenizers.Digits(individual_digits=SCREAMING_SNAKE_CASE_      ),
                pre_tokenizers.Punctuation(),
            ]      )
        __snake_case           =						decoders.Metaspace(replacement=SCREAMING_SNAKE_CASE_       ,   add_prefix_space=SCREAMING_SNAKE_CASE_      )

        __snake_case           =						TemplateProcessing(
            single=f'$A {self.special_tokens["eos"]["token"]}'       ,   special_tokens=[(self.special_tokens['eos']['token'], self.special_tokens['eos']['id'])]       ,   )

        __snake_case           =						{
            'model': 'SentencePieceUnigram',
            'replacement': replacement,
            'add_prefix_space': add_prefix_space,
        }

        super().__init__(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      )
   def a  (						self       :      Optional[Any]       ,   SCREAMING_SNAKE_CASE_       :      Union[str, List[str]]       ,   SCREAMING_SNAKE_CASE_       :      int = 8000       ,   SCREAMING_SNAKE_CASE_       :      bool = True       ,   ) ->							Tuple:
        __snake_case           =						trainers.UnigramTrainer(
            vocab_size=SCREAMING_SNAKE_CASE_       ,   special_tokens=self.special_tokens_list       ,   show_progress=SCREAMING_SNAKE_CASE_       ,   )

        if isinstance(SCREAMING_SNAKE_CASE_       ,   SCREAMING_SNAKE_CASE_      ):
             __snake_case           =						[files]
        self._tokenizer.train(SCREAMING_SNAKE_CASE_       ,   trainer=SCREAMING_SNAKE_CASE_      )

        self.add_unk_id()
   def a  (						self       :      str       ,   SCREAMING_SNAKE_CASE_       :      Union[Iterator[str], Iterator[Iterator[str]]]       ,   SCREAMING_SNAKE_CASE_       :      int = 8000       ,   SCREAMING_SNAKE_CASE_       :      bool = True       ,   ) ->							str:
        __snake_case           =						trainers.UnigramTrainer(
            vocab_size=SCREAMING_SNAKE_CASE_       ,   special_tokens=self.special_tokens_list       ,   show_progress=SCREAMING_SNAKE_CASE_       ,   )

        self._tokenizer.train_from_iterator(SCREAMING_SNAKE_CASE_       ,   trainer=SCREAMING_SNAKE_CASE_      )

        self.add_unk_id()





   def a  (						self       :      Dict      ) ->							str:
        __snake_case           =						json.loads(self._tokenizer.to_str()      )

        __snake_case           =						self.special_tokens['unk']['id']

        __snake_case           =						Tokenizer.from_str(json.dumps(SCREAMING_SNAKE_CASE_      )      )












from ...configuration_utils import PretrainedConfig


lowerCamelCase__       		=  {
    """google/tapas-base-finetuned-sqa""": (
        """https://huggingface.co/google/tapas-base-finetuned-sqa/resolve/main/config.json"""
    ),
    """google/tapas-base-finetuned-wtq""": (
        """https://huggingface.co/google/tapas-base-finetuned-wtq/resolve/main/config.json"""
    ),
    """google/tapas-base-finetuned-wikisql-supervised""": (
        """https://huggingface.co/google/tapas-base-finetuned-wikisql-supervised/resolve/main/config.json"""
    ),
    """google/tapas-base-finetuned-tabfact""": (
        """https://huggingface.co/google/tapas-base-finetuned-tabfact/resolve/main/config.json"""
    ),
}




class       snake_case__ (				lowercase_):

       '''simple docstring'''
       lowerCamelCase				:					Tuple								=			"tapas"



       def __init__(				self      ,    a__=3_05_22      ,    a__=7_68      ,    a__=12      ,    a__=12      ,    a__=30_72      ,    a__="gelu"      ,    a__=0.1      ,    a__=0.1      ,    a__=10_24      ,    a__=[3, 2_56, 2_56, 2, 2_56, 2_56, 10]      ,    a__=0.02      ,    a__=1e-12      ,    a__=0      ,    a__=10.0      ,    a__=0      ,    a__=1.0      ,    a__=None      ,    a__=1.0      ,    a__=False      ,    a__=None      ,    a__=1.0      ,    a__=1.0      ,    a__=False      ,    a__=False      ,    a__="ratio"      ,    a__=None      ,    a__=None      ,    a__=64      ,    a__=32      ,    a__=False      ,    a__=True      ,    a__=False      ,    a__=False      ,    a__=True      ,    a__=False      ,    a__=None      ,    a__=None      ,    **a__      ,    )			->   Any:






            '''simple docstring'''





            super().__init__(pad_token_id=a__      ,    **a__     )

            # BERT hyperparameters (with updated max_position_embeddings and type_vocab_sizes)
            __snake_case      :Optional[Any] 							=    vocab_size
            __snake_case      :Tuple 							=    hidden_size
            __snake_case      :Tuple 							=    num_hidden_layers
            __snake_case      :Any 							=    num_attention_heads
            __snake_case      :List[str] 							=    hidden_act
            __snake_case      :List[str] 							=    intermediate_size
            __snake_case      :int 							=    hidden_dropout_prob
            __snake_case      :List[Any] 							=    attention_probs_dropout_prob
            __snake_case      :Any 							=    max_position_embeddings
            __snake_case      :Optional[int] 							=    type_vocab_sizes
            __snake_case      :Tuple 							=    initializer_range
            __snake_case      :Any 							=    layer_norm_eps

            # Fine-tuning task hyperparameters
            __snake_case      :Optional[int] 							=    positive_label_weight
            __snake_case      :Optional[Any] 							=    num_aggregation_labels
            __snake_case      :Dict 							=    aggregation_loss_weight
            __snake_case      :str 							=    use_answer_as_supervision
            __snake_case      :int 							=    answer_loss_importance
            __snake_case      :List[Any] 							=    use_normalized_answer_loss
            __snake_case      :List[str] 							=    huber_loss_delta
            __snake_case      :Any 							=    temperature
            __snake_case      :Optional[int] 							=    aggregation_temperature
            __snake_case      :List[str] 							=    use_gumbel_for_cells
            __snake_case      :Optional[int] 							=    use_gumbel_for_aggregation
            __snake_case      :Optional[Any] 							=    average_approximation_function
            __snake_case      :Union[str, Any] 							=    cell_selection_preference
            __snake_case      :Tuple 							=    answer_loss_cutoff
            __snake_case      :Union[str, Any] 							=    max_num_rows
            __snake_case      :Optional[int] 							=    max_num_columns
            __snake_case      :str 							=    average_logits_per_cell
            __snake_case      :Union[str, Any] 							=    select_one_column
            __snake_case      :Optional[int] 							=    allow_empty_column_selection
            __snake_case      :Optional[Any] 							=    init_cell_selection_weights_to_zero
            __snake_case      :Tuple 							=    reset_position_index_per_cell
            __snake_case      :Any 							=    disable_per_token_loss

            # Aggregation hyperparameters
            __snake_case      :List[str] 							=    aggregation_labels
            __snake_case      :int 							=    no_aggregation_label_index

            if isinstance(self.aggregation_labels      ,    a__     ):
                 __snake_case      :Any 							=    {int(a__     ): v for k, v in aggregation_labels.items()}


