

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


__A			:					Optional[Any]						=			logging.get_logger(__name__)

__A			:					int						=			{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__A			:					int						=			{
    'vocab_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/vocab.txt',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/vocab.txt',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt'
        ),
        'bert-base-multilingual-cased': 'https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt',
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt'
        ),
        'bert-base-german-dbmdz-cased': 'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt',
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'bert-base-uncased': 'https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json',
        'bert-large-uncased': 'https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json',
        'bert-base-cased': 'https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json',
        'bert-large-cased': 'https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json',
        'bert-base-multilingual-uncased': (
            'https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json'
        ),
        'bert-base-multilingual-cased': (
            'https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-chinese': 'https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json',
        'bert-base-german-cased': 'https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json',
        'bert-large-uncased-whole-word-masking': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking': (
            'https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json'
        ),
        'bert-large-uncased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-large-cased-whole-word-masking-finetuned-squad': (
            'https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json'
        ),
        'bert-base-cased-finetuned-mrpc': (
            'https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-cased': (
            'https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json'
        ),
        'bert-base-german-dbmdz-uncased': (
            'https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-cased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json'
        ),
        'TurkuNLP/bert-base-finnish-uncased-v1': (
            'https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json'
        ),
        'wietsedv/bert-base-dutch-cased': (
            'https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json'
        ),
    },
}

__A			:					Dict						=			{
    'bert-base-uncased': 5_1_2,
    'bert-large-uncased': 5_1_2,
    'bert-base-cased': 5_1_2,
    'bert-large-cased': 5_1_2,
    'bert-base-multilingual-uncased': 5_1_2,
    'bert-base-multilingual-cased': 5_1_2,
    'bert-base-chinese': 5_1_2,
    'bert-base-german-cased': 5_1_2,
    'bert-large-uncased-whole-word-masking': 5_1_2,
    'bert-large-cased-whole-word-masking': 5_1_2,
    'bert-large-uncased-whole-word-masking-finetuned-squad': 5_1_2,
    'bert-large-cased-whole-word-masking-finetuned-squad': 5_1_2,
    'bert-base-cased-finetuned-mrpc': 5_1_2,
    'bert-base-german-dbmdz-cased': 5_1_2,
    'bert-base-german-dbmdz-uncased': 5_1_2,
    'TurkuNLP/bert-base-finnish-cased-v1': 5_1_2,
    'TurkuNLP/bert-base-finnish-uncased-v1': 5_1_2,
    'wietsedv/bert-base-dutch-cased': 5_1_2,
}

__A			:					Optional[Any]						=			{
    'bert-base-uncased': {'do_lower_case': True},
    'bert-large-uncased': {'do_lower_case': True},
    'bert-base-cased': {'do_lower_case': False},
    'bert-large-cased': {'do_lower_case': False},
    'bert-base-multilingual-uncased': {'do_lower_case': True},
    'bert-base-multilingual-cased': {'do_lower_case': False},
    'bert-base-chinese': {'do_lower_case': False},
    'bert-base-german-cased': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking': {'do_lower_case': False},
    'bert-large-uncased-whole-word-masking-finetuned-squad': {'do_lower_case': True},
    'bert-large-cased-whole-word-masking-finetuned-squad': {'do_lower_case': False},
    'bert-base-cased-finetuned-mrpc': {'do_lower_case': False},
    'bert-base-german-dbmdz-cased': {'do_lower_case': False},
    'bert-base-german-dbmdz-uncased': {'do_lower_case': True},
    'TurkuNLP/bert-base-finnish-cased-v1': {'do_lower_case': False},
    'TurkuNLP/bert-base-finnish-uncased-v1': {'do_lower_case': True},
    'wietsedv/bert-base-dutch-cased': {'do_lower_case': False},
}





class 			_SCREAMING_SNAKE_CASE (   __snake_case		):







   '''simple docstring'''




   lowerCamelCase__				=      VOCAB_FILES_NAMES
   lowerCamelCase__				=      PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase__				=      PRETRAINED_INIT_CONFIGURATION
   lowerCamelCase__				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCamelCase__				=      BertTokenizer





   def __init__(       self					:			Dict     , __lowerCamelCase					:			Optional[Any]=None     , __lowerCamelCase					:			Optional[int]=None     , __lowerCamelCase					:			Union[str, Any]=True     , __lowerCamelCase					:			List[Any]="[UNK]"     , __lowerCamelCase					:			Dict="[SEP]"     , __lowerCamelCase					:			List[str]="[PAD]"     , __lowerCamelCase					:			Optional[int]="[CLS]"     , __lowerCamelCase					:			Optional[Any]="[MASK]"     , __lowerCamelCase					:			Any=True     , __lowerCamelCase					:			List[str]=None     , **__lowerCamelCase					:			Dict     , ):
       super().__init__(
           __lowerCamelCase     , tokenizer_file=__lowerCamelCase     , do_lower_case=__lowerCamelCase     , unk_token=__lowerCamelCase     , sep_token=__lowerCamelCase     , pad_token=__lowerCamelCase     , cls_token=__lowerCamelCase     , mask_token=__lowerCamelCase     , tokenize_chinese_chars=__lowerCamelCase     , strip_accents=__lowerCamelCase     , **__lowerCamelCase     , )

       SCREAMING_SNAKE_CASE										=   json.loads(self.backend_tokenizer.normalizer.__getstate__()			)
       if (
           normalizer_state.get("lowercase"     , __lowerCamelCase			) != do_lower_case
           or normalizer_state.get("strip_accents"     , __lowerCamelCase			) != strip_accents
           or normalizer_state.get("handle_chinese_chars"     , __lowerCamelCase			) != tokenize_chinese_chars
       ):
           SCREAMING_SNAKE_CASE										=   getattr(__lowerCamelCase     , normalizer_state.pop("type"			)			)
           SCREAMING_SNAKE_CASE										=   do_lower_case
           SCREAMING_SNAKE_CASE										=   strip_accents
           SCREAMING_SNAKE_CASE										=   tokenize_chinese_chars
           SCREAMING_SNAKE_CASE										=   normalizer_class(**__lowerCamelCase			)

       SCREAMING_SNAKE_CASE										=   do_lower_case





   def      _snake_case		(       self					:			Optional[Any]     , __lowerCamelCase					:			List[str]     , __lowerCamelCase					:			List[Any]=None			):
       SCREAMING_SNAKE_CASE										=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

       return output





   def      _snake_case		(       self					:			Optional[int]     , __lowerCamelCase					:			List[int]     , __lowerCamelCase					:			Optional[List[int]] = None			):
       SCREAMING_SNAKE_CASE										=   [self.sep_token_id]
       SCREAMING_SNAKE_CASE										=   [self.cls_token_id]
       if token_ids_a is None:
           return len(cls + token_ids_a + sep			) * [0]
       return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]





   def      _snake_case		(       self					:			Tuple     , __lowerCamelCase					:			str     , __lowerCamelCase					:			Optional[str] = None			):
       SCREAMING_SNAKE_CASE										=   self._tokenizer.model.save(__lowerCamelCase     , name=__lowerCamelCase			)
       return tuple(__lowerCamelCase			)


import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Union

from .generation.configuration_utils import GenerationConfig
from .training_args import TrainingArguments
from .utils import add_start_docstrings


__A			:					Optional[Any]						=			logging.getLogger(__name__)





@dataclass
@add_start_docstrings(TrainingArguments.__doc__		)
class 			_SCREAMING_SNAKE_CASE (   __snake_case		):







   '''simple docstring'''




   lowerCamelCase__				=      field(default=__snake_case					,       metadata={"help": "Whether to use SortishSampler or not."}		)
   lowerCamelCase__				=      field(
       default=__snake_case					,       metadata={"help": "Whether to use generate to calculate generative metrics (ROUGE, BLEU)."}		)
   lowerCamelCase__				=      field(
       default=__snake_case					,       metadata={
           "help": (
               "The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default "
               "to the `max_length` value of the model configuration."
           )
       }					,       )
   lowerCamelCase__				=      field(
       default=__snake_case					,       metadata={
           "help": (
               "The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default "
               "to the `num_beams` value of the model configuration."
           )
       }					,       )
   lowerCamelCase__				=      field(
       default=__snake_case					,       metadata={
           "help": "Model id, file path or url pointing to a GenerationConfig json file, to use during prediction."
       }					,       )





   def      _snake_case		(       self					:			Union[str, Any]			):
       SCREAMING_SNAKE_CASE										=   super().to_dict()
       for k, v in d.items():
           if isinstance(__lowerCamelCase     , __lowerCamelCase			):
               SCREAMING_SNAKE_CASE										=   v.to_dict()
       return d