





"""simple docstring"""





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowercase__				:Dict				   =       logging.get_logger(__name__)

lowercase__				:Any				   =       {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowercase__				:Tuple				   =       {
    'vocab_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt'
        ),
        'google/realm-orqa-nq-openqa': 'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-nq-reader': 'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt',
        'google/realm-orqa-wq-openqa': 'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt',
        'google/realm-orqa-wq-reader': 'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'google/realm-cc-news-pretrained-embedder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont'
        ),
        'google/realm-cc-news-pretrained-encoder': (
            'https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-scorer': (
            'https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json'
        ),
        'google/realm-cc-news-pretrained-openqa': (
            'https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-openqa': (
            'https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-nq-reader': (
            'https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-openqa': (
            'https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json'
        ),
        'google/realm-orqa-wq-reader': (
            'https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json'
        ),
    },
}

lowercase__				:Dict				   =       {
    'google/realm-cc-news-pretrained-embedder': 5_1_2,
    'google/realm-cc-news-pretrained-encoder': 5_1_2,
    'google/realm-cc-news-pretrained-scorer': 5_1_2,
    'google/realm-cc-news-pretrained-openqa': 5_1_2,
    'google/realm-orqa-nq-openqa': 5_1_2,
    'google/realm-orqa-nq-reader': 5_1_2,
    'google/realm-orqa-wq-openqa': 5_1_2,
    'google/realm-orqa-wq-reader': 5_1_2,
}

lowercase__				:List[str]				   =       {
    'google/realm-cc-news-pretrained-embedder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-encoder': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-scorer': {'do_lower_case': True},
    'google/realm-cc-news-pretrained-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-openqa': {'do_lower_case': True},
    'google/realm-orqa-nq-reader': {'do_lower_case': True},
    'google/realm-orqa-wq-openqa': {'do_lower_case': True},
    'google/realm-orqa-wq-reader': {'do_lower_case': True},
}




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       Dict         =       VOCAB_FILES_NAMES
      _A     :       List[Any]         =       PRETRAINED_VOCAB_FILES_MAP
      _A     :       Optional[Any]         =       PRETRAINED_INIT_CONFIGURATION
      _A     :       Dict         =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _A     :       Any         =       RealmTokenizer

      def __init__(							self       :      Union[str, Any]						,				__lowercase       :      List[str]=None						,				__lowercase       :      List[Any]=None						,				__lowercase       :      int=True						,				__lowercase       :      int="[UNK]"						,				__lowercase       :      Dict="[SEP]"						,				__lowercase       :      int="[PAD]"						,				__lowercase       :      Optional[Any]="[CLS]"						,				__lowercase       :      str="[MASK]"						,				__lowercase       :      Optional[Any]=True						,				__lowercase       :      Union[str, Any]=None						,				**__lowercase       :      Union[str, Any]						,				):


        '''simple docstring'''




        super().__init__(
            __lowercase						,				tokenizer_file=__lowercase						,				do_lower_case=__lowercase						,				unk_token=__lowercase						,				sep_token=__lowercase						,				pad_token=__lowercase						,				cls_token=__lowercase						,				mask_token=__lowercase						,				tokenize_chinese_chars=__lowercase						,				strip_accents=__lowercase						,				**__lowercase						,				)

        __UpperCAmelCase    :						Any       =						json.loads(self.backend_tokenizer.normalizer.__getstate__() )
        if (
            normalizer_state.get('''lowercase'''						,				__lowercase ) != do_lower_case
            or normalizer_state.get('''strip_accents'''						,				__lowercase ) != strip_accents
            or normalizer_state.get('''handle_chinese_chars'''						,				__lowercase ) != tokenize_chinese_chars
        ):
          __UpperCAmelCase    :						List[Any]       =						getattr(__lowercase						,				normalizer_state.pop('''type''' ) )
          __UpperCAmelCase    :						List[str]       =						do_lower_case
          __UpperCAmelCase    :						List[Any]       =						strip_accents
          __UpperCAmelCase    :						Optional[Any]       =						tokenize_chinese_chars
          __UpperCAmelCase    :						int       =						normalizer_class(**__lowercase )

        __UpperCAmelCase    :						int       =						do_lower_case

      def       A_					(							self       :      Tuple						,				__lowercase       :      Tuple						,				**__lowercase       :      Optional[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						PaddingStrategy.MAX_LENGTH

        __UpperCAmelCase    :						Union[str, Any]       =						text
        __UpperCAmelCase    :						str       =						kwargs.pop('''text_pair'''						,				__lowercase )
        __UpperCAmelCase    :						Optional[int]       =						kwargs.pop('''return_tensors'''						,				__lowercase )

        __UpperCAmelCase    :						int       =						{
            '''input_ids''': [],
            '''attention_mask''': [],
            '''token_type_ids''': [],
        }

        for idx, candidate_text in enumerate(__lowercase ):
          if batch_text_pair is not None:
            __UpperCAmelCase    :						Optional[Any]       =						batch_text_pair[idx]
          else:
            __UpperCAmelCase    :						Any       =						None

          __UpperCAmelCase    :						str       =						super().__call__(__lowercase						,				__lowercase						,				return_tensors=__lowercase						,				**__lowercase )

          __UpperCAmelCase    :						Any       =						encoded_candidates.get('''input_ids''' )
          __UpperCAmelCase    :						int       =						encoded_candidates.get('''attention_mask''' )
          __UpperCAmelCase    :						str       =						encoded_candidates.get('''token_type_ids''' )

          if encoded_input_ids is not None:
            output_data["input_ids"].append(__lowercase )
          if encoded_attention_mask is not None:
            output_data["attention_mask"].append(__lowercase )
          if encoded_token_type_ids is not None:
            output_data["token_type_ids"].append(__lowercase )

        __UpperCAmelCase    :						Optional[Any]       =						{key: item for key, item in output_data.items() if len(__lowercase ) != 0}

        return BatchEncoding(__lowercase						,				tensor_type=__lowercase )

      def       A_					(							self       :      Any						,				__lowercase       :      Dict						,				__lowercase       :      Optional[int]=None ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
          output += token_ids_a + [self.sep_token_id]

        return output

      def       A_					(							self       :      Optional[Any]						,				__lowercase       :      List[int]						,				__lowercase       :      Optional[List[int]] = None ):


        '''simple docstring'''




        __UpperCAmelCase    :						Any       =						[self.sep_token_id]
        __UpperCAmelCase    :						Any       =						[self.cls_token_id]
        if token_ids_a is None:
          return len(cls + token_ids_a + sep ) * [0]
        return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







      def       A_					(							self       :      int						,				__lowercase       :      str						,				__lowercase       :      Optional[str] = None ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						self._tokenizer.model.save(__lowercase						,				name=__lowercase )
        return tuple(__lowercase )






"""simple docstring"""





import argparse
import os

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_task_guides.py
lowercase__				:int				   =       'src/transformers'
lowercase__				:List[str]				   =       'docs/source/en/tasks'


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  )						->str:





  """simple docstring"""


  with open(UpperCAmelCase_  ,			'''r'''  ,			encoding='''utf-8'''  ,			newline='''\n'''  ) as f:
    __UpperCAmelCase    :						Union[str, Any]       =						f.readlines()
  # Find the start prompt.
  __UpperCAmelCase    :						Any       =						0
  while not lines[start_index].startswith(UpperCAmelCase_  ):
    start_index += 1
  start_index += 1

  __UpperCAmelCase    :						Optional[Any]       =						start_index
  while not lines[end_index].startswith(UpperCAmelCase_  ):
    end_index += 1
  end_index -= 1

  while len(lines[start_index]  ) <= 1:
    start_index += 1
  while len(lines[end_index]  ) <= 1:
    end_index -= 1
  end_index += 1
  return "".join(lines[start_index:end_index]  ), start_index, end_index, lines


# This is to make sure the transformers module imported is the one in the repo.
lowercase__				:Any				   =       direct_transformers_import(TRANSFORMERS_PATH)

lowercase__				:List[Any]				   =       {
    'asr.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_CTC_MAPPING_NAMES,
    'audio_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES,
    'language_modeling.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
    'image_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES,
    'masked_language_modeling.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_MASKED_LM_MAPPING_NAMES,
    'multiple_choice.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES,
    'object_detection.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES,
    'question_answering.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES,
    'semantic_segmentation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING_NAMES,
    'sequence_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES,
    'summarization.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    'token_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES,
    'translation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES,
    'video_classification.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES,
    'document_question_answering.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES,
    'monocular_depth_estimation.md': transformers_module.models.auto.modeling_auto.MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES,
}

# This list contains model types used in some task guides that are not in `CONFIG_MAPPING_NAMES` (therefore not in any
# `MODEL_MAPPING_NAMES` or any `MODEL_FOR_XXX_MAPPING_NAMES`).
lowercase__				:Union[str, Any]				   =       {
    'summarization.md': ('nllb',),
    'translation.md': ('nllb',),
}


def       lowerCamelCase_       (   UpperCAmelCase_  )						->Union[str, Any]:





  """simple docstring"""


  __UpperCAmelCase    :						List[str]       =						TASK_GUIDE_TO_MODELS[task_guide]
  __UpperCAmelCase    :						Dict       =						SPECIAL_TASK_GUIDE_TO_MODEL_TYPES.get(UpperCAmelCase_  ,			set()  )
  __UpperCAmelCase    :						List[Any]       =						{
      code: name
      for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
      if (code in model_maping_names or code in special_model_types)
  }
  return ", ".join([f'''[{name}](../model_doc/{code})''' for code, name in model_names.items()]  ) + "\n"


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_=False  )						->Tuple:





  """simple docstring"""


  __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase       ,     __UpperCAmelCase    :						Dict       =						_find_text_in_file(
      filename=os.path.join(UpperCAmelCase_  ,			UpperCAmelCase_  )  ,			start_prompt='''<!--This tip is automatically generated by `make fix-copies`, do not fill manually!-->'''  ,			end_prompt='''<!--End of the generated tip-->'''  ,			)

  __UpperCAmelCase    :						List[str]       =						get_model_list_for_task(UpperCAmelCase_  )

  if current_list != new_list:
    if overwrite:
      with open(os.path.join(UpperCAmelCase_  ,			UpperCAmelCase_  )  ,			'''w'''  ,			encoding='''utf-8'''  ,			newline='''\n'''  ) as f:
        f.writelines(lines[:start_index] + [new_list] + lines[end_index:]  )
    else:
      raise ValueError(
          f'''The list of models that can be used in the {task_guide} guide needs an update. Run `make fix-copies`'''
          ''' to fix this.'''  )


if __name__ == "__main__":
  lowercase__				:int				   =       argparse.ArgumentParser()
  parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
  lowercase__				:Optional[Any]				   =       parser.parse_args()

  for task_guide in TASK_GUIDE_TO_MODELS.keys():
    check_model_list_for_task(task_guide, args.fix_and_overwrite)