





"""simple docstring"""





import argparse
import json
from pathlib import Path

import requests
import timm
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import AutoImageProcessor, SwinvaConfig, SwinvaForImageClassification
def 					__A  (_SCREAMING_SNAKE_CASE     )   ->Union[str, Any]:





  """simple docstring"""


  lowerCAmelCase__     :Dict											=				SwinvaConfig()
  lowerCAmelCase__     :List[str]											=				swinva_name.split('_'     )

  lowerCAmelCase__     :Tuple											=				name_split[1]
  if "to" in name_split[3]:
    lowerCAmelCase__     :Optional[Any]											=				int(name_split[3][-3:]     )
  else:
    lowerCAmelCase__     :Optional[Any]											=				int(name_split[3]     )
  if "to" in name_split[2]:
    lowerCAmelCase__     :Optional[int]											=				int(name_split[2][-2:]     )
  else:
    lowerCAmelCase__     :List[Any]											=				int(name_split[2][6:]     )

  if model_size == "tiny":
    lowerCAmelCase__     :Tuple											=				96
    lowerCAmelCase__     :List[Any]											=				(2, 2, 6, 2)
    lowerCAmelCase__     :Union[str, Any]											=				(3, 6, 12, 24)
  elif model_size == "small":
    lowerCAmelCase__     :Union[str, Any]											=				96
    lowerCAmelCase__     :List[str]											=				(2, 2, 18, 2)
    lowerCAmelCase__     :str											=				(3, 6, 12, 24)
  elif model_size == "base":
    lowerCAmelCase__     :List[Any]											=				128
    lowerCAmelCase__     :Union[str, Any]											=				(2, 2, 18, 2)
    lowerCAmelCase__     :List[str]											=				(4, 8, 16, 32)
  else:
    lowerCAmelCase__     :List[str]											=				192
    lowerCAmelCase__     :int											=				(2, 2, 18, 2)
    lowerCAmelCase__     :int											=				(6, 12, 24, 48)

  if "to" in swinva_name:
    lowerCAmelCase__     :Optional[Any]											=				(12, 12, 12, 6)

  if ("22k" in swinva_name) and ("to" not in swinva_name):
    lowerCAmelCase__     :Union[str, Any]											=				2_1841
    lowerCAmelCase__     :Any											=				'huggingface/label-files'
    lowerCAmelCase__     :Any											=				'imagenet-22k-id2label.json'
    lowerCAmelCase__     :Optional[int]											=				json.load(open(hf_hub_download(_SCREAMING_SNAKE_CASE	,  _SCREAMING_SNAKE_CASE	,  repo_type='dataset'     )	,  'r'     )     )
    lowerCAmelCase__     :Any											=				{int(_SCREAMING_SNAKE_CASE     ): v for k, v in idalabel.items()}
    lowerCAmelCase__     :Dict											=				idalabel
    lowerCAmelCase__     :str											=				{v: k for k, v in idalabel.items()}

  else:
    lowerCAmelCase__     :Any											=				1000
    lowerCAmelCase__     :str											=				'huggingface/label-files'
    lowerCAmelCase__     :str											=				'imagenet-1k-id2label.json'
    lowerCAmelCase__     :Any											=				json.load(open(hf_hub_download(_SCREAMING_SNAKE_CASE	,  _SCREAMING_SNAKE_CASE	,  repo_type='dataset'     )	,  'r'     )     )
    lowerCAmelCase__     :List[Any]											=				{int(_SCREAMING_SNAKE_CASE     ): v for k, v in idalabel.items()}
    lowerCAmelCase__     :Union[str, Any]											=				idalabel
    lowerCAmelCase__     :List[Any]											=				{v: k for k, v in idalabel.items()}

  lowerCAmelCase__     :Any											=				img_size
  lowerCAmelCase__     :Dict											=				num_classes
  lowerCAmelCase__     :List[str]											=				embed_dim
  lowerCAmelCase__     :Dict											=				depths
  lowerCAmelCase__     :Optional[Any]											=				num_heads
  lowerCAmelCase__     :Optional[int]											=				window_size

  return config
def 					__A  (_SCREAMING_SNAKE_CASE     )   ->Optional[Any]:





  """simple docstring"""


  if "patch_embed.proj" in name:
    lowerCAmelCase__     :Dict											=				name.replace('patch_embed.proj'	,  'embeddings.patch_embeddings.projection'     )
  if "patch_embed.norm" in name:
    lowerCAmelCase__     :int											=				name.replace('patch_embed.norm'	,  'embeddings.norm'     )
  if "layers" in name:
    lowerCAmelCase__     :int											=				'encoder.' + name
  if "attn.proj" in name:
    lowerCAmelCase__     :List[str]											=				name.replace('attn.proj'	,  'attention.output.dense'     )
  if "attn" in name:
    lowerCAmelCase__     :List[str]											=				name.replace('attn'	,  'attention.self'     )
  if "norm1" in name:
    lowerCAmelCase__     :int											=				name.replace('norm1'	,  'layernorm_before'     )
  if "norm2" in name:
    lowerCAmelCase__     :int											=				name.replace('norm2'	,  'layernorm_after'     )
  if "mlp.fc1" in name:
    lowerCAmelCase__     :List[str]											=				name.replace('mlp.fc1'	,  'intermediate.dense'     )
  if "mlp.fc2" in name:
    lowerCAmelCase__     :Tuple											=				name.replace('mlp.fc2'	,  'output.dense'     )
  if "q_bias" in name:
    lowerCAmelCase__     :int											=				name.replace('q_bias'	,  'query.bias'     )
  if "k_bias" in name:
    lowerCAmelCase__     :Any											=				name.replace('k_bias'	,  'key.bias'     )
  if "v_bias" in name:
    lowerCAmelCase__     :Dict											=				name.replace('v_bias'	,  'value.bias'     )
  if "cpb_mlp" in name:
    lowerCAmelCase__     :Optional[int]											=				name.replace('cpb_mlp'	,  'continuous_position_bias_mlp'     )
  if name == "norm.weight":
    lowerCAmelCase__     :Any											=				'layernorm.weight'
  if name == "norm.bias":
    lowerCAmelCase__     :Optional[Any]											=				'layernorm.bias'

  if "head" in name:
    lowerCAmelCase__     :int											=				name.replace('head'	,  'classifier'     )
  else:
    lowerCAmelCase__     :Optional[Any]											=				'swinv2.' + name

  return name
def 					__A  (_SCREAMING_SNAKE_CASE	,  _SCREAMING_SNAKE_CASE     )   ->Union[str, Any]:





  """simple docstring"""


  for key in orig_state_dict.copy().keys():
    lowerCAmelCase__     :Tuple											=				orig_state_dict.pop(_SCREAMING_SNAKE_CASE     )

    if "mask" in key:
      continue
    elif "qkv" in key:
      lowerCAmelCase__     :List[Any]											=				key.split('.'     )
      lowerCAmelCase__     :Union[str, Any]											=				int(key_split[1]     )
      lowerCAmelCase__     :Tuple											=				int(key_split[3]     )
      lowerCAmelCase__     :Tuple											=				model.swinva.encoder.layers[layer_num].blocks[block_num].attention.self.all_head_size

      if "weight" in key:
        lowerCAmelCase__     :Union[str, Any]											=				val[:dim, :]
        lowerCAmelCase__     :Any											=				val[dim : dim * 2, :]
        lowerCAmelCase__     :Dict											=				val[-dim:, :]
      else:
        lowerCAmelCase__     :Dict											=				val[:dim]
        lowerCAmelCase__     :Optional[int]											=				val[
            dim : dim * 2
        ]
        lowerCAmelCase__     :str											=				val[-dim:]
    else:
      lowerCAmelCase__     :Tuple											=				val

  return orig_state_dict
def 					__A  (_SCREAMING_SNAKE_CASE	,  _SCREAMING_SNAKE_CASE     )   ->Optional[int]:





  """simple docstring"""


  lowerCAmelCase__     :Any											=				timm.create_model(_SCREAMING_SNAKE_CASE	,  pretrained=_SCREAMING_SNAKE_CASE     )
  timm_model.eval()

  lowerCAmelCase__     :Tuple											=				get_swinva_config(_SCREAMING_SNAKE_CASE     )
  lowerCAmelCase__     :List[Any]											=				SwinvaForImageClassification(_SCREAMING_SNAKE_CASE     )
  model.eval()

  lowerCAmelCase__     :Dict											=				convert_state_dict(timm_model.state_dict()	,  _SCREAMING_SNAKE_CASE     )
  model.load_state_dict(_SCREAMING_SNAKE_CASE     )

  lowerCAmelCase__     :Dict											=				'http://images.cocodataset.org/val2017/000000039769.jpg'

  lowerCAmelCase__     :List[str]											=				AutoImageProcessor.from_pretrained('microsoft/{}'.format(swinva_name.replace('_'	,  '-'     )     )     )
  lowerCAmelCase__     :Tuple											=				Image.open(requests.get(_SCREAMING_SNAKE_CASE	,  stream=_SCREAMING_SNAKE_CASE     ).raw     )
  lowerCAmelCase__     :str											=				image_processor(images=_SCREAMING_SNAKE_CASE	,  return_tensors='pt'     )

  lowerCAmelCase__     :Optional[Any]											=				timm_model(inputs['pixel_values']     )
  lowerCAmelCase__     :List[Any]											=				model(**_SCREAMING_SNAKE_CASE     ).logits

  assert torch.allclose(_SCREAMING_SNAKE_CASE	,  _SCREAMING_SNAKE_CASE	,  atol=1e-3     )

  print(F"Saving model {swinva_name} to {pytorch_dump_folder_path}"     )
  model.save_pretrained(_SCREAMING_SNAKE_CASE     )

  print(F"Saving image processor to {pytorch_dump_folder_path}"     )
  image_processor.save_pretrained(_SCREAMING_SNAKE_CASE     )

  model.push_to_hub(
      repo_path_or_name=Path(_SCREAMING_SNAKE_CASE	,  _SCREAMING_SNAKE_CASE     )	,  organization='nandwalritik'	,  commit_message='Add model'	,  )


if __name__ == "__main__":
  __A											=						argparse.ArgumentParser()
  # Required parameters
  parser.add_argument(
      """--swinv2_name""",
      default="""swinv2_tiny_patch4_window8_256""",
      type=str,
      help="""Name of the Swinv2 timm model you'd like to convert.""",
  )
  parser.add_argument(
      """--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model directory."""
  )

  __A											=						parser.parse_args()
  convert_swinva_checkpoint(args.swinva_name, args.pytorch_dump_folder_path)


import json
import logging
import math
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

from datasets import Dataset, load_dataset

import transformers
from transformers import (
    CONFIG_MAPPING,
    MODEL_FOR_MASKED_LM_MAPPING,
    AutoConfig,
    AutoModelForMaskedLM,
    AutoTokenizer,
    DataCollatorForWholeWordMask,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import get_last_checkpoint, is_main_process


lowercase   =	logging.getLogger(__name__)
lowercase   =	list(MODEL_FOR_MASKED_LM_MAPPING.keys())
lowercase   =	tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)





@dataclass
class    __A:
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={
					        '''help''': (
					            '''The model checkpoint for weights initialization.Don\'t set if you want to train a model from scratch.'''
					        )
					    }  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''If training from scratch, pass a model type from the list: ''' + ''', '''.join(UpperCAmelCase							)}  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={
					        '''help''': (
					            '''Override some existing default config settings when a model is trained from scratch. Example: '''
					            '''n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index'''
					        )
					    }  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''Pretrained config name or path if not the same as model_name'''}							)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''Pretrained tokenizer name or path if not the same as model_name'''}							)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co'''}  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''Whether to use one of the fast tokenizer (backed by the tokenizers library) or not.'''}  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default='''main'''  ,		metadata={'''help''': '''The specific model version to use (can be a branch name, tag name or commit id).'''}  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={
					        '''help''': (
					            '''Will use the token generated when running `huggingface-cli login` (necessary to use this script '''
					            '''with private models).'''
					        )
					    }  ,		)

					def  lowercase__   (						self       :  str ):
												if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None):
																			raise ValueError(
																			    """--config_overrides can't be used in combination with --config_name or --model_name_or_path""" )





@dataclass
class    __A:
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''The name of the dataset to use (via the datasets library).'''}							)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''The configuration name of the dataset to use (via the datasets library).'''}							)
					SCREAMING_SNAKE_CASE           =     field(default=UpperCAmelCase  ,		metadata={'''help''': '''The input training data file (a text file).'''}							)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''An optional input evaluation data file to evaluate the perplexity on (a text file).'''}  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''An optional input train ref data file for whole word masking in Chinese.'''}  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''An optional input validation ref data file for whole word masking in Chinese.'''}  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''Overwrite the cached training and evaluation sets'''}							)
					SCREAMING_SNAKE_CASE           =     field(
					    default=5  ,		metadata={
					        '''help''': '''The percentage of the train set used as validation set in case there\'s no validation split'''
					    }  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={
					        '''help''': (
					            '''The maximum total input sequence length after tokenization. Sequences longer '''
					            '''than this will be truncated. Default to the max input length of the model.'''
					        )
					    }  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={'''help''': '''The number of processes to use for the preprocessing.'''}  ,		)
					SCREAMING_SNAKE_CASE           =     field(
					    default=0.15  ,		metadata={'''help''': '''Ratio of tokens to mask for masked language modeling loss'''}							)
					SCREAMING_SNAKE_CASE           =     field(
					    default=UpperCAmelCase  ,		metadata={
					        '''help''': (
					            '''Whether to pad all samples to `max_seq_length`. '''
					            '''If False, will pad the samples dynamically when batching to the maximum length in the batch.'''
					        )
					    }  ,		)

					def  lowercase__   (						self       :  Tuple ):
												if self.train_file is not None:
																			lowerCamelCase_					=       self.train_file.split(""".""" )[-1]
																			assert extension in ["csv", "json", "txt"], "`train_file` should be a csv, a json or a txt file."
												if self.validation_file is not None:
																			lowerCamelCase_					=       self.validation_file.split(""".""" )[-1]
																			assert extension in ["csv", "json", "txt"], "`validation_file` should be a csv, a json or a txt file."



def   __lowerCAmelCase    (				UpperCAmelCase__					:   Any      ,					UpperCAmelCase__					:   Tuple							)	->     Dict:
							with open(UpperCAmelCase__      ,					"""r"""      ,					encoding="""utf-8"""							) as f:
														lowerCamelCase_					=       [json.loads(UpperCAmelCase__							) for line in f.read().splitlines() if (len(UpperCAmelCase__							) > 0 and not line.isspace())]
							assert len(UpperCAmelCase__							) == len(UpperCAmelCase__							)

							lowerCamelCase_					=       {c: dataset[c] for c in dataset.column_names}
							lowerCamelCase_					=       refs
							return Dataset.from_dict(UpperCAmelCase__							)



def   __lowerCAmelCase    (				)	->     Any:
							# See all possible arguments in src/transformers/training_args.py
							# or by passing the --help flag to this script.
							# We now keep distinct sets of args, for a cleaner separation of concerns.

							lowerCamelCase_					=       HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)							)
							if len(sys.argv							) == 2 and sys.argv[1].endswith(""".json"""							):
														# If we pass only one argument to the script and it's the path to a json file,
														# let's parse it to get our arguments.
														lowerCamelCase_	, lowerCamelCase_	, lowerCamelCase_					=       parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]							)							)
							else:
														lowerCamelCase_	, lowerCamelCase_	, lowerCamelCase_					=       parser.parse_args_into_dataclasses()

							# Detecting last checkpoint.
							lowerCamelCase_					=       None
							if os.path.isdir(training_args.output_dir							) and training_args.do_train and not training_args.overwrite_output_dir:
														lowerCamelCase_					=       get_last_checkpoint(training_args.output_dir							)
														if last_checkpoint is None and len(os.listdir(training_args.output_dir							)							) > 0:
																					raise ValueError(
																					    F'''Output directory ({training_args.output_dir}) already exists and is not empty. '''
																					    """Use --overwrite_output_dir to overcome."""							)
														elif last_checkpoint is not None:
																					logger.info(
																					    F'''Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change '''
																					    """the `--output_dir` or add `--overwrite_output_dir` to train from scratch."""							)

    # Setup logging
							logging.basicConfig(
							    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s"""      ,					datefmt="""%m/%d/%Y %H:%M:%S"""      ,					handlers=[logging.StreamHandler(sys.stdout							)]      ,					)
							logger.setLevel(logging.INFO if is_main_process(training_args.local_rank							) else logging.WARN							)

							# Log on each process the small summary:
							logger.warning(
							    F'''Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}'''
							    + F'''distributed training: {bool(training_args.local_rank != -1							)}, 16-bits training: {training_args.fpaa}'''							)
							# Set the verbosity to info of the Transformers logger (on main process only):
							if is_main_process(training_args.local_rank							):
														transformers.utils.logging.set_verbosity_info()
														transformers.utils.logging.enable_default_handler()
														transformers.utils.logging.enable_explicit_format()
							logger.info("""Training/evaluation parameters %s"""      ,					UpperCAmelCase__							)

							# Set seed before initializing model.
							set_seed(training_args.seed							)

							# Get the datasets: you can either provide your own CSV/JSON/TXT training and evaluation files (see below)
							# or just provide the name of one of the public datasets available on the hub at https://huggingface.co/datasets/
							# (the dataset will be downloaded automatically from the datasets Hub).
							#
							# For CSV/JSON files, this script will use the column called 'text' or the first column if no column called
							# 'text' is found. You can easily tweak this behavior (see below).
							#
							# In distributed training, the load_dataset function guarantee that only one local process can concurrently
							# download the dataset.
							if data_args.dataset_name is not None:
														# Downloading and loading a dataset from the hub.
														lowerCamelCase_					=       load_dataset(data_args.dataset_name      ,					data_args.dataset_config_name							)
														if "validation" not in datasets.keys():
																					lowerCamelCase_					=       load_dataset(
																					    data_args.dataset_name      ,					data_args.dataset_config_name      ,					split=F'''train[:{data_args.validation_split_percentage}%]'''      ,					)
																					lowerCamelCase_					=       load_dataset(
																					    data_args.dataset_name      ,					data_args.dataset_config_name      ,					split=F'''train[{data_args.validation_split_percentage}%:]'''      ,					)
							else:
														lowerCamelCase_					=       {}
														if data_args.train_file is not None:
																					lowerCamelCase_					=       data_args.train_file
														if data_args.validation_file is not None:
																					lowerCamelCase_					=       data_args.validation_file
														lowerCamelCase_					=       data_args.train_file.split("""."""							)[-1]
														if extension == "txt":
																					lowerCamelCase_					=       """text"""
														lowerCamelCase_					=       load_dataset(UpperCAmelCase__      ,					data_files=UpperCAmelCase__							)
							# See more about loading any type of standard or custom dataset (from files, python dict, pandas DataFrame, etc) at
							# https://huggingface.co/docs/datasets/loading_datasets.html.

							# Load pretrained model and tokenizer
							#
							# Distributed training:
							# The .from_pretrained methods guarantee that only one local process can concurrently
							# download model & vocab.
							lowerCamelCase_					=       {
							    """cache_dir""": model_args.cache_dir,
							    """revision""": model_args.model_revision,
							    """use_auth_token""": True if model_args.use_auth_token else None,
							}
							if model_args.config_name:
														lowerCamelCase_					=       AutoConfig.from_pretrained(model_args.config_name      ,					**UpperCAmelCase__							)
							elif model_args.model_name_or_path:
														lowerCamelCase_					=       AutoConfig.from_pretrained(model_args.model_name_or_path      ,					**UpperCAmelCase__							)
							else:
														lowerCamelCase_					=       CONFIG_MAPPING[model_args.model_type]()
														logger.warning("""You are instantiating a new config instance from scratch."""							)
														if model_args.config_overrides is not None:
																					logger.info(F'''Overriding config: {model_args.config_overrides}'''							)
																					config.update_from_string(model_args.config_overrides							)
																					logger.info(F'''New config: {config}'''							)

							lowerCamelCase_					=       {
							    """cache_dir""": model_args.cache_dir,
							    """use_fast""": model_args.use_fast_tokenizer,
							    """revision""": model_args.model_revision,
							    """use_auth_token""": True if model_args.use_auth_token else None,
							}
							if model_args.tokenizer_name:
														lowerCamelCase_					=       AutoTokenizer.from_pretrained(model_args.tokenizer_name      ,					**UpperCAmelCase__							)
							elif model_args.model_name_or_path:
														lowerCamelCase_					=       AutoTokenizer.from_pretrained(model_args.model_name_or_path      ,					**UpperCAmelCase__							)
							else:
														raise ValueError(
														    """You are instantiating a new tokenizer from scratch. This is not supported by this script."""
														    """You can do it from another script, save it, and load it from here, using --tokenizer_name."""							)

							if model_args.model_name_or_path:
														lowerCamelCase_					=       AutoModelForMaskedLM.from_pretrained(
														    model_args.model_name_or_path      ,					from_tf=bool(""".ckpt""" in model_args.model_name_or_path							)      ,					config=UpperCAmelCase__      ,					cache_dir=model_args.cache_dir      ,					revision=model_args.model_revision      ,					use_auth_token=True if model_args.use_auth_token else None      ,					)
							else:
														logger.info("""Training new model from scratch"""							)
														lowerCamelCase_					=       AutoModelForMaskedLM.from_config(UpperCAmelCase__							)

							model.resize_token_embeddings(len(UpperCAmelCase__							)							)

							# Preprocessing the datasets.
							# First we tokenize all the texts.
							if training_args.do_train:
														lowerCamelCase_					=       datasets["""train"""].column_names
							else:
														lowerCamelCase_					=       datasets["""validation"""].column_names
							lowerCamelCase_					=       """text""" if """text""" in column_names else column_names[0]

							lowerCamelCase_					=       """max_length""" if data_args.pad_to_max_length else False

							def tokenize_function(UpperCAmelCase__					:   Any							):
														# Remove empty lines
														lowerCamelCase_					=       [line for line in examples["""text"""] if len(UpperCAmelCase__							) > 0 and not line.isspace()]
														return tokenizer(examples["""text"""]      ,					padding=UpperCAmelCase__      ,					truncation=UpperCAmelCase__      ,					max_length=data_args.max_seq_length							)

							lowerCamelCase_					=       datasets.map(
							    UpperCAmelCase__      ,					batched=UpperCAmelCase__      ,					num_proc=data_args.preprocessing_num_workers      ,					remove_columns=[text_column_name]      ,					load_from_cache_file=not data_args.overwrite_cache      ,					)

							# Add the chinese references if provided
							if data_args.train_ref_file is not None:
														lowerCamelCase_					=       add_chinese_references(tokenized_datasets["""train"""]      ,					data_args.train_ref_file							)
							if data_args.validation_ref_file is not None:
														lowerCamelCase_					=       add_chinese_references(
														    tokenized_datasets["""validation"""]      ,					data_args.validation_ref_file							)
							# If we have ref files, need to avoid it removed by trainer
							lowerCamelCase_					=       data_args.train_ref_file or data_args.validation_ref_file
							if has_ref:
														lowerCamelCase_					=       False

							# Data collator
							# This one will take care of randomly masking the tokens.
							lowerCamelCase_					=       DataCollatorForWholeWordMask(tokenizer=UpperCAmelCase__      ,					mlm_probability=data_args.mlm_probability							)

							# Initialize our Trainer
							lowerCamelCase_					=       Trainer(
							    model=UpperCAmelCase__      ,					args=UpperCAmelCase__      ,					train_dataset=tokenized_datasets["""train"""] if training_args.do_train else None      ,					eval_dataset=tokenized_datasets["""validation"""] if training_args.do_eval else None      ,					tokenizer=UpperCAmelCase__      ,					data_collator=UpperCAmelCase__      ,					)

							# Training
							if training_args.do_train:
														if last_checkpoint is not None:
																					lowerCamelCase_					=       last_checkpoint
														elif model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path							):
																					lowerCamelCase_					=       model_args.model_name_or_path
														else:
																					lowerCamelCase_					=       None
														lowerCamelCase_					=       trainer.train(resume_from_checkpoint=UpperCAmelCase__							)
														trainer.save_model()  # Saves the tokenizer too for easy upload

														lowerCamelCase_					=       os.path.join(training_args.output_dir      ,					"""train_results.txt"""							)
														if trainer.is_world_process_zero():
																					with open(UpperCAmelCase__      ,					"""w"""							) as writer:
																												logger.info("""***** Train results *****"""							)
																												for key, value in sorted(train_result.metrics.items()							):
																																			logger.info(F'''  {key} = {value}'''							)
																																			writer.write(F'''{key} = {value}\n'''							)

            # Need to save the state, since Trainer.save_model saves only the tokenizer with the model
																					trainer.state.save_to_json(os.path.join(training_args.output_dir      ,					"""trainer_state.json"""							)							)

    # Evaluation
							lowerCamelCase_					=       {}
							if training_args.do_eval:
														logger.info("""*** Evaluate ***"""							)

														lowerCamelCase_					=       trainer.evaluate()

														lowerCamelCase_					=       math.exp(eval_output["""eval_loss"""]							)
														lowerCamelCase_					=       perplexity

														lowerCamelCase_					=       os.path.join(training_args.output_dir      ,					"""eval_results_mlm_wwm.txt"""							)
														if trainer.is_world_process_zero():
																					with open(UpperCAmelCase__      ,					"""w"""							) as writer:
																												logger.info("""***** Eval results *****"""							)
																												for key, value in sorted(results.items()							):
																																			logger.info(F'''  {key} = {value}'''							)
																																			writer.write(F'''{key} = {value}\n'''							)

							return results



def   __lowerCAmelCase    (				UpperCAmelCase__					:   Optional[Any]							)	->     Optional[Any]:
							# For xla_spawn (TPUs)
							main()


if __name__ == "__main__":
		main()





