




import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


UpperCamelCase__              =						get_logger(__name__)


if TYPE_CHECKING:
							from .builder import DatasetBuilder






def 					snake_case					(							A						,   A = None			)							->							Union[str, Any]:




	'''simple docstring'''


	lowerCamelCase_							:		Optional[int]							   =							importlib.import_module(A			)

	# TODO(QL): always update the module to add subsequent new authentication
	if hasattr(A						,   '_patched_for_streaming'			) and module._patched_for_streaming:
		return

	def wrap_auth(A			):
		@wraps(A			)
		def wrapper(*A						,   **A			):
			return function(*A						,   use_auth_token=A						,   **A			)

		lowerCamelCase_							:		Any							   =							'wrap_auth'
		return wrapper

	# open files in a streaming fashion
	patch_submodule(A						,   'open'						,   wrap_auth(A			)			).start()
	patch_submodule(A						,   'os.listdir'						,   wrap_auth(A			)			).start()
	patch_submodule(A						,   'os.walk'						,   wrap_auth(A			)			).start()
	patch_submodule(A						,   'glob.glob'						,   wrap_auth(A			)			).start()
	# allow to navigate in remote zip files
	patch_submodule(A						,   'os.path.join'						,   A			).start()
	patch_submodule(A						,   'os.path.dirname'						,   A			).start()
	patch_submodule(A						,   'os.path.basename'						,   A			).start()
	patch_submodule(A						,   'os.path.relpath'						,   A			).start()
	patch_submodule(A						,   'os.path.split'						,   A			).start()
	patch_submodule(A						,   'os.path.splitext'						,   A			).start()
	# allow checks on paths
	patch_submodule(A						,   'os.path.exists'						,   wrap_auth(A			)			).start()
	patch_submodule(A						,   'os.path.isdir'						,   wrap_auth(A			)			).start()
	patch_submodule(A						,   'os.path.isfile'						,   wrap_auth(A			)			).start()
	patch_submodule(A						,   'os.path.getsize'						,   wrap_auth(A			)			).start()
	patch_submodule(A						,   'pathlib.Path'						,   A			).start()
	# file readers
	patch_submodule(A						,   'gzip.open'						,   wrap_auth(A			)			).start()
	patch_submodule(A						,   'numpy.load'						,   wrap_auth(A			)			).start()
	patch_submodule(A						,   'pandas.read_csv'						,   wrap_auth(A			)						,   attrs=['__version__']			).start()
	patch_submodule(A						,   'pandas.read_excel'						,   wrap_auth(A			)						,   attrs=['__version__']			).start()
	patch_submodule(A						,   'scipy.io.loadmat'						,   wrap_auth(A			)						,   attrs=['__version__']			).start()
	patch_submodule(A						,   'xml.etree.ElementTree.parse'						,   wrap_auth(A			)			).start()
	patch_submodule(A						,   'xml.dom.minidom.parse'						,   wrap_auth(A			)			).start()
	lowerCamelCase_							:		str							   =							True






def 					snake_case					(							A			)							->							Optional[Any]:




	'''simple docstring'''


	extend_module_for_streaming(builder.__module__						,   use_auth_token=builder.use_auth_token			)
	# if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
	if not builder.__module__.startswith('datasets.'			):  # check that it's not a packaged builder like csv
		for imports in get_imports(inspect.getfile(builder.__class__			)			):
			if imports[0] == "internal":
				lowerCamelCase_							:		Any							   =							imports[1]
				lowerCamelCase_							:		Dict							   =							'.'.join(builder.__module__.split('.'			)[:-1] + [internal_import_name]			)
				extend_module_for_streaming(A						,   use_auth_token=builder.use_auth_token			)

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
	from .builder import DatasetBuilder

	lowerCamelCase_							:		Tuple							   =							[
	    cls.__module__
	    for cls in type(A			).__mro__[1:]  # make sure it's not the same module we've already patched
	    if issubclass(cls						,   A			) and cls.__module__ != DatasetBuilder.__module__
	]  # check it's not a standard builder from datasets.builder
	for module in parent_builder_modules:
		extend_module_for_streaming(A						,   use_auth_token=builder.use_auth_token			)







import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


_snake_case				= logging.getLogger(__name__)





@dataclass
class 					lowerCAmelCase							:
  __lowerCamelCase		=			field(
      metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}					)
  __lowerCamelCase		=			field(
      default=lowercase_							,							metadata={'help': 'Pretrained config name or path if not the same as model_name'}					)
  __lowerCamelCase		=			field(
      default=lowercase_							,							metadata={'help': 'Pretrained tokenizer name or path if not the same as model_name'}					)
  __lowerCamelCase		=			field(default=lowercase_							,							metadata={'help': 'Set this flag to use fast tokenization.'}					)
  # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
  # or just modify its tokenizer_config.json.
  __lowerCamelCase		=			field(
      default=lowercase_							,							metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}							,							)


def _A   (    ):
   # See all possible arguments in src/transformers/training_args.py
   # or by passing the --help flag to this script.
   # We now keep distinct sets of args, for a cleaner separation of concerns.

   lowercase__    =  HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)       )

   if len(sys.argv       ) == 2 and sys.argv[1].endswith(".json"       ):
      # If we pass only one argument to the script and it's the path to a json file,
      # let's parse it to get our arguments.
      lowercase__   ,							lowercase__   ,							lowercase__    =  parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]       )       )
   else:
      lowercase__   ,							lowercase__   ,							lowercase__    =  parser.parse_args_into_dataclasses()

   if (
       os.path.exists(training_args.output_dir       )
       and os.listdir(training_args.output_dir       )
       and training_args.do_train
       and not training_args.overwrite_output_dir
   ):
      raise ValueError(
          f'''Output directory ({training_args.output_dir}) already exists and is not empty. Use'''
          " --overwrite_output_dir to overcome."       )

   # Setup logging
   logging.basicConfig(
       format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"  ,					datefmt="%m/%d/%Y %H:%M:%S"  ,					level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN  ,					)
   logger.warning(
       "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"  ,					training_args.local_rank  ,					training_args.device  ,					training_args.n_gpu  ,					bool(training_args.local_rank != -1       )  ,					training_args.fpaa  ,					)
   # Set the verbosity to info of the Transformers logger (on main process only):
   if is_main_process(training_args.local_rank       ):
      transformers.utils.logging.set_verbosity_info()
      transformers.utils.logging.enable_default_handler()
      transformers.utils.logging.enable_explicit_format()
   logger.info("Training/evaluation parameters %s"  ,					__magic_name__       )

   # Prepare Question-Answering task
   # Load pretrained model and tokenizer
   #
   # Distributed training:
   # The .from_pretrained methods guarantee that only one local process can concurrently
   # download model & vocab.

   lowercase__    =  AutoConfig.from_pretrained(
       model_args.config_name if model_args.config_name else model_args.model_name_or_path  ,					cache_dir=model_args.cache_dir  ,					)
   lowercase__    =  AutoTokenizer.from_pretrained(
       model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path  ,					cache_dir=model_args.cache_dir  ,					use_fast=__magic_name__  ,					)
   lowercase__    =  AutoModelForQuestionAnswering.from_pretrained(
       model_args.model_name_or_path  ,					from_tf=bool(".ckpt" in model_args.model_name_or_path       )  ,					config=__magic_name__  ,					cache_dir=model_args.cache_dir  ,					)

   # Get datasets
   lowercase__    =  hasattr(model.config  ,					"lang2id"       )
   lowercase__    =  (
       SquadDataset(
           __magic_name__  ,					tokenizer=__magic_name__  ,					is_language_sensitive=__magic_name__  ,					cache_dir=model_args.cache_dir       )
       if training_args.do_train
       else None
   )
   lowercase__    =  (
       SquadDataset(
           __magic_name__  ,					tokenizer=__magic_name__  ,					mode="dev"  ,					is_language_sensitive=__magic_name__  ,					cache_dir=model_args.cache_dir  ,					)
       if training_args.do_eval
       else None
   )

   # Data collator
   lowercase__    =  DataCollatorWithPadding(__magic_name__  ,					pad_to_multiple_of=8       ) if training_args.fpaa else None

   # Initialize our Trainer
   lowercase__    =  Trainer(
       model=__magic_name__  ,					args=__magic_name__  ,					train_dataset=__magic_name__  ,					eval_dataset=__magic_name__  ,					data_collator=__magic_name__  ,					)

   # Training
   if training_args.do_train:
      trainer.train(
          model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path       ) else None       )
      trainer.save_model()
      # For convenience, we also re-save the tokenizer to the same directory,
      # so that you can share your model easily on huggingface.co/models =)
      if trainer.is_world_master():
         tokenizer.save_pretrained(training_args.output_dir       )


def _A   (    __magic_name__       ):
   # For xla_spawn (TPUs)
   main()


if __name__ == "__main__":
  main()






