




"""simple docstring"""







import argparse
import dataclasses
import json
import logging
import os
import shutil
from typing import List, Optional

import datasets
from accelerate import Accelerator
from datasets import load_dataset
from finetuning import finetune
from tqdm.auto import tqdm

import transformers
from transformers import AutoConfig, set_seed
from transformers.trainer_utils import IntervalStrategy


__A   :  Optional[int]					=      logging.getLogger(__name__)

__A   :  List[Any]					=      'pytorch_model.bin'




@dataclasses.dataclass
class       lowerCAmelCase__       :


				"""simple docstring"""





				__UpperCAmelCase :     str       =      dataclasses.field(
				    metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models."}    )
				__UpperCAmelCase :     Optional[str]       =      dataclasses.field(
				    default=lowerCAmelCase_  ,			metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co."}  ,			)





@dataclasses.dataclass
class       lowerCAmelCase__       :


				"""simple docstring"""





				__UpperCAmelCase :     str       =      dataclasses.field(metadata={"help": "A csv or a json file containing the training data."}    )
				__UpperCAmelCase :     str       =      dataclasses.field(metadata={"help": "A csv or a json file containing the data to predict on."}    )
				__UpperCAmelCase :     Optional[str]       =      dataclasses.field(
				    default=lowerCAmelCase_  ,			metadata={"help": "A csv or a json file containing the validation data."}    )
				__UpperCAmelCase :     Optional[str]       =      dataclasses.field(
				    default=lowerCAmelCase_  ,			metadata={"help": "The name of the task to train on."}  ,			)
				__UpperCAmelCase :     Optional[List[str]]       =      dataclasses.field(
				    default=lowerCAmelCase_  ,			metadata={"help": "The list of labels for the task."}    )





@dataclasses.dataclass
class       lowerCAmelCase__       :


				"""simple docstring"""





				__UpperCAmelCase :     str       =      dataclasses.field(
				    metadata={"help": "The output directory where the model predictions and checkpoints will be written."}    )
				__UpperCAmelCase :     Optional[str]       =      dataclasses.field(
				    default="accuracy"  ,			metadata={"help": "The evaluation metric used for the task."}    )
				__UpperCAmelCase :     Optional[str]       =      dataclasses.field(
				    default="no"  ,			metadata={
				        "help": "The evaluation strategy to adopt during training. Possible values are: [\"no\", \"step\", \"epoch]"
				    }  ,			)
				__UpperCAmelCase :     Optional[int]       =      dataclasses.field(
				    default=10  ,			metadata={"help": "Number of evaluation calls with no improvement after which training will be stopped."}  ,			)
				__UpperCAmelCase :     Optional[float]       =      dataclasses.field(
				    default=0.0  ,			metadata={
				        "help": "How much the specified evaluation metric must improve to satisfy early stopping conditions."
				    }  ,			)
				__UpperCAmelCase :     Optional[bool]       =      dataclasses.field(
				    default=lowerCAmelCase_  ,			metadata={"help": "Whether to filter the pseudo-labeled data based on the confidence score."}  ,			)
				__UpperCAmelCase :     Optional[bool]       =      dataclasses.field(
				    default=lowerCAmelCase_  ,			metadata={"help": "Whether to filter the pseudo-labeled data based on the validation performance."}  ,			)
				__UpperCAmelCase :     Optional[bool]       =      dataclasses.field(
				    default=lowerCAmelCase_  ,			metadata={"help": "Whether to fine-tune on labeled data after pseudo training."}  ,			)
				__UpperCAmelCase :     Optional[float]       =      dataclasses.field(
				    default=0.0  ,			metadata={"help": "Confidence threshold for pseudo-labeled data filtering."}  ,			)
				__UpperCAmelCase :     Optional[int]       =      dataclasses.field(
				    default=100  ,			metadata={"help": "Number of evaluation calls with no improvement after which training will be stopped."}  ,			)
				__UpperCAmelCase :     Optional[int]       =      dataclasses.field(
				    default=lowerCAmelCase_  ,			metadata={"help": "Random seed for initialization."}  ,			)





def 						snake_case__       (					_lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase				)			->Dict:






		"""simple docstring"""


		__lowercase				:		Any												=				datasets.concatenate_datasets([infer_input, infer_output],  axis=1				)

		if args.do_filter_by_confidence:
				__lowercase				:		Any												=				dataset.filter(lambda _lowerCamelCase				: example["probability"] > args.confidence_threshold				)

		if args.do_filter_by_val_performance:
				assert eval_result >= 0.0 and eval_result <= 1.0
				__lowercase				:		Union[str, Any]												=				int(eval_result * len(_lowerCamelCase				)				)
				print(_lowerCamelCase				)
				__lowercase				:		Any												=				dataset.sort("probability",  reverse=_lowerCamelCase				)
				__lowercase				:		Optional[int]												=				dataset.select(range(_lowerCamelCase				)				)

		__lowercase				:		Union[str, Any]												=				dataset.remove_columns(["label", "probability"]				)
		__lowercase				:		List[Any]												=				dataset.rename_column("prediction",  "label"				)
		__lowercase				:		List[str]												=				dataset.map(lambda _lowerCamelCase				: {"label": idalabel[example["label"]]}				)
		__lowercase				:		Union[str, Any]												=				dataset.shuffle(seed=args.seed				)

		__lowercase				:		int												=				os.path.join(_lowerCamelCase,  F'train_pseudo.{args.data_file_extension}'				)
		if args.data_file_extension == "csv":
				dataset.to_csv(_lowerCamelCase,  index=_lowerCamelCase				)
		else:
				dataset.to_json(_lowerCamelCase				)





def 						snake_case__       (					_lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase,  **_lowerCamelCase				)			->str:






		"""simple docstring"""


		__lowercase				:		List[str]												=				Accelerator()
		# Make one log on every process with the configuration for debugging.
		logging.basicConfig(
		    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",  datefmt="%m/%d/%Y %H:%M:%S",  level=logging.INFO,  )
		logger.info(accelerator.state				)

		# Setup logging, we only want one process per machine to log things on the
		# screen. accelerator.is_local_main_process is only True for one process per
		# machine.
		logger.setLevel(logging.INFO if accelerator.is_local_main_process else logging.ERROR				)

		if accelerator.is_local_main_process:
				datasets.utils.logging.set_verbosity_warning()
				transformers.utils.logging.set_verbosity_info()
		else:
				datasets.utils.logging.set_verbosity_error()
				transformers.utils.logging.set_verbosity_error()

		__lowercase				:		List[Any]												=				STModelArguments(model_name_or_path=_lowerCamelCase				)
		__lowercase				:		List[Any]												=				STDataArguments(train_file=_lowerCamelCase,  infer_file=_lowerCamelCase				)
		__lowercase				:		Optional[Any]												=				STTrainingArguments(output_dir=_lowerCamelCase				)
		__lowercase				:		Dict												=				argparse.Namespace()

		for arg_class in (model_args, data_args, training_args):
				for key, value in vars(_lowerCamelCase				).items():
						setattr(_lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase				)

		for key, value in kwargs.items():
				if hasattr(_lowerCamelCase,  _lowerCamelCase				):
						setattr(_lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase				)

    # Sanity checks
		__lowercase				:		Optional[int]												=				{}
		__lowercase				:		int												=				None

		# You need to provide the training data and the data to predict on
		assert args.train_file is not None
		assert args.infer_file is not None
		__lowercase				:		Optional[int]												=				args.train_file
		__lowercase				:		Optional[Any]												=				args.infer_file

		if args.evaluation_strategy != IntervalStrategy.NO.value:
				assert args.eval_file is not None
				__lowercase				:		Tuple												=				args.eval_file

		for key in data_files:
				__lowercase				:		List[Any]												=				data_files[key].split("."				)[-1]
				assert extension in ["csv", "json"], F'`{key}_file` should be a csv or a json file.'
				if args.data_file_extension is None:
						__lowercase				:		Optional[Any]												=				extension
				else:
						assert extension == args.data_file_extension, F'`{key}_file` should be a {args.data_file_extension} file`.'

		assert (
		    args.eval_metric in datasets.list_metrics()
		), F'{args.eval_metric} not in the list of supported metrics {datasets.list_metrics()}.'

		# If passed along, set the training seed now.
		if args.seed is not None:
				set_seed(args.seed				)

		logger.info("Creating the initial data directory for self-training..."				)
		__lowercase				:		int												=				F'{args.output_dir}/self-train_iter-{{}}'.format
		__lowercase				:		Optional[int]												=				data_dir_format(0				)

		if accelerator.is_main_process:
				if args.output_dir is not None:
						os.makedirs(args.output_dir,  exist_ok=_lowerCamelCase				)
						os.makedirs(_lowerCamelCase,  exist_ok=_lowerCamelCase				)
		accelerator.wait_for_everyone()

		__lowercase				:		Optional[Any]												=				None
		__lowercase				:		List[str]												=				None
		__lowercase				:		int												=				0
		__lowercase				:		List[Any]												=				False
		# Show the progress bar
		__lowercase				:		List[Any]												=				tqdm(range(args.max_selftrain_iterations				),  disable=not accelerator.is_local_main_process				)

		# Self-train
		for iteration in range(0,  int(args.max_selftrain_iterations				)				):
				__lowercase				:		Dict												=				data_dir_format(_lowerCamelCase				)
				assert os.path.exists(_lowerCamelCase				)

				# Stage 1: initial fine-tuning for iteration = 0 or pseudo-training for
				# iteration > 0
				__lowercase				:		int												=				os.path.join(_lowerCamelCase,  "stage-1"				)
				__lowercase				:		int												=				{
				    "accelerator": accelerator,
				    "model_name_or_path": args.model_name_or_path,
				    "cache_dir": args.cache_dir,
				    "do_train": True,
				    "train_file": data_files["train"] if iteration == 0 else data_files["train_pseudo"],
				    "do_eval": True if args.eval_file is not None else False,
				    "eval_file": data_files["eval"],
				    "do_predict": True,
				    "infer_file": data_files["infer"],
				    "task_name": args.task_name,
				    "label_list": args.label_list,
				    "output_dir": current_output_dir,
				    "eval_metric": args.eval_metric,
				    "evaluation_strategy": args.evaluation_strategy,
				    "early_stopping_patience": args.early_stopping_patience,
				    "early_stopping_threshold": args.early_stopping_threshold,
				    "seed": args.seed,
				}
				# Add additional training arguments
				for key, value in kwargs.items():
						if key not in arguments_dict and not hasattr(_lowerCamelCase,  _lowerCamelCase				):
								arguments_dict.update({key: value}				)

				__lowercase				:		str												=				os.path.join(_lowerCamelCase,  "best-checkpoint",  _lowerCamelCase				)
				if os.path.exists(_lowerCamelCase				):
						logger.info(
						    "Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 1.",  _lowerCamelCase,  _lowerCamelCase,  )
				else:
						logger.info("***** Running self-training: iteration: %d, stage: 1 *****",  _lowerCamelCase				)
						finetune(**_lowerCamelCase				)
						accelerator.wait_for_everyone()
						assert os.path.exists(_lowerCamelCase				)
						logger.info("Self-training job completed: iteration: %d, stage: 1.",  _lowerCamelCase				)

				if iteration > 0 and args.finetune_on_labeled_data:
						# Stage 2 (optional): fine-tuning on the original labeled data
						__lowercase				:		Optional[int]												=				os.path.join(_lowerCamelCase,  "best-checkpoint"				)
						__lowercase				:		Optional[Any]												=				os.path.join(_lowerCamelCase,  "stage-2"				)
						# Update arguments_dict
						__lowercase				:		Optional[int]												=				model_path
						__lowercase				:		int												=				data_files["train"]
						__lowercase				:		Optional[Any]												=				current_output_dir

						__lowercase				:		int												=				os.path.join(_lowerCamelCase,  "best-checkpoint",  _lowerCamelCase				)
						if os.path.exists(_lowerCamelCase				):
								logger.info(
								    "Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 2.",  _lowerCamelCase,  _lowerCamelCase,  )
						else:
								logger.info("***** Running self-training: iteration: %d, stage: 2 *****",  _lowerCamelCase				)
								finetune(**_lowerCamelCase				)
								accelerator.wait_for_everyone()
								assert os.path.exists(_lowerCamelCase				)
								logger.info("Self-training job completed: iteration: %d, stage: 2.",  _lowerCamelCase				)

				__lowercase				:		Any												=				iteration
				__lowercase				:		List[str]												=				data_dir_format(iteration + 1				)

				__lowercase				:		Optional[Any]												=				AutoConfig.from_pretrained(os.path.join(_lowerCamelCase,  "best-checkpoint"				)				)
				__lowercase				:		int												=				config.idalabel
				__lowercase				:		Union[str, Any]												=				os.path.join(_lowerCamelCase,  "eval_results_best-checkpoint.json"				)
				__lowercase				:		int												=				os.path.join(_lowerCamelCase,  "test_results_best-checkpoint.json"				)
				assert os.path.exists(_lowerCamelCase				)

				with open(_lowerCamelCase,  "r"				) as f:
						__lowercase				:		Dict												=				float(json.load(_lowerCamelCase				)[args.eval_metric]				)
				__lowercase				:		Optional[Any]												=				os.path.join(_lowerCamelCase,  "infer_output_best-checkpoint.csv"				)
				assert os.path.exists(_lowerCamelCase				)
				# Loading the dataset from local csv or json files.
				__lowercase				:		List[Any]												=				load_dataset(args.data_file_extension,  data_files={"data": data_files["infer"]}				)["data"]
				__lowercase				:		List[str]												=				load_dataset("csv",  data_files={"data": infer_output_file}				)["data"]

				if accelerator.is_main_process:
						os.makedirs(_lowerCamelCase,  exist_ok=_lowerCamelCase				)
						shutil.copy(_lowerCamelCase,  os.path.join(_lowerCamelCase,  F'eval_results_iter-{iteration}.json'				)				)
						if os.path.exists(_lowerCamelCase				):
								shutil.copy(_lowerCamelCase,  os.path.join(_lowerCamelCase,  F'test_results_iter-{iteration}.json'				)				)
						create_pseudo_labeled_data(_lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase,  _lowerCamelCase				)
				accelerator.wait_for_everyone()

				__lowercase				:		List[str]												=				os.path.join(_lowerCamelCase,  F'train_pseudo.{args.data_file_extension}'				)

				if args.evaluation_strategy != IntervalStrategy.NO.value:
						__lowercase				:		List[str]												=				eval_result

						if best_iteration is None:
								__lowercase				:		Dict												=				new_iteration
								__lowercase				:		str												=				new_eval_result
						else:
								if new_eval_result - best_eval_result > args.early_stopping_threshold:
										__lowercase				:		int												=				new_iteration
										__lowercase				:		Optional[Any]												=				new_eval_result
										__lowercase				:		Union[str, Any]												=				0
								else:
										if new_eval_result == best_eval_result:
												__lowercase				:		str												=				new_iteration
												__lowercase				:		Tuple												=				new_eval_result
										early_stopping_patience_counter += 1

								if early_stopping_patience_counter >= args.early_stopping_patience:
										__lowercase				:		Dict												=				True

				progress_bar.update(1				)

				if should_training_stop:
						break

		if best_iteration is not None:
				# Save the best iteration
				logger.info("Best iteration: %d",  _lowerCamelCase				)
				logger.info("Best evaluation result: %s = %f",  args.eval_metric,  _lowerCamelCase				)
				accelerator.wait_for_everyone()
				if accelerator.is_main_process:
						shutil.copy(
						    os.path.join(_lowerCamelCase,  F'eval_results_iter-{iteration}.json'				),  os.path.join(_lowerCamelCase,  "eval_results_best-iteration.json"				),  )
		else:
				# Assume that the last iteration is the best
				logger.info("Best iteration: %d",  args.max_selftrain_iterations - 1				)
				logger.info("Best evaluation result: %s = %f",  args.eval_metric,  _lowerCamelCase				)
				accelerator.wait_for_everyone()
				if accelerator.is_main_process:
						shutil.copy(
						    os.path.join(_lowerCamelCase,  F'eval_results_iter-{args.max_selftrain_iterations - 1}.json'				),  os.path.join(_lowerCamelCase,  "eval_results_best-iteration.json"				),  )










"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A   :  str					=      logging.get_logger(__name__)

__A   :  str					=      {
    'uclanlp/visualbert-vqa': 'https://huggingface.co/uclanlp/visualbert-vqa/resolve/main/config.json',
    'uclanlp/visualbert-vqa-pre': 'https://huggingface.co/uclanlp/visualbert-vqa-pre/resolve/main/config.json',
    'uclanlp/visualbert-vqa-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vqa-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-vcr': 'https://huggingface.co/uclanlp/visualbert-vcr/resolve/main/config.json',
    'uclanlp/visualbert-vcr-pre': 'https://huggingface.co/uclanlp/visualbert-vcr-pre/resolve/main/config.json',
    'uclanlp/visualbert-vcr-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-vcr-coco-pre/resolve/main/config.json'
    ),
    'uclanlp/visualbert-nlvr2': 'https://huggingface.co/uclanlp/visualbert-nlvr2/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-pre': 'https://huggingface.co/uclanlp/visualbert-nlvr2-pre/resolve/main/config.json',
    'uclanlp/visualbert-nlvr2-coco-pre': (
        'https://huggingface.co/uclanlp/visualbert-nlvr2-coco-pre/resolve/main/config.json'
    )
    # See all VisualBERT models at https://huggingface.co/models?filter=visual_bert
}




class       lowerCAmelCase__       (			lowerCAmelCase_    ):


				"""simple docstring"""





				__UpperCAmelCase :     Tuple       =      "visual_bert"



				def __init__(     self					:	List[Any]    ,   lowercase__					:	Dict=3_0_5_2_2    ,   lowercase__					:	Any=7_6_8    ,   lowercase__					:	Dict=5_1_2    ,   lowercase__					:	Tuple=1_2    ,   lowercase__					:	Any=1_2    ,   lowercase__					:	Union[str, Any]=3_0_7_2    ,   lowercase__					:	Tuple="gelu"    ,   lowercase__					:	Dict=0.1    ,   lowercase__					:	Tuple=0.1    ,   lowercase__					:	str=5_1_2    ,   lowercase__					:	str=2    ,   lowercase__					:	Any=0.0_2    ,   lowercase__					:	Any=1e-12    ,   lowercase__					:	Any=False    ,   lowercase__					:	List[str]=True    ,   lowercase__					:	Dict=1    ,   lowercase__					:	List[Any]=0    ,   lowercase__					:	Tuple=2    ,   **lowercase__					:	Any    ,   ):
						super().__init__(pad_token_id=lowercase__    ,   bos_token_id=lowercase__    ,   eos_token_id=lowercase__    ,   **lowercase__						)

						__lowercase				:		Optional[int]												=				vocab_size
						__lowercase				:		Any												=				max_position_embeddings
						__lowercase				:		Optional[Any]												=				hidden_size
						__lowercase				:		List[str]												=				visual_embedding_dim
						__lowercase				:		Dict												=				num_hidden_layers
						__lowercase				:		Any												=				num_attention_heads
						__lowercase				:		str												=				intermediate_size
						__lowercase				:		Dict												=				hidden_act
						__lowercase				:		Any												=				hidden_dropout_prob
						__lowercase				:		Optional[int]												=				attention_probs_dropout_prob
						__lowercase				:		Dict												=				initializer_range
						__lowercase				:		str												=				type_vocab_size
						__lowercase				:		List[Any]												=				layer_norm_eps
						__lowercase				:		Tuple												=				bypass_transformer
						__lowercase				:		List[str]												=				special_visual_initialize
