


import logging
import os
import random
import sys
from dataclasses import dataclass, field
from typing import Optional

import datasets
import evaluate
import numpy as np
from datasets import load_dataset

import transformers
from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    DataCollatorWithPadding,
    EvalPrediction,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    default_data_collator,
    set_seed,
)
from transformers.trainer_utils import get_last_checkpoint
from transformers.utils import check_min_version, send_example_telemetry
from transformers.utils.versions import require_version


# Will error if the minimal version of Transformers is not installed. Remove at your own risks.
check_min_version('4.31.0')

require_version('datasets>=1.8.0', 'To fix: pip install -r examples/pytorch/text-classification/requirements.txt')

lowerCAmelCase__			:      List[Any]								=logging.getLogger(__name__)







@dataclass
class 	__lowercase    :





							"""simple docstring"""







							_UpperCAmelCase      =						field(
							    default=1_2_8	,       metadata={
							        """help""": (
							            """The maximum total input sequence length after tokenization. Sequences longer """
							            """than this will be truncated, sequences shorter will be padded."""
							        )
							    }	,       )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={"""help""": """Overwrite the cached preprocessed datasets or not."""}  )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={
							        """help""": (
							            """Whether to pad all samples to `max_seq_length`. """
							            """If False, will pad the samples dynamically when batching to the maximum length in the batch."""
							        )
							    }	,       )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={
							        """help""": (
							            """For debugging purposes or quicker training, truncate the number of training examples to this """
							            """value if set."""
							        )
							    }	,       )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={
							        """help""": (
							            """For debugging purposes or quicker training, truncate the number of evaluation examples to this """
							            """value if set."""
							        )
							    }	,       )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={
							        """help""": (
							            """For debugging purposes or quicker training, truncate the number of prediction examples to this """
							            """value if set."""
							        )
							    }	,       )







@dataclass
class 	__lowercase    :





							"""simple docstring"""







							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}  )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={"""help""": """Evaluation language. Also train language if `train_language` is set to None."""}  )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={"""help""": """Train language if it is different from the evaluation language."""}  )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}  )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}  )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}	,       )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={"""help""": """arg to indicate if tokenizer should do lower case in AutoTokenizer.from_pretrained()"""}	,       )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={"""help""": """Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."""}	,       )
							_UpperCAmelCase      =						field(
							    default="""main"""	,       metadata={"""help""": """The specific model version to use (can be a branch name, tag name or commit id)."""}	,       )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={
							        """help""": (
							            """Will use the token generated when running `huggingface-cli login` (necessary to use this script """
							            """with private models)."""
							        )
							    }	,       )
							_UpperCAmelCase      =						field(
							    default=__SCREAMING_SNAKE_CASE	,       metadata={"""help""": """Will enable to load a pretrained model whose head dimensions are different."""}	,       )







def 						a__			(				):
			# See all possible arguments in src/transformers/training_args.py
			# or by passing the --help flag to this script.
			# We now keep distinct sets of args, for a cleaner separation of concerns.

			SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)				)
			SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		:     List[str]					=					parser.parse_args_into_dataclasses()

			# Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The
			# information sent is the one passed as arguments along with your Python/PyTorch versions.
			send_example_telemetry('run_xnli',      A__				)

			# Setup logging
			logging.basicConfig(
			    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',      datefmt='%m/%d/%Y %H:%M:%S',      handlers=[logging.StreamHandler(sys.stdout				)],      )

			if training_args.should_log:
						# The default of training_args.log_level is passive, so we set log level at info here to have that default.
						transformers.utils.logging.set_verbosity_info()

			SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					training_args.get_process_log_level()
			logger.setLevel(A__				)
			datasets.utils.logging.set_verbosity(A__				)
			transformers.utils.logging.set_verbosity(A__				)
			transformers.utils.logging.enable_default_handler()
			transformers.utils.logging.enable_explicit_format()

			# Log on each process the small summary:
			logger.warning(
			    F'''Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}'''
			    + F'''distributed training: {bool(training_args.local_rank != -1				)}, 16-bits training: {training_args.fpaa}'''				)
			logger.info(F'''Training/evaluation parameters {training_args}'''				)

			# Detecting last checkpoint.
			SCREAMING_SNAKE_CASE_		:     int					=					None
			if os.path.isdir(training_args.output_dir				) and training_args.do_train and not training_args.overwrite_output_dir:
						SCREAMING_SNAKE_CASE_		:     List[Any]					=					get_last_checkpoint(training_args.output_dir				)
						if last_checkpoint is None and len(os.listdir(training_args.output_dir				)				) > 0:
									raise ValueError(
									    F'''Output directory ({training_args.output_dir}) already exists and is not empty. '''
									    'Use --overwrite_output_dir to overcome.'				)
						elif last_checkpoint is not None:
									logger.info(
									    F'''Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change '''
									    'the `--output_dir` or add `--overwrite_output_dir` to train from scratch.'				)

    # Set seed before initializing model.
			set_seed(training_args.seed				)

			# In distributed training, the load_dataset function guarantees that only one local process can concurrently
			# download the dataset.
			# Downloading and loading xnli dataset from the hub.
			if training_args.do_train:
						if model_args.train_language is None:
									SCREAMING_SNAKE_CASE_		:     List[Any]					=					load_dataset(
									    'xnli',      model_args.language,      split='train',      cache_dir=model_args.cache_dir,      use_auth_token=True if model_args.use_auth_token else None,      )
						else:
									SCREAMING_SNAKE_CASE_		:     List[Any]					=					load_dataset(
									    'xnli',      model_args.train_language,      split='train',      cache_dir=model_args.cache_dir,      use_auth_token=True if model_args.use_auth_token else None,      )
						SCREAMING_SNAKE_CASE_		:     Any					=					train_dataset.features['label'].names

			if training_args.do_eval:
						SCREAMING_SNAKE_CASE_		:     List[str]					=					load_dataset(
						    'xnli',      model_args.language,      split='validation',      cache_dir=model_args.cache_dir,      use_auth_token=True if model_args.use_auth_token else None,      )
						SCREAMING_SNAKE_CASE_		:     List[str]					=					eval_dataset.features['label'].names

			if training_args.do_predict:
						SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					load_dataset(
						    'xnli',      model_args.language,      split='test',      cache_dir=model_args.cache_dir,      use_auth_token=True if model_args.use_auth_token else None,      )
						SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					predict_dataset.features['label'].names

			# Labels
			SCREAMING_SNAKE_CASE_		:     List[Any]					=					len(A__				)

			# Load pretrained model and tokenizer
			# In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently
			# download model & vocab.
			SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					AutoConfig.from_pretrained(
			    model_args.config_name if model_args.config_name else model_args.model_name_or_path,      num_labels=A__,      idalabel={str(A__				): label for i, label in enumerate(A__				)},      labelaid={label: i for i, label in enumerate(A__				)},      finetuning_task='xnli',      cache_dir=model_args.cache_dir,      revision=model_args.model_revision,      use_auth_token=True if model_args.use_auth_token else None,      )
			SCREAMING_SNAKE_CASE_		:     Tuple					=					AutoTokenizer.from_pretrained(
			    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path,      do_lower_case=model_args.do_lower_case,      cache_dir=model_args.cache_dir,      use_fast=model_args.use_fast_tokenizer,      revision=model_args.model_revision,      use_auth_token=True if model_args.use_auth_token else None,      )
			SCREAMING_SNAKE_CASE_		:     List[str]					=					AutoModelForSequenceClassification.from_pretrained(
			    model_args.model_name_or_path,      from_tf=bool('.ckpt' in model_args.model_name_or_path				),      config=A__,      cache_dir=model_args.cache_dir,      revision=model_args.model_revision,      use_auth_token=True if model_args.use_auth_token else None,      ignore_mismatched_sizes=model_args.ignore_mismatched_sizes,      )

			# Preprocessing the datasets
			# Padding strategy
			if data_args.pad_to_max_length:
						SCREAMING_SNAKE_CASE_		:     Dict					=					'max_length'
			else:
						# We will pad later, dynamically at batch creation, to the max sequence length in each batch
						SCREAMING_SNAKE_CASE_		:     Tuple					=					False

			def preprocess_function(A__				):
						# Tokenize the texts
						return tokenizer(
						    examples['premise'],      examples['hypothesis'],      padding=A__,      max_length=data_args.max_seq_length,      truncation=A__,      )

			if training_args.do_train:
						if data_args.max_train_samples is not None:
									SCREAMING_SNAKE_CASE_		:     Tuple					=					min(len(A__				),      data_args.max_train_samples				)
									SCREAMING_SNAKE_CASE_		:     List[str]					=					train_dataset.select(range(A__				)				)
						with training_args.main_process_first(desc='train dataset map pre-processing'				):
									SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					train_dataset.map(
									    A__,      batched=A__,      load_from_cache_file=not data_args.overwrite_cache,      desc='Running tokenizer on train dataset',      )
						# Log a few random samples from the training set:
						for index in random.sample(range(len(A__				)				),      3				):
									logger.info(F'''Sample {index} of the training set: {train_dataset[index]}.'''				)

			if training_args.do_eval:
						if data_args.max_eval_samples is not None:
									SCREAMING_SNAKE_CASE_		:     List[str]					=					min(len(A__				),      data_args.max_eval_samples				)
									SCREAMING_SNAKE_CASE_		:     str					=					eval_dataset.select(range(A__				)				)
						with training_args.main_process_first(desc='validation dataset map pre-processing'				):
									SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					eval_dataset.map(
									    A__,      batched=A__,      load_from_cache_file=not data_args.overwrite_cache,      desc='Running tokenizer on validation dataset',      )

			if training_args.do_predict:
						if data_args.max_predict_samples is not None:
									SCREAMING_SNAKE_CASE_		:     Optional[int]					=					min(len(A__				),      data_args.max_predict_samples				)
									SCREAMING_SNAKE_CASE_		:     int					=					predict_dataset.select(range(A__				)				)
						with training_args.main_process_first(desc='prediction dataset map pre-processing'				):
									SCREAMING_SNAKE_CASE_		:     Tuple					=					predict_dataset.map(
									    A__,      batched=A__,      load_from_cache_file=not data_args.overwrite_cache,      desc='Running tokenizer on prediction dataset',      )

    # Get the metric function
			SCREAMING_SNAKE_CASE_		:     int					=					evaluate.load('xnli'				)

			# You can define your custom compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a
			# predictions and label_ids field) and has to return a dictionary string to float.
			def compute_metrics(A__				):
						SCREAMING_SNAKE_CASE_		:     List[str]					=					p.predictions[0] if isinstance(p.predictions,      A__				) else p.predictions
						SCREAMING_SNAKE_CASE_		:     List[Any]					=					np.argmax(A__,      axis=1				)
						return metric.compute(predictions=A__,      references=p.label_ids				)

			# Data collator will default to DataCollatorWithPadding, so we change it if we already did the padding.
			if data_args.pad_to_max_length:
						SCREAMING_SNAKE_CASE_		:     Optional[int]					=					default_data_collator
			elif training_args.fpaa:
						SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					DataCollatorWithPadding(A__,      pad_to_multiple_of=8				)
			else:
						SCREAMING_SNAKE_CASE_		:     Optional[int]					=					None

			# Initialize our Trainer
			SCREAMING_SNAKE_CASE_		:     int					=					Trainer(
			    model=A__,      args=A__,      train_dataset=train_dataset if training_args.do_train else None,      eval_dataset=eval_dataset if training_args.do_eval else None,      compute_metrics=A__,      tokenizer=A__,      data_collator=A__,      )

			# Training
			if training_args.do_train:
						SCREAMING_SNAKE_CASE_		:     Optional[int]					=					None
						if training_args.resume_from_checkpoint is not None:
									SCREAMING_SNAKE_CASE_		:     Any					=					training_args.resume_from_checkpoint
						elif last_checkpoint is not None:
									SCREAMING_SNAKE_CASE_		:     Optional[int]					=					last_checkpoint
						SCREAMING_SNAKE_CASE_		:     Tuple					=					trainer.train(resume_from_checkpoint=A__				)
						SCREAMING_SNAKE_CASE_		:     Dict					=					train_result.metrics
						SCREAMING_SNAKE_CASE_		:     Union[str, Any]					=					(
						    data_args.max_train_samples if data_args.max_train_samples is not None else len(A__				)
						)
						SCREAMING_SNAKE_CASE_		:     Dict					=					min(A__,      len(A__				)				)

						trainer.save_model()  # Saves the tokenizer too for easy upload

						trainer.log_metrics('train',      A__				)
						trainer.save_metrics('train',      A__				)
						trainer.save_state()

			# Evaluation
			if training_args.do_eval:
						logger.info('*** Evaluate ***'				)
						SCREAMING_SNAKE_CASE_		:     List[Any]					=					trainer.evaluate(eval_dataset=A__				)

						SCREAMING_SNAKE_CASE_		:     Any					=					data_args.max_eval_samples if data_args.max_eval_samples is not None else len(A__				)
						SCREAMING_SNAKE_CASE_		:     List[Any]					=					min(A__,      len(A__				)				)

						trainer.log_metrics('eval',      A__				)
						trainer.save_metrics('eval',      A__				)

			# Prediction
			if training_args.do_predict:
						logger.info('*** Predict ***'				)
						SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		,      SCREAMING_SNAKE_CASE_		:     Dict					=					trainer.predict(A__,      metric_key_prefix='predict'				)

						SCREAMING_SNAKE_CASE_		:     Dict					=					(
						    data_args.max_predict_samples if data_args.max_predict_samples is not None else len(A__				)
						)
						SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					min(A__,      len(A__				)				)

						trainer.log_metrics('predict',      A__				)
						trainer.save_metrics('predict',      A__				)

						SCREAMING_SNAKE_CASE_		:     List[Any]					=					np.argmax(A__,      axis=1				)
						SCREAMING_SNAKE_CASE_		:     List[str]					=					os.path.join(training_args.output_dir,      'predictions.txt'				)
						if trainer.is_world_process_zero():
									with open(A__,      'w'				) as writer:
												writer.write('index\tprediction\n'				)
												for index, item in enumerate(A__				):
															SCREAMING_SNAKE_CASE_		:     Optional[Any]					=					label_list[item]
															writer.write(F'''{index}\t{item}\n'''				)


if __name__ == "__main__":
				main()







'''simple docstring'''

import torch
from torch import nn
from transformers import CLIPPreTrainedModel, CLIPVisionModel

from ...models.attention import BasicTransformerBlock
from ...utils import logging


snake_case									=    logging.get_logger(__name__)  # pylint: disable=invalid-name




class    lowerCAmelCase       (     UpperCamelCase_  ):


 def __init__(    self  : Union[str, Any]			,							a__  : int			,							a__  : List[Any]=768    ):
  '''simple docstring'''

  super().__init__(a__    )
  lowerCAmelCase__     :   int     		=	proj_size

  lowerCAmelCase__     :   str     		=	CLIPVisionModel(a__    )
  lowerCAmelCase__     :   int     		=	PaintByExampleMapper(a__    )
  lowerCAmelCase__     :   Optional[int]     		=	nn.LayerNorm(config.hidden_size    )
  lowerCAmelCase__     :   Optional[Any]     		=	nn.Linear(config.hidden_size			,							self.proj_size    )

  # uncondition for scaling
  lowerCAmelCase__     :   Dict     		=	nn.Parameter(torch.randn((1, 1, self.proj_size)    )    )


 def    _A	(    self  : Tuple			,							a__  : int			,							a__  : Optional[Any]=False    ):
  '''simple docstring'''

  lowerCAmelCase__     :   List[Any]     		=	self.model(pixel_values=a__    )
  lowerCAmelCase__     :   Any     		=	clip_output.pooler_output
  lowerCAmelCase__     :   Union[str, Any]     		=	self.mapper(latent_states[:, None]    )
  lowerCAmelCase__     :   List[Any]     		=	self.final_layer_norm(a__    )
  lowerCAmelCase__     :   Optional[int]     		=	self.proj_out(a__    )
  if return_uncond_vector:
   return latent_states, self.uncond_vector

  return latent_states




class    lowerCAmelCase       (     nn.Module  ):


 def __init__(    self  : Optional[Any]			,							a__  : Optional[Any]    ):
  '''simple docstring'''

  super().__init__()
  lowerCAmelCase__     :   Optional[Any]     		=	(config.num_hidden_layers + 1) // 5
  lowerCAmelCase__     :   str     		=	config.hidden_size
  lowerCAmelCase__     :   Dict     		=	1
  lowerCAmelCase__     :   int     		=	nn.ModuleList(
      [
          BasicTransformerBlock(a__			,							a__			,							a__			,							activation_fn="gelu"			,							attention_bias=a__    )
          for _ in range(a__    )
      ]    )


 def    _A	(    self  : int			,							a__  : Union[str, Any]    ):
  '''simple docstring'''

  for block in self.blocks:
   lowerCAmelCase__     :   List[Any]     		=	block(a__    )

  return hidden_states





