





'''simple docstring'''





import argparse
import dataclasses
import json
import logging
import os
import shutil
from typing import List, Optional

import datasets
from accelerate import Accelerator
from datasets import load_dataset
from finetuning import finetune
from tqdm.auto import tqdm

import transformers
from transformers import AutoConfig, set_seed
from transformers.trainer_utils import IntervalStrategy


__lowercase  :   Any						      =logging.getLogger(__name__)

__lowercase  :   List[str]						      ='pytorch_model.bin'

@dataclasses.dataclass
class      A       :
      _snake_case							     =dataclasses.field(
          metadata={'''help''': '''Path to pretrained model or model identifier from huggingface.co/models.'''}						)
      _snake_case							     =dataclasses.field(
          default=lowerCamelCase__		,      metadata={'''help''': '''Where do you want to store the pretrained models downloaded from huggingface.co.'''}		,      )

@dataclasses.dataclass
class      A       :
      _snake_case							     =dataclasses.field(metadata={'''help''': '''A csv or a json file containing the training data.'''}						)
      _snake_case							     =dataclasses.field(metadata={'''help''': '''A csv or a json file containing the data to predict on.'''}						)
      _snake_case							     =dataclasses.field(
          default=lowerCamelCase__		,      metadata={'''help''': '''A csv or a json file containing the validation data.'''}						)
      _snake_case							     =dataclasses.field(
          default=lowerCamelCase__		,      metadata={'''help''': '''The name of the task to train on.'''}		,      )
      _snake_case							     =dataclasses.field(
          default=lowerCamelCase__		,      metadata={'''help''': '''The list of labels for the task.'''}						)

@dataclasses.dataclass
class      A       :
      _snake_case							     =dataclasses.field(
          metadata={'''help''': '''The output directory where the model predictions and checkpoints will be written.'''}						)
      _snake_case							     =dataclasses.field(
          default='''accuracy'''		,      metadata={'''help''': '''The evaluation metric used for the task.'''}						)
      _snake_case							     =dataclasses.field(
          default='''no'''		,      metadata={
              '''help''': '''The evaluation strategy to adopt during training. Possible values are: [\"no\", \"step\", \"epoch]'''
          }		,      )
      _snake_case							     =dataclasses.field(
          default=10		,      metadata={'''help''': '''Number of evaluation calls with no improvement after which training will be stopped.'''}		,      )
      _snake_case							     =dataclasses.field(
          default=0.0		,      metadata={
              '''help''': '''How much the specified evaluation metric must improve to satisfy early stopping conditions.'''
          }		,      )
      _snake_case							     =dataclasses.field(
          default=lowerCamelCase__		,      metadata={'''help''': '''Whether to filter the pseudo-labeled data based on the confidence score.'''}		,      )
      _snake_case							     =dataclasses.field(
          default=lowerCamelCase__		,      metadata={'''help''': '''Whether to filter the pseudo-labeled data based on the validation performance.'''}		,      )
      _snake_case							     =dataclasses.field(
          default=lowerCamelCase__		,      metadata={'''help''': '''Whether to fine-tune on labeled data after pseudo training.'''}		,      )
      _snake_case							     =dataclasses.field(
          default=0.0		,      metadata={'''help''': '''Confidence threshold for pseudo-labeled data filtering.'''}		,      )
      _snake_case							     =dataclasses.field(
          default=100		,      metadata={'''help''': '''Number of evaluation calls with no improvement after which training will be stopped.'''}		,      )
      _snake_case							     =dataclasses.field(
          default=lowerCamelCase__		,      metadata={'''help''': '''Random seed for initialization.'''}		,      )




def     a__ (					lowercase__  ,    lowercase__  ,    lowercase__  ,    lowercase__  ,    lowercase__  ,    lowercase__  ):






   '''simple docstring'''






   UpperCAmelCase_											=datasets.concatenate_datasets([infer_input, infer_output]  ,    axis=1  )

   if args.do_filter_by_confidence:
      UpperCAmelCase_											=dataset.filter(lambda lowercase__  : example["probability"] > args.confidence_threshold  )

   if args.do_filter_by_val_performance:
      assert eval_result >= 0.0 and eval_result <= 1.0
      UpperCAmelCase_											=int(eval_result * len(lowerCamelCase_  )  )
      print(lowerCamelCase_  )
      UpperCAmelCase_											=dataset.sort("probability"  ,    reverse=lowerCamelCase_  )
      UpperCAmelCase_											=dataset.select(range(lowerCamelCase_  )  )

   UpperCAmelCase_											=dataset.remove_columns(["label", "probability"]  )
   UpperCAmelCase_											=dataset.rename_column("prediction"  ,    "label"  )
   UpperCAmelCase_											=dataset.map(lambda lowercase__  : {"label": idalabel[example["label"]]}  )
   UpperCAmelCase_											=dataset.shuffle(seed=args.seed  )

   UpperCAmelCase_											=os.path.join(lowerCamelCase_  ,    F'train_pseudo.{args.data_file_extension}'  )
   if args.data_file_extension == "csv":
      dataset.to_csv(lowerCamelCase_  ,    index=lowerCamelCase_  )
   else:
      dataset.to_json(lowerCamelCase_  )





def     a__ (					lowercase__  ,    lowercase__  ,    lowercase__  ,    lowercase__  ,    **lowercase__  ):






   '''simple docstring'''






   UpperCAmelCase_											=Accelerator()
   # Make one log on every process with the configuration for debugging.
   logging.basicConfig(
       format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"  ,    datefmt="%m/%d/%Y %H:%M:%S"  ,    level=logging.INFO  ,    )
   logger.info(accelerator.state  )

   # Setup logging, we only want one process per machine to log things on the
   # screen. accelerator.is_local_main_process is only True for one process per
   # machine.
   logger.setLevel(logging.INFO if accelerator.is_local_main_process else logging.ERROR  )

   if accelerator.is_local_main_process:
      datasets.utils.logging.set_verbosity_warning()
      transformers.utils.logging.set_verbosity_info()
   else:
      datasets.utils.logging.set_verbosity_error()
      transformers.utils.logging.set_verbosity_error()

   UpperCAmelCase_											=STModelArguments(model_name_or_path=lowerCamelCase_  )
   UpperCAmelCase_											=STDataArguments(train_file=lowerCamelCase_  ,    infer_file=lowerCamelCase_  )
   UpperCAmelCase_											=STTrainingArguments(output_dir=lowerCamelCase_  )
   UpperCAmelCase_											=argparse.Namespace()

   for arg_class in (model_args, data_args, training_args):
      for key, value in vars(lowerCamelCase_  ).items():
         setattr(lowerCamelCase_  ,    lowerCamelCase_  ,    lowerCamelCase_  )

   for key, value in kwargs.items():
      if hasattr(lowerCamelCase_  ,    lowerCamelCase_  ):
         setattr(lowerCamelCase_  ,    lowerCamelCase_  ,    lowerCamelCase_  )

    # Sanity checks
   UpperCAmelCase_											={}
   UpperCAmelCase_											=None

   # You need to provide the training data and the data to predict on
   assert args.train_file is not None
   assert args.infer_file is not None
   UpperCAmelCase_											=args.train_file
   UpperCAmelCase_											=args.infer_file

   if args.evaluation_strategy != IntervalStrategy.NO.value:
      assert args.eval_file is not None
      UpperCAmelCase_											=args.eval_file

   for key in data_files:
      UpperCAmelCase_											=data_files[key].split("."  )[-1]
      assert extension in ["csv", "json"], F'`{key}_file` should be a csv or a json file.'
      if args.data_file_extension is None:
         UpperCAmelCase_											=extension
      else:
         assert extension == args.data_file_extension, F'`{key}_file` should be a {args.data_file_extension} file`.'

   assert (
       args.eval_metric in datasets.list_metrics()
   ), F'{args.eval_metric} not in the list of supported metrics {datasets.list_metrics()}.'

   # If passed along, set the training seed now.
   if args.seed is not None:
      set_seed(args.seed  )

   logger.info("Creating the initial data directory for self-training..."  )
   UpperCAmelCase_											=F'{args.output_dir}/self-train_iter-{{}}'.format
   UpperCAmelCase_											=data_dir_format(0  )

   if accelerator.is_main_process:
      if args.output_dir is not None:
         os.makedirs(args.output_dir  ,    exist_ok=lowerCamelCase_  )
         os.makedirs(lowerCamelCase_  ,    exist_ok=lowerCamelCase_  )
   accelerator.wait_for_everyone()

   UpperCAmelCase_											=None
   UpperCAmelCase_											=None
   UpperCAmelCase_											=0
   UpperCAmelCase_											=False
   # Show the progress bar
   UpperCAmelCase_											=tqdm(range(args.max_selftrain_iterations  )  ,    disable=not accelerator.is_local_main_process  )

   # Self-train
   for iteration in range(0  ,    int(args.max_selftrain_iterations  )  ):
      UpperCAmelCase_											=data_dir_format(lowerCamelCase_  )
      assert os.path.exists(lowerCamelCase_  )

      # Stage 1: initial fine-tuning for iteration = 0 or pseudo-training for
      # iteration > 0
      UpperCAmelCase_											=os.path.join(lowerCamelCase_  ,    "stage-1"  )
      UpperCAmelCase_											={
          '''accelerator''': accelerator,
          '''model_name_or_path''': args.model_name_or_path,
          '''cache_dir''': args.cache_dir,
          '''do_train''': True,
          '''train_file''': data_files['''train'''] if iteration == 0 else data_files['''train_pseudo'''],
          '''do_eval''': True if args.eval_file is not None else False,
          '''eval_file''': data_files['''eval'''],
          '''do_predict''': True,
          '''infer_file''': data_files['''infer'''],
          '''task_name''': args.task_name,
          '''label_list''': args.label_list,
          '''output_dir''': current_output_dir,
          '''eval_metric''': args.eval_metric,
          '''evaluation_strategy''': args.evaluation_strategy,
          '''early_stopping_patience''': args.early_stopping_patience,
          '''early_stopping_threshold''': args.early_stopping_threshold,
          '''seed''': args.seed,
      }
      # Add additional training arguments
      for key, value in kwargs.items():
         if key not in arguments_dict and not hasattr(lowerCamelCase_  ,    lowerCamelCase_  ):
            arguments_dict.update({key: value}  )

      UpperCAmelCase_											=os.path.join(lowerCamelCase_  ,    "best-checkpoint"  ,    lowerCamelCase_  )
      if os.path.exists(lowerCamelCase_  ):
         logger.info(
             "Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 1."  ,    lowerCamelCase_  ,    lowerCamelCase_  ,    )
      else:
         logger.info("***** Running self-training: iteration: %d, stage: 1 *****"  ,    lowerCamelCase_  )
         finetune(**lowerCamelCase_  )
         accelerator.wait_for_everyone()
         assert os.path.exists(lowerCamelCase_  )
         logger.info("Self-training job completed: iteration: %d, stage: 1."  ,    lowerCamelCase_  )

      if iteration > 0 and args.finetune_on_labeled_data:
         # Stage 2 (optional): fine-tuning on the original labeled data
         UpperCAmelCase_											=os.path.join(lowerCamelCase_  ,    "best-checkpoint"  )
         UpperCAmelCase_											=os.path.join(lowerCamelCase_  ,    "stage-2"  )
         # Update arguments_dict
         UpperCAmelCase_											=model_path
         UpperCAmelCase_											=data_files['''train''']
         UpperCAmelCase_											=current_output_dir

         UpperCAmelCase_											=os.path.join(lowerCamelCase_  ,    "best-checkpoint"  ,    lowerCamelCase_  )
         if os.path.exists(lowerCamelCase_  ):
            logger.info(
                "Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 2."  ,    lowerCamelCase_  ,    lowerCamelCase_  ,    )
         else:
            logger.info("***** Running self-training: iteration: %d, stage: 2 *****"  ,    lowerCamelCase_  )
            finetune(**lowerCamelCase_  )
            accelerator.wait_for_everyone()
            assert os.path.exists(lowerCamelCase_  )
            logger.info("Self-training job completed: iteration: %d, stage: 2."  ,    lowerCamelCase_  )

      UpperCAmelCase_											=iteration
      UpperCAmelCase_											=data_dir_format(iteration + 1  )

      UpperCAmelCase_											=AutoConfig.from_pretrained(os.path.join(lowerCamelCase_  ,    "best-checkpoint"  )  )
      UpperCAmelCase_											=config.idalabel
      UpperCAmelCase_											=os.path.join(lowerCamelCase_  ,    "eval_results_best-checkpoint.json"  )
      UpperCAmelCase_											=os.path.join(lowerCamelCase_  ,    "test_results_best-checkpoint.json"  )
      assert os.path.exists(lowerCamelCase_  )

      with open(lowerCamelCase_  ,    "r"  ) as f:
         UpperCAmelCase_											=float(json.load(lowerCamelCase_  )[args.eval_metric]  )
      UpperCAmelCase_											=os.path.join(lowerCamelCase_  ,    "infer_output_best-checkpoint.csv"  )
      assert os.path.exists(lowerCamelCase_  )
      # Loading the dataset from local csv or json files.
      UpperCAmelCase_											=load_dataset(args.data_file_extension  ,    data_files={"data": data_files["infer"]}  )['''data''']
      UpperCAmelCase_											=load_dataset("csv"  ,    data_files={"data": infer_output_file}  )['''data''']

      if accelerator.is_main_process:
         os.makedirs(lowerCamelCase_  ,    exist_ok=lowerCamelCase_  )
         shutil.copy(lowerCamelCase_  ,    os.path.join(lowerCamelCase_  ,    F'eval_results_iter-{iteration}.json'  )  )
         if os.path.exists(lowerCamelCase_  ):
            shutil.copy(lowerCamelCase_  ,    os.path.join(lowerCamelCase_  ,    F'test_results_iter-{iteration}.json'  )  )
         create_pseudo_labeled_data(lowerCamelCase_  ,    lowerCamelCase_  ,    lowerCamelCase_  ,    lowerCamelCase_  ,    lowerCamelCase_  ,    lowerCamelCase_  )
      accelerator.wait_for_everyone()

      UpperCAmelCase_											=os.path.join(lowerCamelCase_  ,    F'train_pseudo.{args.data_file_extension}'  )

      if args.evaluation_strategy != IntervalStrategy.NO.value:
         UpperCAmelCase_											=eval_result

         if best_iteration is None:
            UpperCAmelCase_											=new_iteration
            UpperCAmelCase_											=new_eval_result
         else:
            if new_eval_result - best_eval_result > args.early_stopping_threshold:
               UpperCAmelCase_											=new_iteration
               UpperCAmelCase_											=new_eval_result
               UpperCAmelCase_											=0
            else:
               if new_eval_result == best_eval_result:
                  UpperCAmelCase_											=new_iteration
                  UpperCAmelCase_											=new_eval_result
               early_stopping_patience_counter += 1

            if early_stopping_patience_counter >= args.early_stopping_patience:
               UpperCAmelCase_											=True

      progress_bar.update(1  )

      if should_training_stop:
         break

   if best_iteration is not None:
      # Save the best iteration
      logger.info("Best iteration: %d"  ,    lowerCamelCase_  )
      logger.info("Best evaluation result: %s = %f"  ,    args.eval_metric  ,    lowerCamelCase_  )
      accelerator.wait_for_everyone()
      if accelerator.is_main_process:
         shutil.copy(
             os.path.join(lowerCamelCase_  ,    F'eval_results_iter-{iteration}.json'  )  ,    os.path.join(lowerCamelCase_  ,    "eval_results_best-iteration.json"  )  ,    )
   else:
      # Assume that the last iteration is the best
      logger.info("Best iteration: %d"  ,    args.max_selftrain_iterations - 1  )
      logger.info("Best evaluation result: %s = %f"  ,    args.eval_metric  ,    lowerCamelCase_  )
      accelerator.wait_for_everyone()
      if accelerator.is_main_process:
         shutil.copy(
             os.path.join(lowerCamelCase_  ,    F'eval_results_iter-{args.max_selftrain_iterations - 1}.json'  )  ,    os.path.join(lowerCamelCase_  ,    "eval_results_best-iteration.json"  )  ,    )











import json
import os
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from requests.exceptions import HTTPError

from transformers.utils import (
    CONFIG_NAME,
    FLAX_WEIGHTS_NAME,
    TF2_WEIGHTS_NAME,
    TRANSFORMERS_CACHE,
    WEIGHTS_NAME,
    cached_file,
    get_file_from_repo,
    has_file,
)


__lowercase  :   Any						      ="""hf-internal-testing/tiny-random-bert"""
__lowercase  :   List[str]						      =os.path.join(TRANSFORMERS_CACHE, """models--hf-internal-testing--tiny-random-bert""")
__lowercase  :   Dict						      ="""9b8c223d42b2188cb49d29af482996f9d0f3e5a6"""

class      A       (	unittest.TestCase						):



      def 	lowerCAmelCase__    (	self:						List[str]       )						->    str:
         '''simple docstring'''

         UpperCAmelCase_											=cached_file(_lowerCAmelCase		,  _lowerCAmelCase       )
         # Should have downloaded the file in here
         self.assertTrue(os.path.isdir(_lowerCAmelCase       )       )
         # Cache should contain at least those three subfolders:
         for subfolder in ["blobs", "refs", "snapshots"]:
            self.assertTrue(os.path.isdir(os.path.join(_lowerCAmelCase		,  _lowerCAmelCase       )       )       )
         with open(os.path.join(_lowerCAmelCase		,  "refs"		,  "main"       )       ) as f:
            UpperCAmelCase_											=f.read()
         self.assertEqual(_lowerCAmelCase		,  os.path.join(_lowerCAmelCase		,  "snapshots"		,  _lowerCAmelCase		,  _lowerCAmelCase       )       )
         self.assertTrue(os.path.isfile(_lowerCAmelCase       )       )

         # File is cached at the same place the second time.
         UpperCAmelCase_											=cached_file(_lowerCAmelCase		,  _lowerCAmelCase       )
         self.assertEqual(_lowerCAmelCase		,  _lowerCAmelCase       )

         # Using a specific revision to test the full commit hash.
         UpperCAmelCase_											=cached_file(_lowerCAmelCase		,  _lowerCAmelCase		,  revision="9b8c223"       )
         self.assertEqual(_lowerCAmelCase		,  os.path.join(_lowerCAmelCase		,  "snapshots"		,  _lowerCAmelCase		,  _lowerCAmelCase       )       )



      def 	lowerCAmelCase__    (	self:						Any       )						->    Optional[Any]:
         '''simple docstring'''

         with self.assertRaisesRegex(_lowerCAmelCase		,  "is not a valid model identifier"       ):
            UpperCAmelCase_											=cached_file("tiny-random-bert"		,  _lowerCAmelCase       )

         with self.assertRaisesRegex(_lowerCAmelCase		,  "is not a valid git identifier"       ):
            UpperCAmelCase_											=cached_file(_lowerCAmelCase		,  _lowerCAmelCase		,  revision="aaaa"       )

         with self.assertRaisesRegex(_lowerCAmelCase		,  "does not appear to have a file named"       ):
            UpperCAmelCase_											=cached_file(_lowerCAmelCase		,  "conf"       )



      def 	lowerCAmelCase__    (	self:						Tuple       )						->    Optional[Any]:
         '''simple docstring'''

         with self.assertRaisesRegex(_lowerCAmelCase		,  "does not appear to have a file named"       ):
            UpperCAmelCase_											=cached_file(_lowerCAmelCase		,  "conf"       )

         with open(os.path.join(_lowerCAmelCase		,  "refs"		,  "main"       )       ) as f:
            UpperCAmelCase_											=f.read()
         self.assertTrue(os.path.isfile(os.path.join(_lowerCAmelCase		,  ".no_exist"		,  _lowerCAmelCase		,  "conf"       )       )       )

         UpperCAmelCase_											=cached_file(_lowerCAmelCase		,  "conf"		,  _raise_exceptions_for_missing_entries=_lowerCAmelCase       )
         self.assertIsNone(_lowerCAmelCase       )

         UpperCAmelCase_											=cached_file(_lowerCAmelCase		,  "conf"		,  local_files_only=_lowerCAmelCase		,  _raise_exceptions_for_missing_entries=_lowerCAmelCase       )
         self.assertIsNone(_lowerCAmelCase       )

         UpperCAmelCase_											=mock.Mock()
         UpperCAmelCase_											=500
         UpperCAmelCase_											={}
         UpperCAmelCase_											=HTTPError
         UpperCAmelCase_											={}

         # Under the mock environment we get a 500 error when trying to reach the tokenizer.
         with mock.patch("requests.Session.request"		,  return_value=_lowerCAmelCase       ) as mock_head:
            UpperCAmelCase_											=cached_file(_lowerCAmelCase		,  "conf"		,  _raise_exceptions_for_connection_errors=_lowerCAmelCase       )
            self.assertIsNone(_lowerCAmelCase       )
            # This check we did call the fake head request
            mock_head.assert_called()



      def 	lowerCAmelCase__    (	self:						Any       )						->    Tuple:
         '''simple docstring'''

         self.assertTrue(has_file("hf-internal-testing/tiny-bert-pt-only"		,  _lowerCAmelCase       )       )
         self.assertFalse(has_file("hf-internal-testing/tiny-bert-pt-only"		,  _lowerCAmelCase       )       )
         self.assertFalse(has_file("hf-internal-testing/tiny-bert-pt-only"		,  _lowerCAmelCase       )       )



      def 	lowerCAmelCase__    (	self:						Any       )						->    Dict:
         '''simple docstring'''

         self.assertIsNone(get_file_from_repo("bert-base-cased"		,  "ahah.txt"       )       )

         # The function raises if the repository does not exist.
         with self.assertRaisesRegex(_lowerCAmelCase		,  "is not a valid model identifier"       ):
            get_file_from_repo("bert-base-case"		,  _lowerCAmelCase       )

         # The function raises if the revision does not exist.
         with self.assertRaisesRegex(_lowerCAmelCase		,  "is not a valid git identifier"       ):
            get_file_from_repo("bert-base-cased"		,  _lowerCAmelCase		,  revision="ahaha"       )

         UpperCAmelCase_											=get_file_from_repo("bert-base-cased"		,  _lowerCAmelCase       )
         # The name is the cached name which is not very easy to test, so instead we load the content.
         UpperCAmelCase_											=json.loads(open(_lowerCAmelCase		,  "r"       ).read()       )
         self.assertEqual(config["hidden_size"]		,  768       )






      def 	lowerCAmelCase__    (	self:						Optional[int]       )						->    Dict:
         '''simple docstring'''

         with tempfile.TemporaryDirectory() as tmp_dir:
            UpperCAmelCase_											=Path(_lowerCAmelCase       ) / "a.txt"
            filename.touch()
            self.assertEqual(get_file_from_repo(_lowerCAmelCase		,  "a.txt"       )		,  str(_lowerCAmelCase       )       )

            self.assertIsNone(get_file_from_repo(_lowerCAmelCase		,  "b.txt"       )       )




