





"""simple docstring"""





import argparse
import dataclasses
import json
import logging
import os
import shutil
from typing import List, Optional

import datasets
from accelerate import Accelerator
from datasets import load_dataset
from finetuning import finetune
from tqdm.auto import tqdm

import transformers
from transformers import AutoConfig, set_seed
from transformers.trainer_utils import IntervalStrategy


lowercase__				:Tuple				   =       logging.getLogger(__name__)

lowercase__				:int				   =       'pytorch_model.bin'




@dataclasses.dataclass
class 							snake_case							:

      '''simple docstring'''



      _A     :       str         =       dataclasses.field(
          metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models.'} )
      _A     :       Optional[str]         =       dataclasses.field(
          default=__UpperCAmelCase      ,  metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co.'}      ,  )




@dataclasses.dataclass
class 							snake_case							:

      '''simple docstring'''



      _A     :       str         =       dataclasses.field(metadata={'help': 'A csv or a json file containing the training data.'} )
      _A     :       str         =       dataclasses.field(metadata={'help': 'A csv or a json file containing the data to predict on.'} )
      _A     :       Optional[str]         =       dataclasses.field(
          default=__UpperCAmelCase      ,  metadata={'help': 'A csv or a json file containing the validation data.'} )
      _A     :       Optional[str]         =       dataclasses.field(
          default=__UpperCAmelCase      ,  metadata={'help': 'The name of the task to train on.'}      ,  )
      _A     :       Optional[List[str]]         =       dataclasses.field(
          default=__UpperCAmelCase      ,  metadata={'help': 'The list of labels for the task.'} )




@dataclasses.dataclass
class 							snake_case							:

      '''simple docstring'''



      _A     :       str         =       dataclasses.field(
          metadata={'help': 'The output directory where the model predictions and checkpoints will be written.'} )
      _A     :       Optional[str]         =       dataclasses.field(
          default='accuracy'      ,  metadata={'help': 'The evaluation metric used for the task.'} )
      _A     :       Optional[str]         =       dataclasses.field(
          default='no'      ,  metadata={
              'help': 'The evaluation strategy to adopt during training. Possible values are: ["no", "step", "epoch]'
          }      ,  )
      _A     :       Optional[int]         =       dataclasses.field(
          default=10      ,  metadata={'help': 'Number of evaluation calls with no improvement after which training will be stopped.'}      ,  )
      _A     :       Optional[float]         =       dataclasses.field(
          default=0.0      ,  metadata={
              'help': 'How much the specified evaluation metric must improve to satisfy early stopping conditions.'
          }      ,  )
      _A     :       Optional[bool]         =       dataclasses.field(
          default=__UpperCAmelCase      ,  metadata={'help': 'Whether to filter the pseudo-labeled data based on the confidence score.'}      ,  )
      _A     :       Optional[bool]         =       dataclasses.field(
          default=__UpperCAmelCase      ,  metadata={'help': 'Whether to filter the pseudo-labeled data based on the validation performance.'}      ,  )
      _A     :       Optional[bool]         =       dataclasses.field(
          default=__UpperCAmelCase      ,  metadata={'help': 'Whether to fine-tune on labeled data after pseudo training.'}      ,  )
      _A     :       Optional[float]         =       dataclasses.field(
          default=0.0      ,  metadata={'help': 'Confidence threshold for pseudo-labeled data filtering.'}      ,  )
      _A     :       Optional[int]         =       dataclasses.field(
          default=1_00      ,  metadata={'help': 'Number of evaluation calls with no improvement after which training will be stopped.'}      ,  )
      _A     :       Optional[int]         =       dataclasses.field(
          default=__UpperCAmelCase      ,  metadata={'help': 'Random seed for initialization.'}      ,  )


def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  )						->Optional[Any]:





  """simple docstring"""


  __UpperCAmelCase    :						int       =						datasets.concatenate_datasets([infer_input, infer_output]  ,			axis=1  )

  if args.do_filter_by_confidence:
    __UpperCAmelCase    :						str       =						dataset.filter(lambda UpperCAmelCase_  : example["probability"] > args.confidence_threshold  )

  if args.do_filter_by_val_performance:
    assert eval_result >= 0.0 and eval_result <= 1.0
    __UpperCAmelCase    :						List[str]       =						int(eval_result * len(UpperCAmelCase_  )  )
    print(UpperCAmelCase_  )
    __UpperCAmelCase    :						List[str]       =						dataset.sort('''probability'''  ,			reverse=UpperCAmelCase_  )
    __UpperCAmelCase    :						Any       =						dataset.select(range(UpperCAmelCase_  )  )

  __UpperCAmelCase    :						List[str]       =						dataset.remove_columns(['''label''', '''probability''']  )
  __UpperCAmelCase    :						Tuple       =						dataset.rename_column('''prediction'''  ,			'''label'''  )
  __UpperCAmelCase    :						str       =						dataset.map(lambda UpperCAmelCase_  : {"label": idalabel[example["label"]]}  )
  __UpperCAmelCase    :						List[Any]       =						dataset.shuffle(seed=args.seed  )

  __UpperCAmelCase    :						Optional[int]       =						os.path.join(UpperCAmelCase_  ,			f'''train_pseudo.{args.data_file_extension}'''  )
  if args.data_file_extension == "csv":
    dataset.to_csv(UpperCAmelCase_  ,			index=UpperCAmelCase_  )
  else:
    dataset.to_json(UpperCAmelCase_  )







def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			**UpperCAmelCase_  )						->List[Any]:





  """simple docstring"""


  __UpperCAmelCase    :						List[str]       =						Accelerator()
  # Make one log on every process with the configuration for debugging.
  logging.basicConfig(
      format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s'''  ,			datefmt='''%m/%d/%Y %H:%M:%S'''  ,			level=logging.INFO  ,			)
  logger.info(accelerator.state  )

  # Setup logging, we only want one process per machine to log things on the
  # screen. accelerator.is_local_main_process is only True for one process per
  # machine.
  logger.setLevel(logging.INFO if accelerator.is_local_main_process else logging.ERROR  )

  if accelerator.is_local_main_process:
    datasets.utils.logging.set_verbosity_warning()
    transformers.utils.logging.set_verbosity_info()
  else:
    datasets.utils.logging.set_verbosity_error()
    transformers.utils.logging.set_verbosity_error()

  __UpperCAmelCase    :						str       =						STModelArguments(model_name_or_path=UpperCAmelCase_  )
  __UpperCAmelCase    :						Optional[Any]       =						STDataArguments(train_file=UpperCAmelCase_  ,			infer_file=UpperCAmelCase_  )
  __UpperCAmelCase    :						Any       =						STTrainingArguments(output_dir=UpperCAmelCase_  )
  __UpperCAmelCase    :						int       =						argparse.Namespace()

  for arg_class in (model_args, data_args, training_args):
    for key, value in vars(UpperCAmelCase_  ).items():
      setattr(UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  )

  for key, value in kwargs.items():
    if hasattr(UpperCAmelCase_  ,			UpperCAmelCase_  ):
      setattr(UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  )

    # Sanity checks
  __UpperCAmelCase    :						List[Any]       =						{}
  __UpperCAmelCase    :						Union[str, Any]       =						None

  # You need to provide the training data and the data to predict on
  assert args.train_file is not None
  assert args.infer_file is not None
  __UpperCAmelCase    :						List[Any]       =						args.train_file
  __UpperCAmelCase    :						Tuple       =						args.infer_file

  if args.evaluation_strategy != IntervalStrategy.NO.value:
    assert args.eval_file is not None
    __UpperCAmelCase    :						Any       =						args.eval_file

  for key in data_files:
    __UpperCAmelCase    :						Union[str, Any]       =						data_files[key].split('''.'''  )[-1]
    assert extension in ["csv", "json"], f'''`{key}_file` should be a csv or a json file.'''
    if args.data_file_extension is None:
      __UpperCAmelCase    :						str       =						extension
    else:
      assert extension == args.data_file_extension, f'''`{key}_file` should be a {args.data_file_extension} file`.'''

  assert (
      args.eval_metric in datasets.list_metrics()
  ), f'''{args.eval_metric} not in the list of supported metrics {datasets.list_metrics()}.'''

  # If passed along, set the training seed now.
  if args.seed is not None:
    set_seed(args.seed  )

  logger.info('''Creating the initial data directory for self-training...'''  )
  __UpperCAmelCase    :						List[str]       =						f'''{args.output_dir}/self-train_iter-{{}}'''.format
  __UpperCAmelCase    :						List[str]       =						data_dir_format(0  )

  if accelerator.is_main_process:
    if args.output_dir is not None:
      os.makedirs(args.output_dir  ,			exist_ok=UpperCAmelCase_  )
      os.makedirs(UpperCAmelCase_  ,			exist_ok=UpperCAmelCase_  )
  accelerator.wait_for_everyone()

  __UpperCAmelCase    :						Dict       =						None
  __UpperCAmelCase    :						int       =						None
  __UpperCAmelCase    :						Tuple       =						0
  __UpperCAmelCase    :						Optional[int]       =						False
  # Show the progress bar
  __UpperCAmelCase    :						Tuple       =						tqdm(range(args.max_selftrain_iterations  )  ,			disable=not accelerator.is_local_main_process  )

  # Self-train
  for iteration in range(0  ,			int(args.max_selftrain_iterations  )  ):
    __UpperCAmelCase    :						Dict       =						data_dir_format(UpperCAmelCase_  )
    assert os.path.exists(UpperCAmelCase_  )

    # Stage 1: initial fine-tuning for iteration = 0 or pseudo-training for
    # iteration > 0
    __UpperCAmelCase    :						Optional[int]       =						os.path.join(UpperCAmelCase_  ,			'''stage-1'''  )
    __UpperCAmelCase    :						Optional[int]       =						{
        '''accelerator''': accelerator,
        '''model_name_or_path''': args.model_name_or_path,
        '''cache_dir''': args.cache_dir,
        '''do_train''': True,
        '''train_file''': data_files['''train'''] if iteration == 0 else data_files['''train_pseudo'''],
        '''do_eval''': True if args.eval_file is not None else False,
        '''eval_file''': data_files['''eval'''],
        '''do_predict''': True,
        '''infer_file''': data_files['''infer'''],
        '''task_name''': args.task_name,
        '''label_list''': args.label_list,
        '''output_dir''': current_output_dir,
        '''eval_metric''': args.eval_metric,
        '''evaluation_strategy''': args.evaluation_strategy,
        '''early_stopping_patience''': args.early_stopping_patience,
        '''early_stopping_threshold''': args.early_stopping_threshold,
        '''seed''': args.seed,
    }
    # Add additional training arguments
    for key, value in kwargs.items():
      if key not in arguments_dict and not hasattr(UpperCAmelCase_  ,			UpperCAmelCase_  ):
        arguments_dict.update({key: value}  )

    __UpperCAmelCase    :						Tuple       =						os.path.join(UpperCAmelCase_  ,			'''best-checkpoint'''  ,			UpperCAmelCase_  )
    if os.path.exists(UpperCAmelCase_  ):
      logger.info(
          '''Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 1.'''  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			)
    else:
      logger.info('''***** Running self-training: iteration: %d, stage: 1 *****'''  ,			UpperCAmelCase_  )
      finetune(**UpperCAmelCase_  )
      accelerator.wait_for_everyone()
      assert os.path.exists(UpperCAmelCase_  )
      logger.info('''Self-training job completed: iteration: %d, stage: 1.'''  ,			UpperCAmelCase_  )

    if iteration > 0 and args.finetune_on_labeled_data:
      # Stage 2 (optional): fine-tuning on the original labeled data
      __UpperCAmelCase    :						Dict       =						os.path.join(UpperCAmelCase_  ,			'''best-checkpoint'''  )
      __UpperCAmelCase    :						Any       =						os.path.join(UpperCAmelCase_  ,			'''stage-2'''  )
      # Update arguments_dict
      __UpperCAmelCase    :						Optional[Any]       =						model_path
      __UpperCAmelCase    :						Optional[Any]       =						data_files['''train''']
      __UpperCAmelCase    :						List[str]       =						current_output_dir

      __UpperCAmelCase    :						Union[str, Any]       =						os.path.join(UpperCAmelCase_  ,			'''best-checkpoint'''  ,			UpperCAmelCase_  )
      if os.path.exists(UpperCAmelCase_  ):
        logger.info(
            '''Found existing model checkpoint at %s. Skipping self-training: iteration: %d, stage: 2.'''  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			)
      else:
        logger.info('''***** Running self-training: iteration: %d, stage: 2 *****'''  ,			UpperCAmelCase_  )
        finetune(**UpperCAmelCase_  )
        accelerator.wait_for_everyone()
        assert os.path.exists(UpperCAmelCase_  )
        logger.info('''Self-training job completed: iteration: %d, stage: 2.'''  ,			UpperCAmelCase_  )

    __UpperCAmelCase    :						Optional[int]       =						iteration
    __UpperCAmelCase    :						str       =						data_dir_format(iteration + 1  )

    __UpperCAmelCase    :						List[str]       =						AutoConfig.from_pretrained(os.path.join(UpperCAmelCase_  ,			'''best-checkpoint'''  )  )
    __UpperCAmelCase    :						Any       =						config.idalabel
    __UpperCAmelCase    :						int       =						os.path.join(UpperCAmelCase_  ,			'''eval_results_best-checkpoint.json'''  )
    __UpperCAmelCase    :						Optional[int]       =						os.path.join(UpperCAmelCase_  ,			'''test_results_best-checkpoint.json'''  )
    assert os.path.exists(UpperCAmelCase_  )

    with open(UpperCAmelCase_  ,			'''r'''  ) as f:
      __UpperCAmelCase    :						Any       =						float(json.load(UpperCAmelCase_  )[args.eval_metric]  )
    __UpperCAmelCase    :						Dict       =						os.path.join(UpperCAmelCase_  ,			'''infer_output_best-checkpoint.csv'''  )
    assert os.path.exists(UpperCAmelCase_  )
    # Loading the dataset from local csv or json files.
    __UpperCAmelCase    :						Tuple       =						load_dataset(args.data_file_extension  ,			data_files={'''data''': data_files['''infer''']}  )['''data''']
    __UpperCAmelCase    :						Optional[int]       =						load_dataset('''csv'''  ,			data_files={'''data''': infer_output_file}  )['''data''']

    if accelerator.is_main_process:
      os.makedirs(UpperCAmelCase_  ,			exist_ok=UpperCAmelCase_  )
      shutil.copy(UpperCAmelCase_  ,			os.path.join(UpperCAmelCase_  ,			f'''eval_results_iter-{iteration}.json'''  )  )
      if os.path.exists(UpperCAmelCase_  ):
        shutil.copy(UpperCAmelCase_  ,			os.path.join(UpperCAmelCase_  ,			f'''test_results_iter-{iteration}.json'''  )  )
      create_pseudo_labeled_data(UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  ,			UpperCAmelCase_  )
    accelerator.wait_for_everyone()

    __UpperCAmelCase    :						Optional[Any]       =						os.path.join(UpperCAmelCase_  ,			f'''train_pseudo.{args.data_file_extension}'''  )

    if args.evaluation_strategy != IntervalStrategy.NO.value:
      __UpperCAmelCase    :						List[str]       =						eval_result

      if best_iteration is None:
        __UpperCAmelCase    :						Optional[Any]       =						new_iteration
        __UpperCAmelCase    :						Optional[int]       =						new_eval_result
      else:
        if new_eval_result - best_eval_result > args.early_stopping_threshold:
          __UpperCAmelCase    :						Dict       =						new_iteration
          __UpperCAmelCase    :						List[str]       =						new_eval_result
          __UpperCAmelCase    :						List[Any]       =						0
        else:
          if new_eval_result == best_eval_result:
            __UpperCAmelCase    :						Tuple       =						new_iteration
            __UpperCAmelCase    :						Optional[Any]       =						new_eval_result
          early_stopping_patience_counter += 1

        if early_stopping_patience_counter >= args.early_stopping_patience:
          __UpperCAmelCase    :						str       =						True

    progress_bar.update(1  )

    if should_training_stop:
      break

  if best_iteration is not None:
    # Save the best iteration
    logger.info('''Best iteration: %d'''  ,			UpperCAmelCase_  )
    logger.info('''Best evaluation result: %s = %f'''  ,			args.eval_metric  ,			UpperCAmelCase_  )
    accelerator.wait_for_everyone()
    if accelerator.is_main_process:
      shutil.copy(
          os.path.join(UpperCAmelCase_  ,			f'''eval_results_iter-{iteration}.json'''  )  ,			os.path.join(UpperCAmelCase_  ,			'''eval_results_best-iteration.json'''  )  ,			)
  else:
    # Assume that the last iteration is the best
    logger.info('''Best iteration: %d'''  ,			args.max_selftrain_iterations - 1  )
    logger.info('''Best evaluation result: %s = %f'''  ,			args.eval_metric  ,			UpperCAmelCase_  )
    accelerator.wait_for_everyone()
    if accelerator.is_main_process:
      shutil.copy(
          os.path.join(UpperCAmelCase_  ,			f'''eval_results_iter-{args.max_selftrain_iterations - 1}.json'''  )  ,			os.path.join(UpperCAmelCase_  ,			'''eval_results_best-iteration.json'''  )  ,			)






"""simple docstring"""





import gc
import unittest

from diffusers import FlaxDPMSolverMultistepScheduler, FlaxStableDiffusionPipeline
from diffusers.utils import is_flax_available, slow
from diffusers.utils.testing_utils import require_flax


if is_flax_available():
  import jax
  import jax.numpy as jnp
  from flax.jax_utils import replicate
  from flax.training.common_utils import shard




@slow
@require_flax
class 							snake_case							(			unittest.TestCase ):

      '''simple docstring'''



      def       A_					(							self       :      str ):


        '''simple docstring'''




        super().tearDown()
        gc.collect()

      def       A_					(							self       :      List[Any] ):


        '''simple docstring'''




        __UpperCAmelCase       ,     __UpperCAmelCase    :						List[Any]       =						FlaxStableDiffusionPipeline.from_pretrained(
            '''stabilityai/stable-diffusion-2'''						,				revision='''bf16'''						,				dtype=jnp.bfloataa						,				)

        __UpperCAmelCase    :						Dict       =						'''A painting of a squirrel eating a burger'''
        __UpperCAmelCase    :						Any       =						jax.device_count()
        __UpperCAmelCase    :						Optional[int]       =						num_samples * [prompt]
        __UpperCAmelCase    :						Tuple       =						sd_pipe.prepare_inputs(__lowercase )

        __UpperCAmelCase    :						List[Any]       =						replicate(__lowercase )
        __UpperCAmelCase    :						Optional[Any]       =						shard(__lowercase )

        __UpperCAmelCase    :						str       =						jax.random.PRNGKey(0 )
        __UpperCAmelCase    :						Dict       =						jax.random.split(__lowercase						,				jax.device_count() )

        __UpperCAmelCase    :						Tuple       =						sd_pipe(__lowercase						,				__lowercase						,				__lowercase						,				num_inference_steps=25						,				jit=__lowercase )[0]
        assert images.shape == (jax.device_count(), 1, 768, 768, 3)

        __UpperCAmelCase    :						Any       =						images.reshape((images.shape[0] * images.shape[1],) + images.shape[-3:] )
        __UpperCAmelCase    :						Union[str, Any]       =						images[0, 253:256, 253:256, -1]

        __UpperCAmelCase    :						str       =						jnp.asarray(jax.device_get(image_slice.flatten() ) )
        __UpperCAmelCase    :						List[Any]       =						jnp.array([0.4_2_3_8, 0.4_4_1_4, 0.4_3_9_5, 0.4_4_5_3, 0.4_6_2_9, 0.4_5_9_0, 0.4_5_3_1, 0.4_5_5_0_8, 0.4_5_1_2] )
        print(f'''output_slice: {output_slice}''' )
        assert jnp.abs(output_slice - expected_slice ).max() < 1e-2







      def       A_					(							self       :      Union[str, Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[int]       =						'''stabilityai/stable-diffusion-2'''
        __UpperCAmelCase       ,     __UpperCAmelCase    :						Any       =						FlaxDPMSolverMultistepScheduler.from_pretrained(__lowercase						,				subfolder='''scheduler''' )
        __UpperCAmelCase       ,     __UpperCAmelCase    :						Optional[int]       =						FlaxStableDiffusionPipeline.from_pretrained(
            __lowercase						,				scheduler=__lowercase						,				revision='''bf16'''						,				dtype=jnp.bfloataa						,				)
        __UpperCAmelCase    :						Dict       =						scheduler_params

        __UpperCAmelCase    :						int       =						'''A painting of a squirrel eating a burger'''
        __UpperCAmelCase    :						List[Any]       =						jax.device_count()
        __UpperCAmelCase    :						Optional[int]       =						num_samples * [prompt]
        __UpperCAmelCase    :						Tuple       =						sd_pipe.prepare_inputs(__lowercase )

        __UpperCAmelCase    :						Optional[int]       =						replicate(__lowercase )
        __UpperCAmelCase    :						List[Any]       =						shard(__lowercase )

        __UpperCAmelCase    :						Union[str, Any]       =						jax.random.PRNGKey(0 )
        __UpperCAmelCase    :						List[Any]       =						jax.random.split(__lowercase						,				jax.device_count() )

        __UpperCAmelCase    :						List[Any]       =						sd_pipe(__lowercase						,				__lowercase						,				__lowercase						,				num_inference_steps=25						,				jit=__lowercase )[0]
        assert images.shape == (jax.device_count(), 1, 768, 768, 3)

        __UpperCAmelCase    :						Tuple       =						images.reshape((images.shape[0] * images.shape[1],) + images.shape[-3:] )
        __UpperCAmelCase    :						Any       =						images[0, 253:256, 253:256, -1]

        __UpperCAmelCase    :						List[str]       =						jnp.asarray(jax.device_get(image_slice.flatten() ) )
        __UpperCAmelCase    :						Any       =						jnp.array([0.4_3_3_6, 0.4_2_9_6_9, 0.4_4_5_3, 0.4_1_9_9, 0.4_2_9_7, 0.4_5_3_1, 0.4_4_3_4, 0.4_4_3_4, 0.4_2_9_7] )
        print(f'''output_slice: {output_slice}''' )
        assert jnp.abs(output_slice - expected_slice ).max() < 1e-2