

import argparse
import json
import logging
import os
import shutil
import sys
import tempfile
import unittest
from unittest import mock

import torch
from accelerate.utils import write_basic_config

from transformers.testing_utils import TestCasePlus, get_gpu_count, run_command, slow, torch_device
from transformers.utils import is_apex_available


logging.basicConfig(level=logging.DEBUG)

lowerCamelCase      				=     logging.getLogger()


def        __lowercase    (  ):

					"""simple docstring"""






					__lowerCAmelCase					=							argparse.ArgumentParser()
					parser.add_argument('-f'      )
					__lowerCAmelCase					=							parser.parse_args()
					return args.f


def        __lowercase    (  UpperCAmelCase__      ):

					"""simple docstring"""






					__lowerCAmelCase					=							{}
					__lowerCAmelCase					=							os.path.join(__snake_case		,					'all_results.json'      )
					if os.path.exists(__snake_case      ):
										with open(__snake_case		,					'r'      ) as f:
															__lowerCAmelCase					=							json.load(__snake_case      )
					else:
										raise ValueError(F"""can\'t find {path}"""      )
					return results


def        __lowercase    (  ):

					"""simple docstring"""






					__lowerCAmelCase					=							torch.cuda.is_available() and torch_device == 'cuda'
					return is_using_cuda and is_apex_available()


lowerCamelCase      				=     logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)





class     snake_case_			(				_a							):


							"""simple docstring"""





							@classmethod
							def        A__  ( cls							):
												__lowerCAmelCase					=							tempfile.mkdtemp()
												__lowerCAmelCase					=							os.path.join(cls.tmpdir					,						'default_config.yml'							)
												write_basic_config(save_location=cls.configPath							)
												__lowerCAmelCase					=							['accelerate', 'launch', '--config_file', cls.configPath]




							@classmethod
							def        A__  ( cls							):
												shutil.rmtree(cls.tmpdir							)




							@mock.patch.dict(os.environ					,						{'WANDB_MODE': 'offline'}							)
							def        A__  ( self							):
												__lowerCAmelCase					=							self.get_auto_remove_tmp_dir()
												__lowerCAmelCase					=							F"""
            {self.examples_dir}/pytorch/text-classification/run_glue_no_trainer.py
            --model_name_or_path distilbert-base-uncased
            --output_dir {tmp_dir}
            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv
            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --learning_rate=1e-4
            --seed=42
            --checkpointing_steps epoch
            --with_tracking
        """.split()

												if is_cuda_and_apex_available():
																	testargs.append('--fp16'							)

												run_command(self._launch_args + testargs							)
												__lowerCAmelCase					=							get_results(__a							)
												self.assertGreaterEqual(result['eval_accuracy']					,						0.75							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'epoch_0'							)							)							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'glue_no_trainer'							)							)							)




							@mock.patch.dict(os.environ					,						{'WANDB_MODE': 'offline'}							)
							def        A__  ( self							):
												__lowerCAmelCase					=							self.get_auto_remove_tmp_dir()
												__lowerCAmelCase					=							F"""
            {self.examples_dir}/pytorch/language-modeling/run_clm_no_trainer.py
            --model_name_or_path distilgpt2
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --block_size 128
            --per_device_train_batch_size 5
            --per_device_eval_batch_size 5
            --num_train_epochs 2
            --output_dir {tmp_dir}
            --checkpointing_steps epoch
            --with_tracking
        """.split()

												if torch.cuda.device_count() > 1:
																	# Skipping because there are not enough batches to train the model + would need a drop_last to work.
																	return

												run_command(self._launch_args + testargs							)
												__lowerCAmelCase					=							get_results(__a							)
												self.assertLess(result['perplexity']					,						1_0_0							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'epoch_0'							)							)							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'clm_no_trainer'							)							)							)




							@mock.patch.dict(os.environ					,						{'WANDB_MODE': 'offline'}							)
							def        A__  ( self							):
												__lowerCAmelCase					=							self.get_auto_remove_tmp_dir()
												__lowerCAmelCase					=							F"""
            {self.examples_dir}/pytorch/language-modeling/run_mlm_no_trainer.py
            --model_name_or_path distilroberta-base
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --output_dir {tmp_dir}
            --num_train_epochs=1
            --checkpointing_steps epoch
            --with_tracking
        """.split()

												run_command(self._launch_args + testargs							)
												__lowerCAmelCase					=							get_results(__a							)
												self.assertLess(result['perplexity']					,						4_2							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'epoch_0'							)							)							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'mlm_no_trainer'							)							)							)




							@mock.patch.dict(os.environ					,						{'WANDB_MODE': 'offline'}							)
							def        A__  ( self							):
												__lowerCAmelCase					=							7 if get_gpu_count() > 1 else 2

												__lowerCAmelCase					=							self.get_auto_remove_tmp_dir()
												__lowerCAmelCase					=							F"""
            {self.examples_dir}/pytorch/token-classification/run_ner_no_trainer.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/conll/sample.json
            --validation_file tests/fixtures/tests_samples/conll/sample.json
            --output_dir {tmp_dir}
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=2
            --num_train_epochs={epochs}
            --seed 7
            --checkpointing_steps epoch
            --with_tracking
        """.split()

												run_command(self._launch_args + testargs							)
												__lowerCAmelCase					=							get_results(__a							)
												self.assertGreaterEqual(result['eval_accuracy']					,						0.75							)
												self.assertLess(result['train_loss']					,						0.5							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'epoch_0'							)							)							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'ner_no_trainer'							)							)							)




							@unittest.skip(reason='Fix me @muellerzr'							)
							@mock.patch.dict(os.environ					,						{'WANDB_MODE': 'offline'}							)
							def        A__  ( self							):
												__lowerCAmelCase					=							self.get_auto_remove_tmp_dir()
												__lowerCAmelCase					=							F"""
            {self.examples_dir}/pytorch/question-answering/run_qa_no_trainer.py
            --model_name_or_path bert-base-uncased
            --version_2_with_negative
            --train_file tests/fixtures/tests_samples/SQUAD/sample.json
            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json
            --output_dir {tmp_dir}
            --seed=42
            --max_train_steps=10
            --num_warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --checkpointing_steps epoch
            --with_tracking
        """.split()

												run_command(self._launch_args + testargs							)
												__lowerCAmelCase					=							get_results(__a							)
												# Because we use --version_2_with_negative the testing script uses SQuAD v2 metrics.
												self.assertGreaterEqual(result['eval_f1']					,						2_8							)
												self.assertGreaterEqual(result['eval_exact']					,						2_8							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'epoch_0'							)							)							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'qa_no_trainer'							)							)							)




							@mock.patch.dict(os.environ					,						{'WANDB_MODE': 'offline'}							)
							def        A__  ( self							):
												__lowerCAmelCase					=							self.get_auto_remove_tmp_dir()
												__lowerCAmelCase					=							F"""
            {self.examples_dir}/pytorch/multiple-choice/run_swag_no_trainer.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/swag/sample.json
            --validation_file tests/fixtures/tests_samples/swag/sample.json
            --output_dir {tmp_dir}
            --max_train_steps=20
            --num_warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --with_tracking
        """.split()

												run_command(self._launch_args + testargs							)
												__lowerCAmelCase					=							get_results(__a							)
												self.assertGreaterEqual(result['eval_accuracy']					,						0.8							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'swag_no_trainer'							)							)							)




							@slow
							@mock.patch.dict(os.environ					,						{'WANDB_MODE': 'offline'}							)
							def        A__  ( self							):
												__lowerCAmelCase					=							self.get_auto_remove_tmp_dir()
												__lowerCAmelCase					=							F"""
            {self.examples_dir}/pytorch/summarization/run_summarization_no_trainer.py
            --model_name_or_path t5-small
            --train_file tests/fixtures/tests_samples/xsum/sample.json
            --validation_file tests/fixtures/tests_samples/xsum/sample.json
            --output_dir {tmp_dir}
            --max_train_steps=50
            --num_warmup_steps=8
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --checkpointing_steps epoch
            --with_tracking
        """.split()

												run_command(self._launch_args + testargs							)
												__lowerCAmelCase					=							get_results(__a							)
												self.assertGreaterEqual(result['eval_rouge1']					,						1_0							)
												self.assertGreaterEqual(result['eval_rouge2']					,						2							)
												self.assertGreaterEqual(result['eval_rougeL']					,						7							)
												self.assertGreaterEqual(result['eval_rougeLsum']					,						7							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'epoch_0'							)							)							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'summarization_no_trainer'							)							)							)




							@slow
							@mock.patch.dict(os.environ					,						{'WANDB_MODE': 'offline'}							)
							def        A__  ( self							):
												__lowerCAmelCase					=							self.get_auto_remove_tmp_dir()
												__lowerCAmelCase					=							F"""
            {self.examples_dir}/pytorch/translation/run_translation_no_trainer.py
            --model_name_or_path sshleifer/student_marian_en_ro_6_1
            --source_lang en
            --target_lang ro
            --train_file tests/fixtures/tests_samples/wmt16/sample.json
            --validation_file tests/fixtures/tests_samples/wmt16/sample.json
            --output_dir {tmp_dir}
            --max_train_steps=50
            --num_warmup_steps=8
            --num_beams=6
            --learning_rate=3e-3
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --source_lang en_XX
            --target_lang ro_RO
            --checkpointing_steps epoch
            --with_tracking
        """.split()

												run_command(self._launch_args + testargs							)
												__lowerCAmelCase					=							get_results(__a							)
												self.assertGreaterEqual(result['eval_bleu']					,						3_0							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'epoch_0'							)							)							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'translation_no_trainer'							)							)							)




							@slow
							def        A__  ( self							):
												__lowerCAmelCase					=							logging.StreamHandler(sys.stdout							)
												logger.addHandler(__a							)

												__lowerCAmelCase					=							self.get_auto_remove_tmp_dir()
												__lowerCAmelCase					=							F"""
            {self.examples_dir}/pytorch/semantic-segmentation/run_semantic_segmentation_no_trainer.py
            --dataset_name huggingface/semantic-segmentation-test-sample
            --output_dir {tmp_dir}
            --max_train_steps=10
            --num_warmup_steps=2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --checkpointing_steps epoch
        """.split()

												run_command(self._launch_args + testargs							)
												__lowerCAmelCase					=							get_results(__a							)
												self.assertGreaterEqual(result['eval_overall_accuracy']					,						0.10							)




							@mock.patch.dict(os.environ					,						{'WANDB_MODE': 'offline'}							)
							def        A__  ( self							):
												__lowerCAmelCase					=							self.get_auto_remove_tmp_dir()
												__lowerCAmelCase					=							F"""
            {self.examples_dir}/pytorch/image-classification/run_image_classification_no_trainer.py
            --model_name_or_path google/vit-base-patch16-224-in21k
            --dataset_name hf-internal-testing/cats_vs_dogs_sample
            --learning_rate 1e-4
            --per_device_train_batch_size 2
            --per_device_eval_batch_size 1
            --max_train_steps 2
            --train_val_split 0.1
            --seed 42
            --output_dir {tmp_dir}
            --with_tracking
            --checkpointing_steps 1
        """.split()

												if is_cuda_and_apex_available():
																	testargs.append('--fp16'							)

												run_command(self._launch_args + testargs							)
												__lowerCAmelCase					=							get_results(__a							)
												# The base model scores a 25%
												self.assertGreaterEqual(result['eval_accuracy']					,						0.6							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'step_1'							)							)							)
												self.assertTrue(os.path.exists(os.path.join(__a					,						'image_classification_no_trainer'							)							)							)









import argparse
import re
from typing import Dict

import torch
from datasets import Audio, Dataset, load_dataset, load_metric

from transformers import AutoFeatureExtractor, pipeline


def        __lowercase    (  UpperCAmelCase__		,					UpperCAmelCase__      ):

					"""simple docstring"""






					__lowerCAmelCase					=							args.log_outputs
					__lowerCAmelCase					=							'_'.join(args.dataset.split('/'      ) + [args.config, args.split]      )

					# load metric
					__lowerCAmelCase					=							load_metric('wer'      )
					__lowerCAmelCase					=							load_metric('cer'      )

					# compute metrics
					__lowerCAmelCase					=							wer.compute(references=result['target']		,					predictions=result['prediction']      )
					__lowerCAmelCase					=							cer.compute(references=result['target']		,					predictions=result['prediction']      )

					# print & log results
					__lowerCAmelCase					=							F"""WER: {wer_result}\nCER: {cer_result}"""
					print(UpperCAmelCase__      )

					with open(F"""{dataset_id}_eval_results.txt"""		,					'w'      ) as f:
										f.write(UpperCAmelCase__      )

					# log all results in text file. Possibly interesting for analysis
					if log_outputs is not None:
										__lowerCAmelCase					=							F"""log_{dataset_id}_predictions.txt"""
										__lowerCAmelCase					=							F"""log_{dataset_id}_targets.txt"""

										with open(UpperCAmelCase__		,					'w'      ) as p, open(UpperCAmelCase__		,					'w'      ) as t:
															# mapping function to write output
															def write_to_file(UpperCAmelCase__		,					UpperCAmelCase__      ):
																				p.write(F"""{i}""" + '\n'      )
																				p.write(batch['prediction'] + '\n'      )
																				t.write(F"""{i}""" + '\n'      )
																				t.write(batch['target'] + '\n'      )

															result.map(UpperCAmelCase__		,					with_indices=UpperCAmelCase__      )


def        __lowercase    (  UpperCAmelCase__      ):

					"""simple docstring"""






					__lowerCAmelCase					=							'[,?.!\-\;\:"“%‘”�—’…–]'  # noqa: W605 IMPORTANT: this should correspond to the chars that were ignored during training

					__lowerCAmelCase					=							re.sub(UpperCAmelCase__		,					''		,					text.lower()      )

					# In addition, we can normalize the target text, e.g. removing new lines characters etc...
					# note that order is important here!
					__lowerCAmelCase					=							['\n\n', '\n', '   ', '  ']

					for t in token_sequences_to_ignore:
										__lowerCAmelCase					=							' '.join(text.split(UpperCAmelCase__      )      )

					return text


def        __lowercase    (  UpperCAmelCase__      ):

					"""simple docstring"""






					__lowerCAmelCase					=							load_dataset(args.dataset		,					args.config		,					split=args.split		,					use_auth_token=UpperCAmelCase__      )

					# for testing: only process the first two examples as a test
					# dataset = dataset.select(range(10))

					# load processor
					__lowerCAmelCase					=							AutoFeatureExtractor.from_pretrained(args.model_id      )
					__lowerCAmelCase					=							feature_extractor.sampling_rate

					# resample audio
					__lowerCAmelCase					=							dataset.cast_column('audio'		,					Audio(sampling_rate=UpperCAmelCase__      )      )

					# load eval pipeline
					if args.device is None:
										__lowerCAmelCase					=							0 if torch.cuda.is_available() else -1
					__lowerCAmelCase					=							pipeline('automatic-speech-recognition'		,					model=args.model_id		,					device=args.device      )

					# map function to decode audio
					def map_to_pred(UpperCAmelCase__      ):
										__lowerCAmelCase					=							asr(
										    batch['audio']['array']		,					chunk_length_s=args.chunk_length_s		,					stride_length_s=args.stride_length_s      )

										__lowerCAmelCase					=							prediction['text']
										__lowerCAmelCase					=							normalize_text(batch['sentence']      )
										return batch

					# run inference on all examples
					__lowerCAmelCase					=							dataset.map(UpperCAmelCase__		,					remove_columns=dataset.column_names      )

					# compute and log_results
					# do not change function below
					log_results(UpperCAmelCase__		,					UpperCAmelCase__      )


if __name__ == "__main__":
		lowerCamelCase      				=     argparse.ArgumentParser()

		parser.add_argument(
		    '''--model_id''', type=str, required=True, help='''Model identifier. Should be loadable with 🤗 Transformers'''
		)
		parser.add_argument(
		    '''--dataset''',
		    type=str,
		    required=True,
		    help='''Dataset name to evaluate the `model_id`. Should be loadable with 🤗 Datasets''',
		)
		parser.add_argument(
		    '''--config''', type=str, required=True, help='''Config of the dataset. *E.g.* `\'en\'`  for Common Voice'''
		)
		parser.add_argument('''--split''', type=str, required=True, help='''Split of the dataset. *E.g.* `\'test\'`''')
		parser.add_argument(
		    '''--chunk_length_s''', type=float, default=None, help='''Chunk length in seconds. Defaults to 5 seconds.'''
		)
		parser.add_argument(
		    '''--stride_length_s''', type=float, default=None, help='''Stride of the audio chunks. Defaults to 1 second.'''
		)
		parser.add_argument(
		    '''--log_outputs''', action='''store_true''', help='''If defined, write outputs to log file for analysis.'''
		)
		parser.add_argument(
		    '''--device''',
		    type=int,
		    default=None,
		    help='''The device to run the pipeline on. -1 for CPU (default), 0 for the first GPU and so on.''',
		)
		lowerCamelCase      				=     parser.parse_args()

		main(args)






