





import argparse
import json
import logging
import os
import sys
from unittest.mock import patch

from transformers.testing_utils import TestCasePlus, get_gpu_count, slow


lowercase    : Union[str, Any]    	=  [
    os.path.join(os.path.dirname(__file__), dirname)
    for dirname in [
        """text-classification""",
        """language-modeling""",
        """summarization""",
        """token-classification""",
        """question-answering""",
    ]
]
sys.path.extend(SRC_DIRS)


if SRC_DIRS is not None:
       import run_clm_flax
       import run_flax_glue
       import run_flax_ner
       import run_mlm_flax
       import run_qa
       import run_summarization_flax
       import run_ta_mlm_flax


logging.basicConfig(level=logging.DEBUG)

lowercase    : List[Any]    	=  logging.getLogger()


def     SCREAMING_SNAKE_CASE			(      ):
       _UpperCamelCase              =						argparse.ArgumentParser()
       parser.add_argument('''-f'''					)
       _UpperCamelCase              =						parser.parse_args()
       return args.f


def     SCREAMING_SNAKE_CASE			(      lowerCAmelCase    ,       lowerCAmelCase="eval"					):
       _UpperCamelCase              =						os.path.join(lowerCAmelCase    ,       f'''{split}_results.json'''					)
       if os.path.exists(lowerCAmelCase					):
              with open(lowerCAmelCase    ,       '''r'''					) as f:
                     return json.load(lowerCAmelCase					)
       raise ValueError(f'''can\'t find {path}'''					)


lowercase    : Union[str, Any]    	=  logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)

class  __A(						UpperCamelCase_ ):

     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self.get_auto_remove_tmp_dir()
            _UpperCamelCase              =						F'''
            run_glue.py
            --model_name_or_path distilbert-base-uncased
            --output_dir {tmp_dir}
            --train_file ./tests/fixtures/tests_samples/MRPC/train.csv
            --validation_file ./tests/fixtures/tests_samples/MRPC/dev.csv
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --learning_rate=1e-4
            --eval_steps=2
            --warmup_steps=2
            --seed=42
            --max_seq_length=128
            '''.split()

            with patch.object(__a,       '''argv''',       __a      ):
                   run_flax_glue.main()
                   _UpperCamelCase              =						get_results(__a      )
                   self.assertGreaterEqual(result['''eval_accuracy'''],       0.75      )

     @slow
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self.get_auto_remove_tmp_dir()
            _UpperCamelCase              =						F'''
            run_clm_flax.py
            --model_name_or_path distilgpt2
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --do_train
            --do_eval
            --block_size 128
            --per_device_train_batch_size 4
            --per_device_eval_batch_size 4
            --num_train_epochs 2
            --logging_steps 2 --eval_steps 2
            --output_dir {tmp_dir}
            --overwrite_output_dir
            '''.split()

            with patch.object(__a,       '''argv''',       __a      ):
                   run_clm_flax.main()
                   _UpperCamelCase              =						get_results(__a      )
                   self.assertLess(result['''eval_perplexity'''],       100      )

     @slow
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self.get_auto_remove_tmp_dir()
            _UpperCamelCase              =						F'''
            run_summarization.py
            --model_name_or_path t5-small
            --train_file tests/fixtures/tests_samples/xsum/sample.json
            --validation_file tests/fixtures/tests_samples/xsum/sample.json
            --test_file tests/fixtures/tests_samples/xsum/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --num_train_epochs=3
            --warmup_steps=8
            --do_train
            --do_eval
            --do_predict
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
            --predict_with_generate
        '''.split()

            with patch.object(__a,       '''argv''',       __a      ):
                   run_summarization_flax.main()
                   _UpperCamelCase              =						get_results(__a,       split='''test'''      )
                   self.assertGreaterEqual(result['''test_rouge1'''],       10      )
                   self.assertGreaterEqual(result['''test_rouge2'''],       2      )
                   self.assertGreaterEqual(result['''test_rougeL'''],       7      )
                   self.assertGreaterEqual(result['''test_rougeLsum'''],       7      )

     @slow
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self.get_auto_remove_tmp_dir()
            _UpperCamelCase              =						F'''
            run_mlm.py
            --model_name_or_path distilroberta-base
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --max_seq_length 128
            --per_device_train_batch_size 4
            --per_device_eval_batch_size 4
            --logging_steps 2 --eval_steps 2
            --do_train
            --do_eval
            --num_train_epochs=1
        '''.split()

            with patch.object(__a,       '''argv''',       __a      ):
                   run_mlm_flax.main()
                   _UpperCamelCase              =						get_results(__a      )
                   self.assertLess(result['''eval_perplexity'''],       42      )

     @slow
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self.get_auto_remove_tmp_dir()
            _UpperCamelCase              =						F'''
            run_t5_mlm_flax.py
            --model_name_or_path t5-small
            --train_file ./tests/fixtures/sample_text.txt
            --validation_file ./tests/fixtures/sample_text.txt
            --do_train
            --do_eval
            --max_seq_length 128
            --per_device_train_batch_size 4
            --per_device_eval_batch_size 4
            --num_train_epochs 2
            --logging_steps 2 --eval_steps 2
            --output_dir {tmp_dir}
            --overwrite_output_dir
            '''.split()

            with patch.object(__a,       '''argv''',       __a      ):
                   run_ta_mlm_flax.main()
                   _UpperCamelCase              =						get_results(__a      )
                   self.assertGreaterEqual(result['''eval_accuracy'''],       0.42      )

     @slow
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						7 if get_gpu_count() > 1 else 2

            _UpperCamelCase              =						self.get_auto_remove_tmp_dir()
            _UpperCamelCase              =						F'''
            run_flax_ner.py
            --model_name_or_path bert-base-uncased
            --train_file tests/fixtures/tests_samples/conll/sample.json
            --validation_file tests/fixtures/tests_samples/conll/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --do_train
            --do_eval
            --warmup_steps=2
            --learning_rate=2e-4
            --logging_steps 2 --eval_steps 2
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=2
            --num_train_epochs={epochs}
            --seed 7
        '''.split()

            with patch.object(__a,       '''argv''',       __a      ):
                   run_flax_ner.main()
                   _UpperCamelCase              =						get_results(__a      )
                   self.assertGreaterEqual(result['''eval_accuracy'''],       0.75      )
                   self.assertGreaterEqual(result['''eval_f1'''],       0.3      )



     @slow
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						self.get_auto_remove_tmp_dir()
            _UpperCamelCase              =						F'''
            run_qa.py
            --model_name_or_path bert-base-uncased
            --version_2_with_negative
            --train_file tests/fixtures/tests_samples/SQUAD/sample.json
            --validation_file tests/fixtures/tests_samples/SQUAD/sample.json
            --output_dir {tmp_dir}
            --overwrite_output_dir
            --num_train_epochs=3
            --warmup_steps=2
            --do_train
            --do_eval
            --logging_steps 2 --eval_steps 2
            --learning_rate=2e-4
            --per_device_train_batch_size=2
            --per_device_eval_batch_size=1
        '''.split()

            with patch.object(__a,       '''argv''',       __a      ):
                   run_qa.main()
                   _UpperCamelCase              =						get_results(__a      )
                   self.assertGreaterEqual(result['''eval_f1'''],       30      )
                   self.assertGreaterEqual(result['''eval_exact'''],       30      )










import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TextaTextGenerationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch
from transformers.utils import is_torch_available

from .test_pipelines_common import ANY


if is_torch_available():
       import torch

@is_pipeline_test
class  __A(						unittest.TestCase ):
     __A	  =       MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
     __A	  =       TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

     def 	_UpperCamelCase     (      self,       A,       A,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						TextaTextGenerationPipeline(model=A,       tokenizer=A      )
            return generator, ["Something to write", "Something else"]

     def 	_UpperCamelCase     (      self,       A,       A      ):





            """simple docstring"""

            _UpperCamelCase              =						generator('''Something there'''      )
            self.assertEqual(A,       [{'''generated_text''': ANY(A      )}]      )
            # These are encoder decoder, they don't just append to incoming string
            self.assertFalse(outputs[0]['''generated_text'''].startswith('''Something there'''      )      )

            _UpperCamelCase              =						generator(['''This is great !''', '''Something else'''],       num_return_sequences=2,       do_sample=A      )
            self.assertEqual(
                A,       [
                    [{'''generated_text''': ANY(A      )}, {'''generated_text''': ANY(A      )}],
                    [{'''generated_text''': ANY(A      )}, {'''generated_text''': ANY(A      )}],
                ],       )

            _UpperCamelCase              =						generator(
                ['''This is great !''', '''Something else'''],       num_return_sequences=2,       batch_size=2,       do_sample=A      )
            self.assertEqual(
                A,       [
                    [{'''generated_text''': ANY(A      )}, {'''generated_text''': ANY(A      )}],
                    [{'''generated_text''': ANY(A      )}, {'''generated_text''': ANY(A      )}],
                ],       )

            with self.assertRaises(A      ):
                   generator(4      )

     @require_torch
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						pipeline('''text2text-generation''',       model='''patrickvonplaten/t5-tiny-random''',       framework='''pt'''      )
            # do_sample=False necessary for reproducibility
            _UpperCamelCase              =						generator('''Something there''',       do_sample=A      )
            self.assertEqual(A,       [{'''generated_text''': ''''''}]      )

            _UpperCamelCase              =						3
            _UpperCamelCase              =						generator(
                '''Something there''',       num_return_sequences=A,       num_beams=A,       )
            _UpperCamelCase              =						[
                {'''generated_text''': '''Beide Beide Beide Beide Beide Beide Beide Beide Beide'''},
                {'''generated_text''': '''Beide Beide Beide Beide Beide Beide Beide Beide'''},
                {'''generated_text''': ''''''},
            ]
            self.assertEqual(A,       A      )

            _UpperCamelCase              =						generator('''This is a test''',       do_sample=A,       num_return_sequences=2,       return_tensors=A      )
            self.assertEqual(
                A,       [
                    {'''generated_token_ids''': ANY(torch.Tensor      )},
                    {'''generated_token_ids''': ANY(torch.Tensor      )},
                ],       )
            _UpperCamelCase              =						generator.model.config.eos_token_id
            _UpperCamelCase              =						'''<pad>'''
            _UpperCamelCase              =						generator(
                ['''This is a test''', '''This is a second test'''],       do_sample=A,       num_return_sequences=2,       batch_size=2,       return_tensors=A,       )
            self.assertEqual(
                A,       [
                    [
                        {'''generated_token_ids''': ANY(torch.Tensor      )},
                        {'''generated_token_ids''': ANY(torch.Tensor      )},
                    ],
                    [
                        {'''generated_token_ids''': ANY(torch.Tensor      )},
                        {'''generated_token_ids''': ANY(torch.Tensor      )},
                    ],
                ],       )



     @require_tf
     def 	_UpperCamelCase     (      self      ):





            """simple docstring"""

            _UpperCamelCase              =						pipeline('''text2text-generation''',       model='''patrickvonplaten/t5-tiny-random''',       framework='''tf'''      )
            # do_sample=False necessary for reproducibility
            _UpperCamelCase              =						generator('''Something there''',       do_sample=A      )
            self.assertEqual(A,       [{'''generated_text''': ''''''}]      )






