






import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
_SCREAMING_SNAKE_CASE   =  logging.getLogger()

_SCREAMING_SNAKE_CASE   =  logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)






class 					SCREAMING_SNAKE_CASE_  (       lowercase__ ):
       """simple docstring"""


       def 				UpperCamelCase__						(				self	:str,      snake_case	:Optional[Any]):



          """simple docstring"""

          os.makedirs(UpperCAmelCase__,      exist_ok=UpperCAmelCase__)
          _lowercase              ={'''source''': '''What is love ?''', '''target''': '''life'''}
          _lowercase              ={'''train''': 12, '''val''': 2, '''test''': 2}
          for split in ["train", "test", "val"]:
             for field in ["source", "target"]:
                _lowercase              ='''\n'''.join([contents[field]] * n_lines[split])
                with open(os.path.join(UpperCAmelCase__,      f'''{split}.{field}'''),      'w') as f:
                   f.write(UpperCAmelCase__)





       def 				UpperCamelCase__						(				self	:Union[str, Any],      snake_case	:int,      snake_case	:str = "pytorch"):



          """simple docstring"""

          _lowercase              =self.get_auto_remove_tmp_dir()
          _lowercase              =os.path.join(UpperCAmelCase__,      'output')
          _lowercase              =os.path.join(UpperCAmelCase__,      'data')
          self._create_dummy_data(data_dir=UpperCAmelCase__)

          _lowercase              =f'''
                --data_dir {data_dir} \
                --output_dir {output_dir} \
                --model_name_or_path facebook/rag-sequence-base \
                --model_type rag_sequence \
                --do_train \
                --do_predict \
                --n_val -1 \
                --val_check_interval 1.0 \
                --train_batch_size 2 \
                --eval_batch_size 1 \
                --max_source_length 25 \
                --max_target_length 25 \
                --val_max_target_length 25 \
                --test_max_target_length 25 \
                --label_smoothing 0.1 \
                --dropout 0.1 \
                --attention_dropout 0.1 \
                --weight_decay 0.001 \
                --adam_epsilon 1e-08 \
                --max_grad_norm 0.1 \
                --lr_scheduler polynomial \
                --learning_rate 3e-04 \
                --num_train_epochs 1 \
                --warmup_steps 4 \
                --gradient_accumulation_steps 1 \
                --distributed-port 8787 \
                --use_dummy_dataset 1 \
                --distributed_retriever {distributed_retriever} \
            '''.split()

          if gpus > 0:
             testargs.append(f'''--gpus={gpus}''')
             if is_apex_available():
                testargs.append('--fp16')
          else:
             testargs.append('--gpus=0')
             testargs.append('--distributed_backend=ddp_cpu')
             testargs.append('--num_processes=2')

          _lowercase              =[sys.executable, str(Path(finetune_rag.__file__).resolve())] + testargs
          execute_subprocess_async(UpperCAmelCase__,      env=self.get_env())

          _lowercase              =os.path.join(UpperCAmelCase__,      'metrics.json')
          with open(UpperCAmelCase__) as f:
             _lowercase              =json.load(UpperCAmelCase__)
          return result





       @require_torch_gpu
       def 				UpperCamelCase__						(				self	:Optional[int]):



          """simple docstring"""

          _lowercase              =self._run_finetune(gpus=1)
          self.assertGreaterEqual(result['test'][0]['test_avg_em'],      0.2)





       @require_torch_multi_gpu
       def 				UpperCamelCase__						(				self	:List[Any]):



          """simple docstring"""

          _lowercase              =self._run_finetune(gpus=2)
          self.assertGreaterEqual(result['test'][0]['test_avg_em'],      0.2)





       @require_torch_gpu
       @require_ray
       def 				UpperCamelCase__						(				self	:Dict):



          """simple docstring"""

          _lowercase              =self._run_finetune(gpus=1,      distributed_retriever='ray')
          self.assertGreaterEqual(result['test'][0]['test_avg_em'],      0.2)





       @require_torch_multi_gpu
       @require_ray
       def 				UpperCamelCase__						(				self	:int):



          """simple docstring"""

          _lowercase              =self._run_finetune(gpus=1,      distributed_retriever='ray')
          self.assertGreaterEqual(result['test'][0]['test_avg_em'],      0.2)





'''simple docstring'''





import datasets


UpperCamelCase_    						=       """\
@InProceedings{conneau2018xnli,
  author = \"Conneau, Alexis
                 and Rinott, Ruty
                 and Lample, Guillaume
                 and Williams, Adina
                 and Bowman, Samuel R.
                 and Schwenk, Holger
                 and Stoyanov, Veselin\",
  title = \"XNLI: Evaluating Cross-lingual Sentence Representations\",
  booktitle = \"Proceedings of the 2018 Conference on Empirical Methods
               in Natural Language Processing\",
  year = \"2018\",
  publisher = \"Association for Computational Linguistics\",
  location = \"Brussels, Belgium\",
}
"""

UpperCamelCase_    						=       """\
XNLI is a subset of a few thousand examples from MNLI which has been translated
into a 14 different languages (some low-ish resource). As with MNLI, the goal is
to predict textual entailment (does sentence A imply/contradict/neither sentence
B) and is a classification task (given two sentences, predict one of three
labels).
"""

UpperCamelCase_    						=       """
Computes XNLI score which is just simple accuracy.
Args:
    predictions: Predicted labels.
    references: Ground truth labels.
Returns:
    'accuracy': accuracy
Examples:

    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> xnli_metric = datasets.load_metric(\"xnli\")
    >>> results = xnli_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}
"""






def 				_lowerCAmelCase			(   __magic_name__						:     Dict   , __magic_name__						:     Union[str, Any]						)				->  Union[str, Any]:
						return (preds == labels).mean()







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,							_KWARGS_DESCRIPTION      )
class 		__SCREAMING_SNAKE_CASE							(			datasets.Metric      ):







				def  lowerCamelCase_	(       self     :		str   ):
										'''simple docstring'''
										return datasets.MetricInfo(
										    description=_DESCRIPTION		,					citation=_CITATION		,					inputs_description=_KWARGS_DESCRIPTION		,					features=datasets.Features(
										        {
										            '''predictions''': datasets.Value('''int64''' if self.config_name != '''sts-b''' else '''float32'''   ),
										            '''references''': datasets.Value('''int64''' if self.config_name != '''sts-b''' else '''float32'''   ),
										        }   )		,					codebase_urls=[]		,					reference_urls=[]		,					format='''numpy'''		,					)







				def  lowerCamelCase_	(       self     :		int		,					UpperCAmelCase__     :		Dict		,					UpperCAmelCase__     :		Optional[Any]   ):
										'''simple docstring'''
										return {"accuracy": simple_accuracy(UpperCAmelCase__		,					UpperCAmelCase__   )}





