






import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
A_						:						Tuple						      =					logging.getLogger()

A_						:						Any						      =					logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)




class 							_lowerCAmelCase(			UpperCAmelCase_   ):





       """simple docstring"""







       def      _a       (   self    ,     _lowerCamelCase				):
         os.makedirs(_lowerCamelCase    ,     exist_ok=_lowerCamelCase				)
         UpperCamelCase_:   Tuple        =							{'source': 'What is love ?', 'target': 'life'}
         UpperCamelCase_:   Any        =							{'train': 1_2, 'val': 2, 'test': 2}
         for split in ["train", "test", "val"]:
           for field in ["source", "target"]:
             UpperCamelCase_:   List[Any]        =							'\n'.join([contents[field]] * n_lines[split]				)
             with open(os.path.join(_lowerCamelCase    ,     f'''{split}.{field}'''				)    ,     'w'				) as f:
               f.write(_lowerCamelCase				)





       def      _a       (   self    ,     _lowerCamelCase    ,     _lowerCamelCase = "pytorch"				):
         UpperCamelCase_:   int        =							self.get_auto_remove_tmp_dir()
         UpperCamelCase_:   Union[str, Any]        =							os.path.join(_lowerCamelCase    ,     'output'				)
         UpperCamelCase_:   Union[str, Any]        =							os.path.join(_lowerCamelCase    ,     'data'				)
         self._create_dummy_data(data_dir=_lowerCamelCase				)

         UpperCamelCase_:   Optional[Any]        =							f'''
                --data_dir {data_dir} \
                --output_dir {output_dir} \
                --model_name_or_path facebook/rag-sequence-base \
                --model_type rag_sequence \
                --do_train \
                --do_predict \
                --n_val -1 \
                --val_check_interval 1.0 \
                --train_batch_size 2 \
                --eval_batch_size 1 \
                --max_source_length 25 \
                --max_target_length 25 \
                --val_max_target_length 25 \
                --test_max_target_length 25 \
                --label_smoothing 0.1 \
                --dropout 0.1 \
                --attention_dropout 0.1 \
                --weight_decay 0.001 \
                --adam_epsilon 1e-08 \
                --max_grad_norm 0.1 \
                --lr_scheduler polynomial \
                --learning_rate 3e-04 \
                --num_train_epochs 1 \
                --warmup_steps 4 \
                --gradient_accumulation_steps 1 \
                --distributed-port 8787 \
                --use_dummy_dataset 1 \
                --distributed_retriever {distributed_retriever} \
            '''.split()

         if gpus > 0:
           testargs.append(f'''--gpus={gpus}'''				)
           if is_apex_available():
             testargs.append('--fp16'				)
         else:
           testargs.append('--gpus=0'				)
           testargs.append('--distributed_backend=ddp_cpu'				)
           testargs.append('--num_processes=2'				)

         UpperCamelCase_:   str        =							[sys.executable, str(Path(finetune_rag.__file__				).resolve()				)] + testargs
         execute_subprocess_async(_lowerCamelCase    ,     env=self.get_env()				)

         UpperCamelCase_:   Optional[Any]        =							os.path.join(_lowerCamelCase    ,     'metrics.json'				)
         with open(_lowerCamelCase				) as f:
           UpperCamelCase_:   Union[str, Any]        =							json.load(_lowerCamelCase				)
         return result





       @require_torch_gpu
       def      _a       (   self				):
         UpperCamelCase_:   Any        =							self._run_finetune(gpus=1				)
         self.assertGreaterEqual(result['test'][0]['test_avg_em']    ,     0.2				)





       @require_torch_multi_gpu
       def      _a       (   self				):
         UpperCamelCase_:   Optional[int]        =							self._run_finetune(gpus=2				)
         self.assertGreaterEqual(result['test'][0]['test_avg_em']    ,     0.2				)





       @require_torch_gpu
       @require_ray
       def      _a       (   self				):
         UpperCamelCase_:   Dict        =							self._run_finetune(gpus=1    ,     distributed_retriever='ray'				)
         self.assertGreaterEqual(result['test'][0]['test_avg_em']    ,     0.2				)





       @require_torch_multi_gpu
       @require_ray
       def      _a       (   self				):
         UpperCamelCase_:   Optional[Any]        =							self._run_finetune(gpus=1    ,     distributed_retriever='ray'				)
         self.assertGreaterEqual(result['test'][0]['test_avg_em']    ,     0.2				)



import re
import string

import numpy as np

import datasets


__UpperCamelCase     		=							'\nReturns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.\n'

__UpperCamelCase     		=							'\nArgs:\n    predictions: List of predicted texts.\n    references: List of reference texts.\n    regexes_to_ignore: List, defaults to None. Regex expressions of characters to\n        ignore when calculating the exact matches. Note: these regexes are removed\n        from the input data before the changes based on the options below (e.g. ignore_case,\n        ignore_punctuation, ignore_numbers) are applied.\n    ignore_case: Boolean, defaults to False. If true, turns everything\n        to lowercase so that capitalization differences are ignored.\n    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\n    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\nReturns:\n    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.\nExamples:\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    25.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    50.0\n\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    75.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)\n    >>> print(round(results["exact_match"], 1))\n    100.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It\'s like comparing oranges and apples."]\n    >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It\'s like comparing apples and oranges."]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    33.3\n\n'

__UpperCamelCase     		=							'\n'







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION       ,						_KWARGS_DESCRIPTION   )
class 							lowerCamelCase__						(				datasets.Metric   ):






			"""simple docstring"""






			def 					snake_case__					(				self  ):

						'''simple docstring'''


						return datasets.MetricInfo(
						    description=_DESCRIPTION   ,    citation=_CITATION   ,    inputs_description=_KWARGS_DESCRIPTION   ,    features=datasets.Features(
						        {
						            "predictions": datasets.Value("string"   ,    id="sequence"  ),
						            "references": datasets.Value("string"   ,    id="sequence"  ),
						        }  )   ,    reference_urls=[]   ,    )







			def 					snake_case__					(				self   ,    snake_case   ,    snake_case   ,    snake_case=None   ,    snake_case=False   ,    snake_case=False   ,    snake_case=False   ,    ):

						'''simple docstring'''


						if regexes_to_ignore is not None:
									for s in regexes_to_ignore:
												UpperCamelCase__       =							np.array([re.sub(snake_case   ,    ""   ,    snake_case  ) for x in predictions]  )
												UpperCamelCase__       =							np.array([re.sub(snake_case   ,    ""   ,    snake_case  ) for x in references]  )
						else:
									UpperCamelCase__       =							np.asarray(snake_case  )
									UpperCamelCase__       =							np.asarray(snake_case  )

						if ignore_case:
									UpperCamelCase__       =							np.char.lower(snake_case  )
									UpperCamelCase__       =							np.char.lower(snake_case  )

						if ignore_punctuation:
									UpperCamelCase__       =							string.punctuation.maketrans(""   ,    ""   ,    string.punctuation  )
									UpperCamelCase__       =							np.char.translate(snake_case   ,    table=snake_case  )
									UpperCamelCase__       =							np.char.translate(snake_case   ,    table=snake_case  )

						if ignore_numbers:
									UpperCamelCase__       =							string.digits.maketrans(""   ,    ""   ,    string.digits  )
									UpperCamelCase__       =							np.char.translate(snake_case   ,    table=snake_case  )
									UpperCamelCase__       =							np.char.translate(snake_case   ,    table=snake_case  )

						UpperCamelCase__       =							predictions == references

						return {"exact_match": np.mean(snake_case  ) * 100}


