






import numpy as np
from nltk.translate import meteor_score

import datasets
from datasets.config import importlib_metadata, version


__a:     List[Any]   					=					version.parse(importlib_metadata.version('''nltk'''))
if NLTK_VERSION >= version.Version('''3.6.4'''):
  from nltk import word_tokenize


__a:     str   					=					'''\
@inproceedings{banarjee2005,
  title     = {{METEOR}: An Automatic Metric for {MT} Evaluation with Improved Correlation with Human Judgments},
  author    = {Banerjee, Satanjeev  and Lavie, Alon},
  booktitle = {Proceedings of the {ACL} Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and/or Summarization},
  month     = jun,
  year      = {2005},
  address   = {Ann Arbor, Michigan},
  publisher = {Association for Computational Linguistics},
  url       = {https://www.aclweb.org/anthology/W05-0909},
  pages     = {65--72},
}
'''

__a:     int   					=					'''\
METEOR, an automatic metric for machine translation evaluation
that is based on a generalized concept of unigram matching between the
machine-produced translation and human-produced reference translations.
Unigrams can be matched based on their surface forms, stemmed forms,
and meanings; furthermore, METEOR can be easily extended to include more
advanced matching strategies. Once all generalized unigram matches
between the two strings have been found, METEOR computes a score for
this matching using a combination of unigram-precision, unigram-recall, and
a measure of fragmentation that is designed to directly capture how
well-ordered the matched words in the machine translation are in relation
to the reference.

METEOR gets an R correlation value of 0.347 with human evaluation on the Arabic
data and 0.331 on the Chinese data. This is shown to be an improvement on
using simply unigram-precision, unigram-recall and their harmonic F1
combination.
'''

__a:     str   					=					'''
Computes METEOR score of translated segments against one or more references.
Args:
    predictions: list of predictions to score. Each prediction
        should be a string with tokens separated by spaces.
    references: list of reference for each prediction. Each
        reference should be a string with tokens separated by spaces.
    alpha: Parameter for controlling relative weights of precision and recall. default: 0.9
    beta: Parameter for controlling shape of penalty as a function of fragmentation. default: 3
    gamma: Relative weight assigned to fragmentation penalty. default: 0.5
Returns:
    \'meteor\': meteor score.
Examples:

    >>> meteor = datasets.load_metric(\'meteor\')
    >>> predictions = [\"It is a guide to action which ensures that the military always obeys the commands of the party\"]
    >>> references = [\"It is a guide to action that ensures that the military will forever heed Party commands\"]
    >>> results = meteor.compute(predictions=predictions, references=references)
    >>> print(round(results[\"meteor\"], 4))
    0.6944
'''
@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,      _KWARGS_DESCRIPTION     )
class 		SCREAMING_SNAKE_CASE__	(  datasets.Metric     ):

  '''simple docstring'''







  def  lowerCamelCase       (      self		:			Any							)						->	str:


         """simple docstring"""
         return datasets.MetricInfo(
             description=_DESCRIPTION       ,				citation=_CITATION       ,				inputs_description=_KWARGS_DESCRIPTION       ,				features=datasets.Features(
                 {
                     """predictions""": datasets.Value("""string"""       ,				id="""sequence"""							),
                     """references""": datasets.Value("""string"""       ,				id="""sequence"""							),
                 }							)       ,				codebase_urls=["""https://github.com/nltk/nltk/blob/develop/nltk/translate/meteor_score.py"""]       ,				reference_urls=[
                 """https://www.nltk.org/api/nltk.translate.html#module-nltk.translate.meteor_score""",
                 """https://en.wikipedia.org/wiki/METEOR""",
             ]       ,				)


  def  lowerCamelCase       (      self		:			Tuple       ,				lowerCamelCase		:			List[Any]							)						->	Dict:


         """simple docstring"""
         import nltk

         nltk.download("""wordnet"""							)
         if NLTK_VERSION >= version.Version("""3.6.5"""							):
                nltk.download("""punkt"""							)
         if NLTK_VERSION >= version.Version("""3.6.6"""							):
                nltk.download("""omw-1.4"""							)





  def  lowerCamelCase       (      self		:			Dict       ,				lowerCamelCase		:			int       ,				lowerCamelCase		:			Tuple       ,				lowerCamelCase		:			Dict=0.9       ,				lowerCamelCase		:			List[str]=3       ,				lowerCamelCase		:			Tuple=0.5							)						->	Optional[Any]:


         """simple docstring"""


         if NLTK_VERSION >= version.Version("""3.6.5"""							):
                _UpperCAmelCase							=    [
                    meteor_score.single_meteor_score(
                        word_tokenize(lowerCamelCase_							)       ,				word_tokenize(lowerCamelCase_							)       ,				alpha=lowerCamelCase_       ,				beta=lowerCamelCase_       ,				gamma=lowerCamelCase_							)
                    for ref, pred in zip(lowerCamelCase_       ,				lowerCamelCase_							)
                ]
         else:
                _UpperCAmelCase							=    [
                    meteor_score.single_meteor_score(lowerCamelCase_       ,				lowerCamelCase_       ,				alpha=lowerCamelCase_       ,				beta=lowerCamelCase_       ,				gamma=lowerCamelCase_							)
                    for ref, pred in zip(lowerCamelCase_       ,				lowerCamelCase_							)
                ]

         return {"meteor": np.mean(lowerCamelCase_							)}
import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
_SCREAMING_SNAKE_CASE 		=      logging.getLogger()

_SCREAMING_SNAKE_CASE 		=      logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)
class SCREAMING_SNAKE_CASE_		(      __lowerCAmelCase		):







   def 						lowerCamelCase_	(     self						:						Union[str, Any]   ,   lowerCamelCase_						:						List[str]				):





    """simple docstring"""


    os.makedirs(lowerCamelCase_   ,   exist_ok=lowerCamelCase_				)
    UpperCamelCase			  =      {"""source""": """What is love ?""", """target""": """life"""}
    UpperCamelCase			  =      {"""train""": 12, """val""": 2, """test""": 2}
    for split in ["train", "test", "val"]:
     for field in ["source", "target"]:
      UpperCamelCase			  =      """\n""".join([contents[field]] * n_lines[split]				)
      with open(os.path.join(lowerCamelCase_   ,   f"""{split}.{field}"""				)   ,   """w"""				) as f:
       f.write(lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						Tuple   ,   lowerCamelCase_						:						int   ,   lowerCamelCase_						:						str = "pytorch"				):





    """simple docstring"""


    UpperCamelCase			  =      self.get_auto_remove_tmp_dir()
    UpperCamelCase			  =      os.path.join(lowerCamelCase_   ,   """output"""				)
    UpperCamelCase			  =      os.path.join(lowerCamelCase_   ,   """data"""				)
    self._create_dummy_data(data_dir=lowerCamelCase_				)

    UpperCamelCase			  =      f"""
                --data_dir {data_dir} \
                --output_dir {output_dir} \
                --model_name_or_path facebook/rag-sequence-base \
                --model_type rag_sequence \
                --do_train \
                --do_predict \
                --n_val -1 \
                --val_check_interval 1.0 \
                --train_batch_size 2 \
                --eval_batch_size 1 \
                --max_source_length 25 \
                --max_target_length 25 \
                --val_max_target_length 25 \
                --test_max_target_length 25 \
                --label_smoothing 0.1 \
                --dropout 0.1 \
                --attention_dropout 0.1 \
                --weight_decay 0.001 \
                --adam_epsilon 1e-08 \
                --max_grad_norm 0.1 \
                --lr_scheduler polynomial \
                --learning_rate 3e-04 \
                --num_train_epochs 1 \
                --warmup_steps 4 \
                --gradient_accumulation_steps 1 \
                --distributed-port 8787 \
                --use_dummy_dataset 1 \
                --distributed_retriever {distributed_retriever} \
            """.split()

    if gpus > 0:
     testargs.append(f"""--gpus={gpus}"""				)
     if is_apex_available():
      testargs.append("""--fp16"""				)
    else:
     testargs.append("""--gpus=0"""				)
     testargs.append("""--distributed_backend=ddp_cpu"""				)
     testargs.append("""--num_processes=2"""				)

    UpperCamelCase			  =      [sys.executable, str(Path(finetune_rag.__file__				).resolve()				)] + testargs
    execute_subprocess_async(lowerCamelCase_   ,   env=self.get_env()				)

    UpperCamelCase			  =      os.path.join(lowerCamelCase_   ,   """metrics.json"""				)
    with open(lowerCamelCase_				) as f:
     UpperCamelCase			  =      json.load(lowerCamelCase_				)
    return result







   @require_torch_gpu
   def 						lowerCamelCase_	(     self						:						Tuple				):





    """simple docstring"""


    UpperCamelCase			  =      self._run_finetune(gpus=1				)
    self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]   ,   0.2				)







   @require_torch_multi_gpu
   def 						lowerCamelCase_	(     self						:						List[Any]				):





    """simple docstring"""


    UpperCamelCase			  =      self._run_finetune(gpus=2				)
    self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]   ,   0.2				)







   @require_torch_gpu
   @require_ray
   def 						lowerCamelCase_	(     self						:						Union[str, Any]				):





    """simple docstring"""


    UpperCamelCase			  =      self._run_finetune(gpus=1   ,   distributed_retriever="""ray"""				)
    self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]   ,   0.2				)







   @require_torch_multi_gpu
   @require_ray
   def 						lowerCamelCase_	(     self						:						int				):





    """simple docstring"""


    UpperCamelCase			  =      self._run_finetune(gpus=1   ,   distributed_retriever="""ray"""				)
    self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]   ,   0.2				)
