






import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 		SCREAMING_SNAKE_CASE__	(  __lowerCAmelCase					,      unittest.TestCase     ):

  '''simple docstring'''







  _lowerCamelCase    =  FunnelTokenizer
  _lowerCamelCase    =  FunnelTokenizerFast
  _lowerCamelCase    =  True
  _lowerCamelCase    =  True


  def  lowerCamelCase       (      self		:			List[str]							)						->	int:


         """simple docstring"""
         super().setUp()

         _UpperCAmelCase							=    [
             """<unk>""",
             """<cls>""",
             """<sep>""",
             """want""",
             """##want""",
             """##ed""",
             """wa""",
             """un""",
             """runn""",
             """##ing""",
             """,""",
             """low""",
             """lowest""",
         ]
         _UpperCAmelCase							=    os.path.join(self.tmpdirname       ,				VOCAB_FILES_NAMES["""vocab_file"""]							)
         with open(self.vocab_file       ,				"""w"""       ,				encoding="""utf-8"""							) as vocab_writer:
                vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)


  def  lowerCamelCase       (      self		:			List[str]       ,				**lowerCamelCase		:			int							)						->	Union[str, Any]:


         """simple docstring"""
         return FunnelTokenizer.from_pretrained(self.tmpdirname       ,				**lowerCamelCase_							)


  def  lowerCamelCase       (      self		:			Any       ,				**lowerCamelCase		:			Union[str, Any]							)						->	List[str]:


         """simple docstring"""
         return FunnelTokenizerFast.from_pretrained(self.tmpdirname       ,				**lowerCamelCase_							)


  def  lowerCamelCase       (      self		:			Union[str, Any]       ,				lowerCamelCase		:			List[Any]							)						->	int:


         """simple docstring"""
         _UpperCAmelCase							=    """UNwant\u00E9d,running"""
         _UpperCAmelCase							=    """unwanted, running"""
         return input_text, output_text


  def  lowerCamelCase       (      self		:			Tuple							)						->	int:


         """simple docstring"""
         _UpperCAmelCase							=    self.tokenizer_class(self.vocab_file							)

         _UpperCAmelCase							=    tokenizer.tokenize("""UNwant\u00E9d,running"""							)
         self.assertListEqual(lowerCamelCase_       ,				["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]							)
         self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase_							)       ,				[7, 4, 5, 10, 8, 9]							)





  def  lowerCamelCase       (      self		:			Tuple							)						->	List[str]:


         """simple docstring"""
         _UpperCAmelCase							=    self.get_tokenizers(do_lower_case=lowerCamelCase_							)
         for tokenizer in tokenizers:
                _UpperCAmelCase							=    tokenizer("""UNwant\u00E9d,running"""							)
                _UpperCAmelCase							=    len(inputs["""input_ids"""]							) - 1
                self.assertListEqual(inputs["""token_type_ids"""]       ,				[2] + [0] * sentence_len							)

                _UpperCAmelCase							=    tokenizer("""UNwant\u00E9d,running"""       ,				"""UNwant\u00E9d,running"""							)
                self.assertListEqual(inputs["""token_type_ids"""]       ,				[2] + [0] * sentence_len + [1] * sentence_len							)
import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
_SCREAMING_SNAKE_CASE 		=      logging.getLogger()

_SCREAMING_SNAKE_CASE 		=      logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)
class SCREAMING_SNAKE_CASE_		(      __lowerCAmelCase		):







   def 						lowerCamelCase_	(     self						:						Union[str, Any]   ,   lowerCamelCase_						:						List[str]				):





    """simple docstring"""


    os.makedirs(lowerCamelCase_   ,   exist_ok=lowerCamelCase_				)
    UpperCamelCase			  =      {"""source""": """What is love ?""", """target""": """life"""}
    UpperCamelCase			  =      {"""train""": 12, """val""": 2, """test""": 2}
    for split in ["train", "test", "val"]:
     for field in ["source", "target"]:
      UpperCamelCase			  =      """\n""".join([contents[field]] * n_lines[split]				)
      with open(os.path.join(lowerCamelCase_   ,   f"""{split}.{field}"""				)   ,   """w"""				) as f:
       f.write(lowerCamelCase_				)







   def 						lowerCamelCase_	(     self						:						Tuple   ,   lowerCamelCase_						:						int   ,   lowerCamelCase_						:						str = "pytorch"				):





    """simple docstring"""


    UpperCamelCase			  =      self.get_auto_remove_tmp_dir()
    UpperCamelCase			  =      os.path.join(lowerCamelCase_   ,   """output"""				)
    UpperCamelCase			  =      os.path.join(lowerCamelCase_   ,   """data"""				)
    self._create_dummy_data(data_dir=lowerCamelCase_				)

    UpperCamelCase			  =      f"""
                --data_dir {data_dir} \
                --output_dir {output_dir} \
                --model_name_or_path facebook/rag-sequence-base \
                --model_type rag_sequence \
                --do_train \
                --do_predict \
                --n_val -1 \
                --val_check_interval 1.0 \
                --train_batch_size 2 \
                --eval_batch_size 1 \
                --max_source_length 25 \
                --max_target_length 25 \
                --val_max_target_length 25 \
                --test_max_target_length 25 \
                --label_smoothing 0.1 \
                --dropout 0.1 \
                --attention_dropout 0.1 \
                --weight_decay 0.001 \
                --adam_epsilon 1e-08 \
                --max_grad_norm 0.1 \
                --lr_scheduler polynomial \
                --learning_rate 3e-04 \
                --num_train_epochs 1 \
                --warmup_steps 4 \
                --gradient_accumulation_steps 1 \
                --distributed-port 8787 \
                --use_dummy_dataset 1 \
                --distributed_retriever {distributed_retriever} \
            """.split()

    if gpus > 0:
     testargs.append(f"""--gpus={gpus}"""				)
     if is_apex_available():
      testargs.append("""--fp16"""				)
    else:
     testargs.append("""--gpus=0"""				)
     testargs.append("""--distributed_backend=ddp_cpu"""				)
     testargs.append("""--num_processes=2"""				)

    UpperCamelCase			  =      [sys.executable, str(Path(finetune_rag.__file__				).resolve()				)] + testargs
    execute_subprocess_async(lowerCamelCase_   ,   env=self.get_env()				)

    UpperCamelCase			  =      os.path.join(lowerCamelCase_   ,   """metrics.json"""				)
    with open(lowerCamelCase_				) as f:
     UpperCamelCase			  =      json.load(lowerCamelCase_				)
    return result







   @require_torch_gpu
   def 						lowerCamelCase_	(     self						:						Tuple				):





    """simple docstring"""


    UpperCamelCase			  =      self._run_finetune(gpus=1				)
    self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]   ,   0.2				)







   @require_torch_multi_gpu
   def 						lowerCamelCase_	(     self						:						List[Any]				):





    """simple docstring"""


    UpperCamelCase			  =      self._run_finetune(gpus=2				)
    self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]   ,   0.2				)







   @require_torch_gpu
   @require_ray
   def 						lowerCamelCase_	(     self						:						Union[str, Any]				):





    """simple docstring"""


    UpperCamelCase			  =      self._run_finetune(gpus=1   ,   distributed_retriever="""ray"""				)
    self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]   ,   0.2				)







   @require_torch_multi_gpu
   @require_ray
   def 						lowerCamelCase_	(     self						:						int				):





    """simple docstring"""


    UpperCamelCase			  =      self._run_finetune(gpus=1   ,   distributed_retriever="""ray"""				)
    self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]   ,   0.2				)
