


'''simple docstring'''







import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
SCREAMING_SNAKE_CASE_      =			logging.getLogger()

SCREAMING_SNAKE_CASE_      =			logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)


class  lowerCAmelCase_						(       lowerCamelCase__      ):





  """simple docstring"""



  def __a						(					self     :							int				,							SCREAMING_SNAKE_CASE__     :							Tuple						):





       '''simple docstring'''

       os.makedirs(__lowerCamelCase				,							exist_ok=__lowerCamelCase						)
       __a       =   {'''source''': '''What is love ?''', '''target''': '''life'''}
       __a       =   {'''train''': 1_2, '''val''': 2, '''test''': 2}
       for split in ["train", "test", "val"]:
            for field in ["source", "target"]:
                 __a       =   '''\n'''.join([contents[field]] * n_lines[split]						)
                 with open(os.path.join(__lowerCamelCase				,							f'''{split}.{field}'''						)				,							"""w"""						) as f:
                      f.write(__lowerCamelCase						)

  def __a						(					self     :							Dict				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							Optional[Any] = "pytorch"						):





       '''simple docstring'''

       __a       =   self.get_auto_remove_tmp_dir()
       __a       =   os.path.join(__lowerCamelCase				,							"""output"""						)
       __a       =   os.path.join(__lowerCamelCase				,							"""data"""						)
       self._create_dummy_data(data_dir=__lowerCamelCase						)

       __a       =   f'''
                --data_dir {data_dir} \
                --output_dir {output_dir} \
                --model_name_or_path facebook/rag-sequence-base \
                --model_type rag_sequence \
                --do_train \
                --do_predict \
                --n_val -1 \
                --val_check_interval 1.0 \
                --train_batch_size 2 \
                --eval_batch_size 1 \
                --max_source_length 25 \
                --max_target_length 25 \
                --val_max_target_length 25 \
                --test_max_target_length 25 \
                --label_smoothing 0.1 \
                --dropout 0.1 \
                --attention_dropout 0.1 \
                --weight_decay 0.001 \
                --adam_epsilon 1e-08 \
                --max_grad_norm 0.1 \
                --lr_scheduler polynomial \
                --learning_rate 3e-04 \
                --num_train_epochs 1 \
                --warmup_steps 4 \
                --gradient_accumulation_steps 1 \
                --distributed-port 8787 \
                --use_dummy_dataset 1 \
                --distributed_retriever {distributed_retriever} \
            '''.split()

       if gpus > 0:
            testargs.append(f'''--gpus={gpus}'''						)
            if is_apex_available():
                 testargs.append("""--fp16"""						)
       else:
            testargs.append("""--gpus=0"""						)
            testargs.append("""--distributed_backend=ddp_cpu"""						)
            testargs.append("""--num_processes=2"""						)

       __a       =   [sys.executable, str(Path(finetune_rag.__file__						).resolve()						)] + testargs
       execute_subprocess_async(__lowerCamelCase				,							env=self.get_env()						)

       __a       =   os.path.join(__lowerCamelCase				,							"""metrics.json"""						)
       with open(__lowerCamelCase						) as f:
            __a       =   json.load(__lowerCamelCase						)
       return result

  @require_torch_gpu
  def __a						(					self     :							Optional[Any]						):





       '''simple docstring'''

       __a       =   self._run_finetune(gpus=1						)
       self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]				,							0.2						)

  @require_torch_multi_gpu
  def __a						(					self     :							Union[str, Any]						):





       '''simple docstring'''

       __a       =   self._run_finetune(gpus=2						)
       self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]				,							0.2						)

  @require_torch_gpu
  @require_ray
  def __a						(					self     :							Any						):





       '''simple docstring'''

       __a       =   self._run_finetune(gpus=1				,							distributed_retriever="""ray"""						)
       self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]				,							0.2						)


  @require_torch_multi_gpu
  @require_ray
  def __a						(					self     :							Optional[int]						):





       '''simple docstring'''

       __a       =   self._run_finetune(gpus=1				,							distributed_retriever="""ray"""						)
       self.assertGreaterEqual(result["""test"""][0]["""test_avg_em"""]				,							0.2						)










'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE_      =			logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_      =			{'vocab_file': 'sentencepiece.model'}

SCREAMING_SNAKE_CASE_      =			{
    'vocab_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/sentencepiece.model',
    },
}

SCREAMING_SNAKE_CASE_      =			{
    'google/rembert': 2_56,
}


class  lowerCAmelCase_						(       snake_case__      ):





  """simple docstring"""



  a_       :Tuple												=VOCAB_FILES_NAMES
  a_       :Tuple												=PRETRAINED_VOCAB_FILES_MAP
  a_       :Optional[int]												=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

  def __init__(					self     :							Union[str, Any]				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							Tuple=False				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]=True				,							SCREAMING_SNAKE_CASE__     :							Union[str, Any]=True				,							SCREAMING_SNAKE_CASE__     :							int="[CLS]"				,							SCREAMING_SNAKE_CASE__     :							Optional[int]="[SEP]"				,							SCREAMING_SNAKE_CASE__     :							Optional[Any]="[UNK]"				,							SCREAMING_SNAKE_CASE__     :							Any="[SEP]"				,							SCREAMING_SNAKE_CASE__     :							Dict="[PAD]"				,							SCREAMING_SNAKE_CASE__     :							Dict="[CLS]"				,							SCREAMING_SNAKE_CASE__     :							Dict="[MASK]"				,							**SCREAMING_SNAKE_CASE__     :							List[str]				,							):





       '''simple docstring'''

       super().__init__(
           do_lower_case=SCREAMING_SNAKE_CASE__				,							remove_space=SCREAMING_SNAKE_CASE__				,							keep_accents=SCREAMING_SNAKE_CASE__				,							bos_token=SCREAMING_SNAKE_CASE__				,							eos_token=SCREAMING_SNAKE_CASE__				,							unk_token=SCREAMING_SNAKE_CASE__				,							sep_token=SCREAMING_SNAKE_CASE__				,							pad_token=SCREAMING_SNAKE_CASE__				,							cls_token=SCREAMING_SNAKE_CASE__				,							mask_token=SCREAMING_SNAKE_CASE__				,							**SCREAMING_SNAKE_CASE__				,							)

       __a       =   do_lower_case
       __a       =   remove_space
       __a       =   keep_accents
       __a       =   vocab_file

       __a       =   spm.SentencePieceProcessor()
       self.sp_model.Load(SCREAMING_SNAKE_CASE__						)

  @property
  def __a						(					self     :							List[str]						):





       '''simple docstring'''

       return len(self.sp_model						)

  def __a						(					self     :							str						):





       '''simple docstring'''

       __a       =   {self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__						): i for i in range(self.vocab_size						)}
       vocab.update(self.added_tokens_encoder						)
       return vocab

  def __getstate__(					self     :							List[Any]						):





       '''simple docstring'''

       __a       =   self.__dict__.copy()
       __a       =   None
       return state

  def __setstate__(					self     :							Dict				,							SCREAMING_SNAKE_CASE__     :							Optional[int]						):





       '''simple docstring'''

       __a       =   d
       __a       =   spm.SentencePieceProcessor()
       self.sp_model.Load(self.vocab_file						)

  def __a						(					self     :							List[Any]				,							SCREAMING_SNAKE_CASE__     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							List[str]=False						):





       '''simple docstring'''

       __a       =   self.sp_model.EncodeAsPieces(SCREAMING_SNAKE_CASE__						)
       return pieces

  def __a						(					self     :							Optional[Any]				,							SCREAMING_SNAKE_CASE__     :							int						):





       '''simple docstring'''

       return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE__						)

  def __a						(					self     :							List[Any]				,							SCREAMING_SNAKE_CASE__     :							List[Any]						):





       '''simple docstring'''

       return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE__						)

  def __a						(					self     :							List[str]				,							SCREAMING_SNAKE_CASE__     :							str						):





       '''simple docstring'''

       __a       =   self.sp_model.decode_pieces(SCREAMING_SNAKE_CASE__						)
       return out_string

  def __a						(					self     :							Tuple				,							SCREAMING_SNAKE_CASE__     :							List[int]				,							SCREAMING_SNAKE_CASE__     :							Optional[List[int]] = None						):





       '''simple docstring'''

       __a       =   [self.sep_token_id]
       __a       =   [self.cls_token_id]
       if token_ids_a is None:
            return cls + token_ids_a + sep
       return cls + token_ids_a + sep + token_ids_a + sep

  def __a						(					self     :							Any				,							SCREAMING_SNAKE_CASE__     :							List[int]				,							SCREAMING_SNAKE_CASE__     :							Optional[List[int]] = None				,							SCREAMING_SNAKE_CASE__     :							bool = False						):





       '''simple docstring'''

       if already_has_special_tokens:
            if token_ids_a is not None:
                 raise ValueError(
                     """You should not supply a second sequence if the provided sequence of """
                     """ids is already formatted with special tokens for the model."""						)
            return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

       if token_ids_a is not None:
            return [1] + ([0] * len(SCREAMING_SNAKE_CASE__						)) + [1] + ([0] * len(SCREAMING_SNAKE_CASE__						)) + [1]
       return [1] + ([0] * len(SCREAMING_SNAKE_CASE__						)) + [1]

  def __a						(					self     :							Union[str, Any]				,							SCREAMING_SNAKE_CASE__     :							List[int]				,							SCREAMING_SNAKE_CASE__     :							Optional[List[int]] = None						):





       '''simple docstring'''

       __a       =   [self.sep_token_id]
       __a       =   [self.cls_token_id]

       if token_ids_a is None:
            return len(cls + token_ids_a + sep						) * [0]
       return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]


  def __a						(					self     :							int				,							SCREAMING_SNAKE_CASE__     :							str				,							SCREAMING_SNAKE_CASE__     :							Optional[str] = None						):





       '''simple docstring'''

       if not os.path.isdir(SCREAMING_SNAKE_CASE__						):
            logger.error("""Vocabulary path ({}) should be a directory""".format(SCREAMING_SNAKE_CASE__						)						)
            return
       __a       =   os.path.join(
           SCREAMING_SNAKE_CASE__				,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

       if os.path.abspath(self.vocab_file						) != os.path.abspath(SCREAMING_SNAKE_CASE__						):
            copyfile(self.vocab_file				,							SCREAMING_SNAKE_CASE__						)

       return (out_vocab_file,)






