'''simple docstring'''




import json
import logging
import os
import sys
from pathlib import Path

import finetune_rag

from transformers.file_utils import is_apex_available
from transformers.testing_utils import (
    TestCasePlus,
    execute_subprocess_async,
    require_ray,
    require_torch_gpu,
    require_torch_multi_gpu,
)


logging.basicConfig(level=logging.DEBUG)
_UpperCamelCase					:		Optional[int]							      =		logging.getLogger()

_UpperCamelCase					:		Union[str, Any]							      =		logging.StreamHandler(sys.stdout)
logger.addHandler(stream_handler)

class 	_lowercase(  a_					):



   """simple docstring"""





   def       snake_case						(				self:     Tuple      ,a:     Union[str, Any]						):
    os.makedirs(UpperCamelCase_      ,exist_ok=UpperCamelCase_						)
    __UpperCAmelCase								=       {'source': 'What is love ?', 'target': 'life'}
    __UpperCAmelCase								=       {'train': 12, 'val': 2, 'test': 2}
    for split in ["train", "test", "val"]:
     for field in ["source", "target"]:
      __UpperCAmelCase								=       '\n'.join([contents[field]] * n_lines[split]						)
      with open(os.path.join(UpperCamelCase_      ,f"""{split}.{field}"""						)      ,'w'						) as f:
       f.write(UpperCamelCase_						)





   def       snake_case						(				self:     List[str]      ,a:     int      ,a:     str = "pytorch"						):
    __UpperCAmelCase								=       self.get_auto_remove_tmp_dir()
    __UpperCAmelCase								=       os.path.join(UpperCamelCase_      ,'output'						)
    __UpperCAmelCase								=       os.path.join(UpperCamelCase_      ,'data'						)
    self._create_dummy_data(data_dir=UpperCamelCase_						)

    __UpperCAmelCase								=       f"""
                --data_dir {data_dir} \
                --output_dir {output_dir} \
                --model_name_or_path facebook/rag-sequence-base \
                --model_type rag_sequence \
                --do_train \
                --do_predict \
                --n_val -1 \
                --val_check_interval 1.0 \
                --train_batch_size 2 \
                --eval_batch_size 1 \
                --max_source_length 25 \
                --max_target_length 25 \
                --val_max_target_length 25 \
                --test_max_target_length 25 \
                --label_smoothing 0.1 \
                --dropout 0.1 \
                --attention_dropout 0.1 \
                --weight_decay 0.001 \
                --adam_epsilon 1e-08 \
                --max_grad_norm 0.1 \
                --lr_scheduler polynomial \
                --learning_rate 3e-04 \
                --num_train_epochs 1 \
                --warmup_steps 4 \
                --gradient_accumulation_steps 1 \
                --distributed-port 8787 \
                --use_dummy_dataset 1 \
                --distributed_retriever {distributed_retriever} \
            """.split()

    if gpus > 0:
     testargs.append(f"""--gpus={gpus}"""						)
     if is_apex_available():
      testargs.append('--fp16'						)
    else:
     testargs.append('--gpus=0'						)
     testargs.append('--distributed_backend=ddp_cpu'						)
     testargs.append('--num_processes=2'						)

    __UpperCAmelCase								=       [sys.executable, str(Path(finetune_rag.__file__						).resolve()						)] + testargs
    execute_subprocess_async(UpperCamelCase_      ,env=self.get_env()						)

    __UpperCAmelCase								=       os.path.join(UpperCamelCase_      ,'metrics.json'						)
    with open(UpperCamelCase_						) as f:
     __UpperCAmelCase								=       json.load(UpperCamelCase_						)
    return result





   @require_torch_gpu
   def       snake_case						(				self:     Optional[Any]						):
    __UpperCAmelCase								=       self._run_finetune(gpus=1						)
    self.assertGreaterEqual(result['test'][0]['test_avg_em']      ,0.2						)





   @require_torch_multi_gpu
   def       snake_case						(				self:     Optional[int]						):
    __UpperCAmelCase								=       self._run_finetune(gpus=2						)
    self.assertGreaterEqual(result['test'][0]['test_avg_em']      ,0.2						)





   @require_torch_gpu
   @require_ray
   def       snake_case						(				self:     List[Any]						):
    __UpperCAmelCase								=       self._run_finetune(gpus=1      ,distributed_retriever='ray'						)
    self.assertGreaterEqual(result['test'][0]['test_avg_em']      ,0.2						)





   @require_torch_multi_gpu
   @require_ray
   def       snake_case						(				self:     Optional[int]						):
    __UpperCAmelCase								=       self._run_finetune(gpus=1      ,distributed_retriever='ray'						)
    self.assertGreaterEqual(result['test'][0]['test_avg_em']      ,0.2						)











"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


snake_case : Optional[Any]     		=		logging.get_logger(__name__)

snake_case : str     		=		{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

snake_case : List[Any]     		=		{
    """vocab_file""": {"""mobilebert-uncased""": """https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt"""},
    """tokenizer_file""": {
        """mobilebert-uncased""": """https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json"""
    },
}

snake_case : Tuple     		=		{"""mobilebert-uncased""": 5_1_2}


snake_case : List[str]     		=		{}



class        UpperCamelCase__ (				a_):







    """simple docstring"""
    __UpperCAmelCase           = VOCAB_FILES_NAMES
    __UpperCAmelCase           = PRETRAINED_VOCAB_FILES_MAP
    __UpperCAmelCase           = PRETRAINED_INIT_CONFIGURATION
    __UpperCAmelCase           = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __UpperCAmelCase           = MobileBertTokenizer







    def __init__(		self :	Optional[int]				,    UpperCamelCase_ :	Optional[int]=None				,    UpperCamelCase_ :	str=None				,    UpperCamelCase_ :	Tuple=True				,    UpperCamelCase_ :	Any="[UNK]"				,    UpperCamelCase_ :	str="[SEP]"				,    UpperCamelCase_ :	Any="[PAD]"				,    UpperCamelCase_ :	Dict="[CLS]"				,    UpperCamelCase_ :	str="[MASK]"				,    UpperCamelCase_ :	Union[str, Any]=True				,    UpperCamelCase_ :	Dict=None				,    **UpperCamelCase_ :	Dict				,    ):





           '''simple docstring'''

           super().__init__(
               UpperCamelCase_				,    tokenizer_file=UpperCamelCase_				,    do_lower_case=UpperCamelCase_				,    unk_token=UpperCamelCase_				,    sep_token=UpperCamelCase_				,    pad_token=UpperCamelCase_				,    cls_token=UpperCamelCase_				,    mask_token=UpperCamelCase_				,    tokenize_chinese_chars=UpperCamelCase_				,    strip_accents=UpperCamelCase_				,    **UpperCamelCase_				,    )

           __magic_name__      				=    json.loads(self.backend_tokenizer.normalizer.__getstate__() )
           if (
               normalizer_state.get('lowercase'				,    UpperCamelCase_ ) != do_lower_case
               or normalizer_state.get('strip_accents'				,    UpperCamelCase_ ) != strip_accents
               or normalizer_state.get('handle_chinese_chars'				,    UpperCamelCase_ ) != tokenize_chinese_chars
           ):
                  __magic_name__      				=    getattr(UpperCamelCase_				,    normalizer_state.pop('type' ) )
                  __magic_name__      				=    do_lower_case
                  __magic_name__      				=    strip_accents
                  __magic_name__      				=    tokenize_chinese_chars
                  __magic_name__      				=    normalizer_class(**UpperCamelCase_ )

           __magic_name__      				=    do_lower_case







    def 				a__			(		self :	Dict				,    UpperCamelCase_ :	List[str]				,    UpperCamelCase_ :	List[Any]=None ):





           '''simple docstring'''

           __magic_name__      				=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output







    def 				a__			(		self :	List[Any]				,    UpperCamelCase_ :	List[int]				,    UpperCamelCase_ :	Optional[List[int]] = None ):





           '''simple docstring'''

           __magic_name__      				=    [self.sep_token_id]
           __magic_name__      				=    [self.cls_token_id]
           if token_ids_a is None:
                  return len(cls + token_ids_a + sep ) * [0]
           return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







    def 				a__			(		self :	List[str]				,    UpperCamelCase_ :	str				,    UpperCamelCase_ :	Optional[str] = None ):





           '''simple docstring'''

           __magic_name__      				=    self._tokenizer.model.save(UpperCamelCase_				,    name=UpperCamelCase_ )
           return tuple(UpperCamelCase_ )