



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


snake_case    :						Any  =						logging.get_logger(__name__)

snake_case    :						List[Any]  =						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

snake_case    :						Optional[Any]  =						{
    'vocab_file': {'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt'},
    'tokenizer_file': {
        'mobilebert-uncased': 'https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json'
    },
}

snake_case    :						Any  =						{'mobilebert-uncased': 5_1_2}


snake_case    :						str  =						{}




class 							lowerCAmelCase__  (	UpperCamelCase						):
					__A     :    Union[str, Any]           =						VOCAB_FILES_NAMES
					__A     :    Union[str, Any]           =						PRETRAINED_VOCAB_FILES_MAP
					__A     :    List[Any]           =						PRETRAINED_INIT_CONFIGURATION
					__A     :    List[str]           =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					__A     :    Dict           =						MobileBertTokenizer







					def __init__(       self      :     Optional[Any]			,		_A      :     Optional[Any]=None			,		_A      :     List[Any]=None			,		_A      :     Dict=True			,		_A      :     Dict="[UNK]"			,		_A      :     Optional[Any]="[SEP]"			,		_A      :     List[Any]="[PAD]"			,		_A      :     Dict="[CLS]"			,		_A      :     Any="[MASK]"			,		_A      :     Union[str, Any]=True			,		_A      :     str=None			,		**_A      :     Optional[Any]			,		):
											super().__init__(
											    _A			,		tokenizer_file=_A			,		do_lower_case=_A			,		unk_token=_A			,		sep_token=_A			,		pad_token=_A			,		cls_token=_A			,		mask_token=_A			,		tokenize_chinese_chars=_A			,		strip_accents=_A			,		**_A			,		)

											A__	:							Tuple						    = json.loads(self.backend_tokenizer.normalizer.__getstate__())
											if (
											    normalizer_state.get("lowercase"			,		_A) != do_lower_case
											    or normalizer_state.get("strip_accents"			,		_A) != strip_accents
											    or normalizer_state.get("handle_chinese_chars"			,		_A) != tokenize_chinese_chars
											):
																	A__	:							Optional[Any]						    = getattr(_A			,		normalizer_state.pop("type"))
																	A__	:							List[Any]						    = do_lower_case
																	A__	:							Union[str, Any]						    = strip_accents
																	A__	:							List[str]						    = tokenize_chinese_chars
																	A__	:							Optional[Any]						    = normalizer_class(**_A)

											A__	:							int						    = do_lower_case







					def  _lowercase			(       self      :     List[str]			,		_A      :     int			,		_A      :     Dict=None):
											A__	:							List[str]						    = [self.cls_token_id] + token_ids_a + [self.sep_token_id]

											if token_ids_a:
																	output += token_ids_a + [self.sep_token_id]

											return output







					def  _lowercase			(       self      :     Union[str, Any]			,		_A      :     List[int]			,		_A      :     Optional[List[int]] = None):
											A__	:							Union[str, Any]						    = [self.sep_token_id]
											A__	:							int						    = [self.cls_token_id]
											if token_ids_a is None:
																	return len(cls + token_ids_a + sep) * [0]
											return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]







					def  _lowercase			(       self      :     Any			,		_A      :     str			,		_A      :     Optional[str] = None):
											A__	:							List[str]						    = self._tokenizer.model.save(_A			,		name=_A)
											return tuple(_A)




import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


snake_case    :						Any  =						logging.getLogger(__name__)




class 							lowerCAmelCase__  :







					def __init__(       self      :     Union[str, Any]):
											A__	:							Dict						    = False







					def  _lowercase			(       self      :     Optional[int]			,		_A      :     int			,		_A      :     Optional[int]			,		_A      :     List[str]			,		_A      :     Optional[Any]):
											if not self.initialized:
																	A__	:							List[Any]						    = RagRetriever(
																	    _A			,		question_encoder_tokenizer=_A			,		generator_tokenizer=_A			,		index=_A			,		init_retrieval=_A			,		)
																	A__	:							Dict						    = True







					def  _lowercase			(       self      :     Union[str, Any]):
											self.retriever.index.init_index()







					def  _lowercase			(       self      :     List[str]			,		_A      :     Optional[int]			,		_A      :     List[str]):
											A__    ,       A__	:							str						    = self.retriever._main_retrieve(_A			,		_A)
											return doc_ids, retrieved_doc_embeds




class 							lowerCAmelCase__  (	UpperCamelCase						):







					def __init__(       self      :     Optional[Any]			,		_A      :     Union[str, Any]			,		_A      :     Tuple			,		_A      :     int			,		_A      :     Tuple			,		_A      :     Union[str, Any]=None):
											if index is not None and index.is_initialized() and len(_A) > 0:
																	raise ValueError(
																	    "When using Ray for distributed fine-tuning, "
																	    "you'll need to provide the paths instead, "
																	    "as the dataset and the index are loaded "
																	    "separately. More info in examples/rag/use_own_knowledge_dataset.py ")
											super().__init__(
											    _A			,		question_encoder_tokenizer=_A			,		generator_tokenizer=_A			,		index=_A			,		init_retrieval=_A			,		)
											A__	:							str						    = retrieval_workers
											if len(self.retrieval_workers) > 0:
																	ray.get(
																	    [
																	        worker.create_rag_retriever.remote(_A			,		_A			,		_A			,		_A)
																	        for worker in self.retrieval_workers
																	    ])







					def  _lowercase			(       self      :     Optional[Any]):
											logger.info("initializing retrieval")

											if len(self.retrieval_workers) > 0:
																	ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers])
											else:
																	# Non-distributed training. Load index into this same process.
																	self.index.init_index()







					def  _lowercase			(       self      :     Union[str, Any]			,		_A      :     List[str]			,		_A      :     Optional[Any]):
											if len(self.retrieval_workers) > 0:
																	# Select a random retrieval actor.
																	A__	:							Optional[Any]						    = self.retrieval_workers[random.randint(0			,		len(self.retrieval_workers) - 1)]
																	A__    ,       A__	:							List[Any]						    = ray.get(random_worker.retrieve.remote(_A			,		_A))
											else:
																	A__    ,       A__	:							Union[str, Any]						    = self._main_retrieve(_A			,		_A)
											return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(_A)







					@classmethod
					def  _lowercase			(       cls      :     int			,		_A      :     Optional[int]			,		_A      :     List[str]=None			,		**_A      :     Optional[Any]):
											return super(_A			,		cls).get_tokenizers(_A			,		_A			,		**_A)







					@classmethod
					def  _lowercase			(       cls      :     Any			,		_A      :     Union[str, Any]			,		_A      :     Optional[Any]			,		_A      :     List[str]=None			,		**_A      :     Optional[Any]):
											A__	:							str						    = kwargs.pop("config"			,		_A) or RagConfig.from_pretrained(_A			,		**_A)
											A__	:							int						    = RagTokenizer.from_pretrained(_A			,		config=_A)
											A__	:							List[str]						    = rag_tokenizer.question_encoder
											A__	:							str						    = rag_tokenizer.generator
											if indexed_dataset is not None:
																	A__	:							Tuple						    = "custom"
																	A__	:							Tuple						    = CustomHFIndex(config.retrieval_vector_size			,		_A)
											else:
																	A__	:							Union[str, Any]						    = cls._build_index(_A)
											return cls(
											    _A			,		question_encoder_tokenizer=_A			,		generator_tokenizer=_A			,		retrieval_workers=_A			,		index=_A			,		)