



import os
import unittest

from huggingface_hub.utils import are_progress_bars_disabled

import transformers.models.bart.tokenization_bart
from transformers import logging
from transformers.testing_utils import CaptureLogger, mockenv, mockenv_context
from transformers.utils.logging import disable_progress_bar, enable_progress_bar




class 							lowerCAmelCase__  (	unittest.TestCase						):







					def  _lowercase			(       self      :     Any):
											A__	:							List[str]						    = logging.get_logger()

											# the current default level is logging.WARNING
											A__	:							Tuple						    = logging.get_verbosity()

											logging.set_verbosity_error()
											self.assertEqual(logger.getEffectiveLevel()			,		logging.get_verbosity())

											logging.set_verbosity_warning()
											self.assertEqual(logger.getEffectiveLevel()			,		logging.get_verbosity())

											logging.set_verbosity_info()
											self.assertEqual(logger.getEffectiveLevel()			,		logging.get_verbosity())

											logging.set_verbosity_debug()
											self.assertEqual(logger.getEffectiveLevel()			,		logging.get_verbosity())

											# restore to the original level
											logging.set_verbosity(snake_case__)







					def  _lowercase			(       self      :     Optional[int]):
											A__	:							Union[str, Any]						    = logging.get_verbosity()

											A__	:							List[str]						    = logging.get_logger("transformers.models.bart.tokenization_bart")
											A__	:							List[Any]						    = "Testing 1, 2, 3"

											# should be able to log warnings (if default settings weren't overridden by `pytest --log-level-all`)
											if level_origin <= logging.WARNING:
																	with CaptureLogger(snake_case__) as cl:
																							logger.warning(snake_case__)
																	self.assertEqual(cl.out			,		msg + "\n")

											# this is setting the level for all of `transformers.*` loggers
											logging.set_verbosity_error()

											# should not be able to log warnings
											with CaptureLogger(snake_case__) as cl:
																	logger.warning(snake_case__)
											self.assertEqual(cl.out			,		"")

											# should be able to log warnings again
											logging.set_verbosity_warning()
											with CaptureLogger(snake_case__) as cl:
																	logger.warning(snake_case__)
											self.assertEqual(cl.out			,		msg + "\n")

											# restore to the original level
											logging.set_verbosity(snake_case__)







					@mockenv(TRANSFORMERS_VERBOSITY="error")
					def  _lowercase			(       self      :     Optional[int]):
											transformers.utils.logging._reset_library_root_logger()
											# this action activates the env var
											A__	:							Dict						    = logging.get_logger("transformers.models.bart.tokenization_bart")

											A__	:							str						    = os.getenv("TRANSFORMERS_VERBOSITY"			,		snake_case__)
											A__	:							Union[str, Any]						    = logging.log_levels[env_level_str]

											A__	:							int						    = logging.get_verbosity()
											self.assertEqual(
											    snake_case__			,		snake_case__			,		F'TRANSFORMERS_VERBOSITY={env_level_str}/{env_level}, but internal verbosity is {current_level}'			,		)

											# restore to the original level
											A__	:							Union[str, Any]						    = ""
											transformers.utils.logging._reset_library_root_logger()







					@mockenv(TRANSFORMERS_VERBOSITY="super-error")
					def  _lowercase			(       self      :     int):
											transformers.utils.logging._reset_library_root_logger()
											A__	:							str						    = logging.logging.getLogger()
											with CaptureLogger(snake_case__) as cl:
																	# this action activates the env var
																	logging.get_logger("transformers.models.bart.tokenization_bart")
											self.assertIn("Unknown option TRANSFORMERS_VERBOSITY=super-error"			,		cl.out)

											# no need to restore as nothing was changed







					def  _lowercase			(       self      :     Union[str, Any]):
											transformers.utils.logging._reset_library_root_logger()

											A__	:							Dict						    = logging.get_logger("transformers.models.bart.tokenization_bart")
											A__	:							int						    = "Testing 1, 2, 3"

											with mockenv_context(TRANSFORMERS_NO_ADVISORY_WARNINGS="1"):
																	# nothing should be logged as env var disables this method
																	with CaptureLogger(snake_case__) as cl:
																							logger.warning_advice(snake_case__)
																	self.assertEqual(cl.out			,		"")

											with mockenv_context(TRANSFORMERS_NO_ADVISORY_WARNINGS=""):
																	# should log normally as TRANSFORMERS_NO_ADVISORY_WARNINGS is unset
																	with CaptureLogger(snake_case__) as cl:
																							logger.warning_advice(snake_case__)
																	self.assertEqual(cl.out			,		msg + "\n")




def 	snake_case__       (					)		->		List[Any]:







						"""simple docstring"""

						disable_progress_bar()
						assert are_progress_bars_disabled()

						enable_progress_bar()
						assert not are_progress_bars_disabled()










import logging
import random

import ray

from transformers import RagConfig, RagRetriever, RagTokenizer
from transformers.models.rag.retrieval_rag import CustomHFIndex


snake_case    :						Any  =						logging.getLogger(__name__)




class 							lowerCAmelCase__  :







					def __init__(       self      :     Union[str, Any]):
											A__	:							Dict						    = False







					def  _lowercase			(       self      :     Optional[int]			,		_A      :     int			,		_A      :     Optional[int]			,		_A      :     List[str]			,		_A      :     Optional[Any]):
											if not self.initialized:
																	A__	:							List[Any]						    = RagRetriever(
																	    _A			,		question_encoder_tokenizer=_A			,		generator_tokenizer=_A			,		index=_A			,		init_retrieval=_A			,		)
																	A__	:							Dict						    = True







					def  _lowercase			(       self      :     Union[str, Any]):
											self.retriever.index.init_index()







					def  _lowercase			(       self      :     List[str]			,		_A      :     Optional[int]			,		_A      :     List[str]):
											A__    ,       A__	:							str						    = self.retriever._main_retrieve(_A			,		_A)
											return doc_ids, retrieved_doc_embeds




class 							lowerCAmelCase__  (	UpperCamelCase						):







					def __init__(       self      :     Optional[Any]			,		_A      :     Union[str, Any]			,		_A      :     Tuple			,		_A      :     int			,		_A      :     Tuple			,		_A      :     Union[str, Any]=None):
											if index is not None and index.is_initialized() and len(_A) > 0:
																	raise ValueError(
																	    "When using Ray for distributed fine-tuning, "
																	    "you'll need to provide the paths instead, "
																	    "as the dataset and the index are loaded "
																	    "separately. More info in examples/rag/use_own_knowledge_dataset.py ")
											super().__init__(
											    _A			,		question_encoder_tokenizer=_A			,		generator_tokenizer=_A			,		index=_A			,		init_retrieval=_A			,		)
											A__	:							str						    = retrieval_workers
											if len(self.retrieval_workers) > 0:
																	ray.get(
																	    [
																	        worker.create_rag_retriever.remote(_A			,		_A			,		_A			,		_A)
																	        for worker in self.retrieval_workers
																	    ])







					def  _lowercase			(       self      :     Optional[Any]):
											logger.info("initializing retrieval")

											if len(self.retrieval_workers) > 0:
																	ray.get([worker.init_retrieval.remote() for worker in self.retrieval_workers])
											else:
																	# Non-distributed training. Load index into this same process.
																	self.index.init_index()







					def  _lowercase			(       self      :     Union[str, Any]			,		_A      :     List[str]			,		_A      :     Optional[Any]):
											if len(self.retrieval_workers) > 0:
																	# Select a random retrieval actor.
																	A__	:							Optional[Any]						    = self.retrieval_workers[random.randint(0			,		len(self.retrieval_workers) - 1)]
																	A__    ,       A__	:							List[Any]						    = ray.get(random_worker.retrieve.remote(_A			,		_A))
											else:
																	A__    ,       A__	:							Union[str, Any]						    = self._main_retrieve(_A			,		_A)
											return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(_A)







					@classmethod
					def  _lowercase			(       cls      :     int			,		_A      :     Optional[int]			,		_A      :     List[str]=None			,		**_A      :     Optional[Any]):
											return super(_A			,		cls).get_tokenizers(_A			,		_A			,		**_A)







					@classmethod
					def  _lowercase			(       cls      :     Any			,		_A      :     Union[str, Any]			,		_A      :     Optional[Any]			,		_A      :     List[str]=None			,		**_A      :     Optional[Any]):
											A__	:							str						    = kwargs.pop("config"			,		_A) or RagConfig.from_pretrained(_A			,		**_A)
											A__	:							int						    = RagTokenizer.from_pretrained(_A			,		config=_A)
											A__	:							List[str]						    = rag_tokenizer.question_encoder
											A__	:							str						    = rag_tokenizer.generator
											if indexed_dataset is not None:
																	A__	:							Tuple						    = "custom"
																	A__	:							Tuple						    = CustomHFIndex(config.retrieval_vector_size			,		_A)
											else:
																	A__	:							Union[str, Any]						    = cls._build_index(_A)
											return cls(
											    _A			,		question_encoder_tokenizer=_A			,		generator_tokenizer=_A			,		retrieval_workers=_A			,		index=_A			,		)