




'''simple docstring'''






import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TextClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
A_			      =		{"LayoutLMv2Config", "LayoutLMv3Config"}






@is_pipeline_test
class        lowercase_      (						unittest.TestCase							):
     A_											=			MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
     A_											=			TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

     if model_mapping is not None:
          A_											=			{config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
     if tf_model_mapping is not None:
          A_											=			{
              config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
          }





     @require_torch
     def 			_lowerCAmelCase      (     self     :					Dict			):
           snake_case__       :   int	      =    pipeline(
               task='text-classification'     ,							model='hf-internal-testing/tiny-random-distilbert'     ,							framework='pt'			)

           snake_case__       :   str	      =    text_classifier('This is great !'			)
           self.assertEqual(nested_simplify(__lowerCamelCase			)     ,							[{'label': 'LABEL_0', 'score': 0.5_0_4}]			)

           snake_case__       :   Optional[Any]	      =    text_classifier('This is great !'     ,							top_k=2			)
           self.assertEqual(
               nested_simplify(__lowerCamelCase			)     ,							[{'label': 'LABEL_0', 'score': 0.5_0_4}, {'label': 'LABEL_1', 'score': 0.4_9_6}]			)

           snake_case__       :   List[str]	      =    text_classifier(['This is great !', 'This is bad']     ,							top_k=2			)
           self.assertEqual(
               nested_simplify(__lowerCamelCase			)     ,							[
                   [{'label': 'LABEL_0', 'score': 0.5_0_4}, {'label': 'LABEL_1', 'score': 0.4_9_6}],
                   [{'label': 'LABEL_0', 'score': 0.5_0_4}, {'label': 'LABEL_1', 'score': 0.4_9_6}],
               ]     ,							)

           snake_case__       :   Union[str, Any]	      =    text_classifier('This is great !'     ,							top_k=1			)
           self.assertEqual(nested_simplify(__lowerCamelCase			)     ,							[{'label': 'LABEL_0', 'score': 0.5_0_4}]			)

           # Legacy behavior
           snake_case__       :   Tuple	      =    text_classifier('This is great !'     ,							return_all_scores=__lowerCamelCase			)
           self.assertEqual(nested_simplify(__lowerCamelCase			)     ,							[{'label': 'LABEL_0', 'score': 0.5_0_4}]			)

           snake_case__       :   int	      =    text_classifier('This is great !'     ,							return_all_scores=__lowerCamelCase			)
           self.assertEqual(
               nested_simplify(__lowerCamelCase			)     ,							[[{'label': 'LABEL_0', 'score': 0.5_0_4}, {'label': 'LABEL_1', 'score': 0.4_9_6}]]			)

           snake_case__       :   str	      =    text_classifier(['This is great !', 'Something else']     ,							return_all_scores=__lowerCamelCase			)
           self.assertEqual(
               nested_simplify(__lowerCamelCase			)     ,							[
                   [{'label': 'LABEL_0', 'score': 0.5_0_4}, {'label': 'LABEL_1', 'score': 0.4_9_6}],
                   [{'label': 'LABEL_0', 'score': 0.5_0_4}, {'label': 'LABEL_1', 'score': 0.4_9_6}],
               ]     ,							)

           snake_case__       :   Optional[int]	      =    text_classifier(['This is great !', 'Something else']     ,							return_all_scores=__lowerCamelCase			)
           self.assertEqual(
               nested_simplify(__lowerCamelCase			)     ,							[
                   {'label': 'LABEL_0', 'score': 0.5_0_4},
                   {'label': 'LABEL_0', 'score': 0.5_0_4},
               ]     ,							)





     @require_torch
     def 			_lowerCAmelCase      (     self     :					Tuple			):
           import torch

           snake_case__       :   Tuple	      =    pipeline(
               task='text-classification'     ,							model='hf-internal-testing/tiny-random-distilbert'     ,							framework='pt'     ,							device=torch.device('cpu'			)     ,							)

           snake_case__       :   str	      =    text_classifier('This is great !'			)
           self.assertEqual(nested_simplify(__lowerCamelCase			)     ,							[{'label': 'LABEL_0', 'score': 0.5_0_4}]			)





     @require_tf
     def 			_lowerCAmelCase      (     self     :					Optional[int]			):
           snake_case__       :   Union[str, Any]	      =    pipeline(
               task='text-classification'     ,							model='hf-internal-testing/tiny-random-distilbert'     ,							framework='tf'			)

           snake_case__       :   Any	      =    text_classifier('This is great !'			)
           self.assertEqual(nested_simplify(__lowerCamelCase			)     ,							[{'label': 'LABEL_0', 'score': 0.5_0_4}]			)





     @slow
     @require_torch
     def 			_lowerCAmelCase      (     self     :					int			):
           snake_case__       :   Dict	      =    pipeline('text-classification'			)

           snake_case__       :   Dict	      =    text_classifier('This is great !'			)
           self.assertEqual(nested_simplify(__lowerCamelCase			)     ,							[{'label': 'POSITIVE', 'score': 1.0}]			)
           snake_case__       :   str	      =    text_classifier('This is bad !'			)
           self.assertEqual(nested_simplify(__lowerCamelCase			)     ,							[{'label': 'NEGATIVE', 'score': 1.0}]			)
           snake_case__       :   str	      =    text_classifier('Birds are a type of animal'			)
           self.assertEqual(nested_simplify(__lowerCamelCase			)     ,							[{'label': 'POSITIVE', 'score': 0.9_8_8}]			)





     @slow
     @require_tf
     def 			_lowerCAmelCase      (     self     :					Optional[int]			):
           snake_case__       :   List[str]	      =    pipeline('text-classification'     ,							framework='tf'			)

           snake_case__       :   str	      =    text_classifier('This is great !'			)
           self.assertEqual(nested_simplify(__lowerCamelCase			)     ,							[{'label': 'POSITIVE', 'score': 1.0}]			)
           snake_case__       :   List[Any]	      =    text_classifier('This is bad !'			)
           self.assertEqual(nested_simplify(__lowerCamelCase			)     ,							[{'label': 'NEGATIVE', 'score': 1.0}]			)
           snake_case__       :   int	      =    text_classifier('Birds are a type of animal'			)
           self.assertEqual(nested_simplify(__lowerCamelCase			)     ,							[{'label': 'POSITIVE', 'score': 0.9_8_8}]			)





     def 			_lowerCAmelCase      (     self     :					Union[str, Any]     ,							__lowerCamelCase     :					Dict     ,							__lowerCamelCase     :					Union[str, Any]     ,							__lowerCamelCase     :					Tuple			):
           snake_case__       :   Tuple	      =    TextClassificationPipeline(model=__lowerCamelCase     ,							tokenizer=__lowerCamelCase			)
           return text_classifier, ["HuggingFace is in", "This is another test"]





     def 			_lowerCAmelCase      (     self     :					Optional[Any]     ,							__lowerCamelCase     :					Optional[int]     ,							__lowerCamelCase     :					int			):
           snake_case__       :   int	      =    text_classifier.model
           # Small inputs because BartTokenizer tiny has maximum position embeddings = 22
           snake_case__       :   int	      =    'HuggingFace is in'
           snake_case__       :   Tuple	      =    text_classifier(__lowerCamelCase			)

           self.assertEqual(nested_simplify(__lowerCamelCase			)     ,							[{'label': ANY(__lowerCamelCase			), 'score': ANY(__lowerCamelCase			)}]			)
           self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()			)

           snake_case__       :   List[Any]	      =    ['HuggingFace is in ', 'Paris is in France']
           snake_case__       :   Any	      =    text_classifier(__lowerCamelCase			)
           self.assertEqual(
               nested_simplify(__lowerCamelCase			)     ,							[{'label': ANY(__lowerCamelCase			), 'score': ANY(__lowerCamelCase			)}, {'label': ANY(__lowerCamelCase			), 'score': ANY(__lowerCamelCase			)}]     ,							)
           self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()			)
           self.assertTrue(outputs[1]['label'] in model.config.idalabel.values()			)

           # Forcing to get all results with `top_k=None`
           # This is NOT the legacy format
           snake_case__       :   Optional[int]	      =    text_classifier(__lowerCamelCase     ,							top_k=__lowerCamelCase			)
           snake_case__       :   Union[str, Any]	      =    len(model.config.idalabel.values()			)
           self.assertEqual(
               nested_simplify(__lowerCamelCase			)     ,							[[{'label': ANY(__lowerCamelCase			), 'score': ANY(__lowerCamelCase			)}] * N, [{'label': ANY(__lowerCamelCase			), 'score': ANY(__lowerCamelCase			)}] * N]     ,							)

           snake_case__       :   str	      =    {'text': 'HuggingFace is in ', 'text_pair': 'Paris is in France'}
           snake_case__       :   Optional[Any]	      =    text_classifier(__lowerCamelCase			)
           self.assertEqual(
               nested_simplify(__lowerCamelCase			)     ,							{'label': ANY(__lowerCamelCase			), 'score': ANY(__lowerCamelCase			)}     ,							)
           self.assertTrue(outputs['label'] in model.config.idalabel.values()			)

           # This might be used a text pair, but tokenizer + pipe interaction
           # makes it hard to understand that it's not using the pair properly
           # https://github.com/huggingface/transformers/issues/17305
           # We disabled this usage instead as it was outputting wrong outputs.
           snake_case__       :   Optional[Any]	      =    [['HuggingFace is in ', 'Paris is in France']]
           with self.assertRaises(__lowerCamelCase			):
                 text_classifier(__lowerCamelCase			)

           # This used to be valid for doing text pairs
           # We're keeping it working because of backward compatibility
           snake_case__       :   int	      =    text_classifier([[['HuggingFace is in ', 'Paris is in France']]]			)
           self.assertEqual(
               nested_simplify(__lowerCamelCase			)     ,							[{'label': ANY(__lowerCamelCase			), 'score': ANY(__lowerCamelCase			)}]     ,							)
           self.assertTrue(outputs[0]['label'] in model.config.idalabel.values()			)






'''simple docstring'''






import logging
import os
from typing import List, Tuple

import numpy as np
import psutil
import torch
import torch.distributed as dist

from transformers import RagRetriever


A_			      =		logging.getLogger(__name__)






class        lowercase_      (						lowerCAmelCase_							):





     def __init__(     self     :					List[Any]     ,							__lowerCamelCase     :					Tuple     ,							__lowerCamelCase     :					str     ,							__lowerCamelCase     :					Tuple     ,							__lowerCamelCase     :					int=None			):
           super().__init__(
               __lowerCamelCase     ,							question_encoder_tokenizer=__lowerCamelCase     ,							generator_tokenizer=__lowerCamelCase     ,							index=__lowerCamelCase     ,							init_retrieval=__lowerCamelCase     ,							)
           snake_case__       :   Optional[int]	      =    None





     def 			_lowerCAmelCase      (     self     :					List[str]     ,							__lowerCamelCase     :					int			):
           logger.info('initializing retrieval'			)

           # initializing a separate process group for retrieval as the default
           # nccl backend doesn't support gather/scatter operations while gloo
           # is too slow to replace nccl for the core gpu communication
           if dist.is_initialized():
                 logger.info('dist initialized'			)
                 # needs to be set manually
                 snake_case__       :   Any	      =    self._infer_socket_ifname()
                 # avoid clash with the NCCL port
                 snake_case__       :   int	      =    str(distributed_port + 1			)
                 snake_case__       :   Any	      =    dist.new_group(ranks=__lowerCamelCase     ,							backend='gloo'			)

           # initialize retriever only on the main worker
           if not dist.is_initialized() or self._is_main():
                 logger.info('dist not initialized / main'			)
                 self.index.init_index()

           # all processes wait untill the retriever is initialized by the main process
           if dist.is_initialized():
                 torch.distributed.barrier(group=self.process_group			)





     def 			_lowerCAmelCase      (     self     :					List[Any]			):
           return dist.get_rank(group=self.process_group			) == 0





     def 			_lowerCAmelCase      (     self     :					Any     ,							__lowerCamelCase     :					Dict     ,							__lowerCamelCase     :					str     ,							__lowerCamelCase     :					Union[str, Any]=torch.floataa			):
           snake_case__       :   Optional[int]	      =    torch.empty(__lowerCamelCase     ,							dtype=__lowerCamelCase			)
           dist.scatter(__lowerCamelCase     ,							src=0     ,							scatter_list=__lowerCamelCase     ,							group=self.process_group			)
           return target_tensor





     def 			_lowerCAmelCase      (     self     :					Dict			):
           snake_case__       :   Union[str, Any]	      =    psutil.net_if_addrs()
           # a hacky way to deal with varying network interface names
           snake_case__       :   Any	      =    next((addr for addr in addrs if addr.startswith('e'			))     ,							__lowerCamelCase			)
           return ifname





     def 			_lowerCAmelCase      (     self     :					Tuple     ,							__lowerCamelCase     :					np.ndarray     ,							__lowerCamelCase     :					int			):

           # single GPU training
           if not dist.is_initialized():
                 snake_case__      ,							snake_case__       :   Optional[int]	      =    self._main_retrieve(__lowerCamelCase     ,							__lowerCamelCase			)
                 return retrieved_doc_embeds, doc_ids, self.index.get_doc_dicts(__lowerCamelCase			)

           # distributed training
           snake_case__       :   Optional[Any]	      =    dist.get_world_size(group=self.process_group			)

           # gather logic
           snake_case__       :   Optional[int]	      =    None
           if self._is_main():
                 snake_case__       :   Optional[int]	      =    [torch.empty(question_hidden_states.shape     ,							dtype=torch.floataa			) for _ in range(__lowerCamelCase			)]
           dist.gather(torch.tensor(__lowerCamelCase			)     ,							dst=0     ,							gather_list=__lowerCamelCase     ,							group=self.process_group			)

           # scatter logic
           snake_case__       :   Union[str, Any]	      =    question_hidden_states.shape[0]
           snake_case__       :   Tuple	      =    []
           snake_case__       :   List[Any]	      =    []
           if self._is_main():
                 assert len(__lowerCamelCase			) == world_size
                 snake_case__      ,							snake_case__       :   Optional[int]	      =    self._main_retrieve(torch.cat(__lowerCamelCase			).numpy()     ,							__lowerCamelCase			)
                 snake_case__      ,							snake_case__       :   Optional[Any]	      =    torch.tensor(__lowerCamelCase			), torch.tensor(__lowerCamelCase			)
                 snake_case__       :   str	      =    self._chunk_tensor(__lowerCamelCase     ,							__lowerCamelCase			)
                 snake_case__       :   Dict	      =    self._chunk_tensor(__lowerCamelCase     ,							__lowerCamelCase			)
           snake_case__       :   Tuple	      =    self._scattered(__lowerCamelCase     ,							[n_queries, n_docs]     ,							target_type=torch.intaa			)
           snake_case__       :   Dict	      =    self._scattered(__lowerCamelCase     ,							[n_queries, n_docs, question_hidden_states.shape[1]]			)

           return retrieved_doc_embeds.numpy(), doc_ids.numpy(), self.index.get_doc_dicts(__lowerCamelCase			)
