



from typing import List, Union

import numpy as np

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, logging
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, ChunkPipeline


a_  :List[Any]     = logging.get_logger(__name__)




class    snake_case__		(			_lowercase	):






     """simple docstring"""


     def   lowercase_  (			self    :    Tuple,					_snake_case    :    List[str]   )  ->int:
            if isinstance(_snake_case,					_snake_case   ):
                   snake_case__				:      List[Any]        =      [label.strip() for label in labels.split(','   ) if label.strip()]
            return labels






     def __call__(			self    :    List[str],					_snake_case    :    str,					_snake_case    :    List[Any],					_snake_case    :    Optional[Any]   )  ->Optional[int]:
            if len(_snake_case   ) == 0 or len(_snake_case   ) == 0:
                   raise ValueError('You must include at least one label and at least one sequence.'   )
            if hypothesis_template.format(labels[0]   ) == hypothesis_template:
                   raise ValueError(
                       (
                           'The provided hypothesis_template \"{}\" was not able to be formatted with the target labels. '
                           'Make sure the passed template includes formatting syntax such as {{}} where the label should go.'
                       ).format(_snake_case   )   )

            if isinstance(_snake_case,					_snake_case   ):
                   snake_case__				:      Tuple        =      [sequences]

            snake_case__				:      str        =      []
            for sequence in sequences:
                   sequence_pairs.extend([[sequence, hypothesis_template.format(_snake_case   )] for label in labels]   )

            return sequence_pairs, sequences




@add_end_docstrings(_lowercase	)
class    snake_case__		(			_lowercase	):






     """simple docstring"""


     def __init__(			self    :    Tuple,					_snake_case    :    Any=ZeroShotClassificationArgumentHandler(),					*_snake_case    :    Tuple,					**_snake_case    :    Union[str, Any]   )  ->Any:
            snake_case__				:      Dict        =      args_parser
            super().__init__(*_snake_case,					**_snake_case   )
            if self.entailment_id == -1:
                   logger.warning(
                       'Failed to determine \'entailment\' label id from the label2id mapping in the model config. Setting to '
                       '-1. Define a descriptive label2id mapping in the model config to ensure correct outputs.'   )






     @property
     def   lowercase_  (			self    :    str   )  ->Optional[Any]:
            for label, ind in self.model.config.labelaid.items():
                   if label.lower().startswith('entail'   ):
                          return ind
            return -1






     def   lowercase_  (			self    :    str,					_snake_case    :    Any,					_snake_case    :    List[str]=True,					_snake_case    :    List[str]=True,					_snake_case    :    Dict=TruncationStrategy.ONLY_FIRST,					**_snake_case    :    Optional[Any]   )  ->str:
            snake_case__				:      Optional[int]        =      self.framework
            if self.tokenizer.pad_token is None:
                   # Override for tokenizers not supporting padding
                   logger.error(
                       'Tokenizer was not supporting padding necessary for zero-shot, attempting to use '
                       ' `pad_token=eos_token`'   )
                   snake_case__				:      Optional[int]        =      self.tokenizer.eos_token
            try:
                   snake_case__				:      Optional[int]        =      self.tokenizer(
                       _snake_case,					add_special_tokens=_snake_case,					return_tensors=_snake_case,					padding=_snake_case,					truncation=_snake_case,					)
            except Exception as e:
                   if "too short" in str(_snake_case   ):
                          # tokenizers might yell that we want to truncate
                          # to a value that is not even reached by the input.
                          # In that case we don't want to truncate.
                          # It seems there's not a really better way to catch that
                          # exception.

                          snake_case__				:      Tuple        =      self.tokenizer(
                              _snake_case,					add_special_tokens=_snake_case,					return_tensors=_snake_case,					padding=_snake_case,					truncation=TruncationStrategy.DO_NOT_TRUNCATE,					)
                   else:
                          raise e

            return inputs






     def   lowercase_  (			self    :    Union[str, Any],					**_snake_case    :    List[str]   )  ->str:
            if kwargs.get('multi_class',					_snake_case   ) is not None:
                   snake_case__				:      List[Any]        =      kwargs['multi_class']
                   logger.warning(
                       'The `multi_class` argument has been deprecated and renamed to `multi_label`. '
                       '`multi_class` will be removed in a future version of Transformers.'   )
            snake_case__				:      int        =      {}
            if "candidate_labels" in kwargs:
                   snake_case__				:      Dict        =      self._args_parser._parse_labels(kwargs['candidate_labels']   )
            if "hypothesis_template" in kwargs:
                   snake_case__				:      Dict        =      kwargs['hypothesis_template']

            snake_case__				:      str        =      {}
            if "multi_label" in kwargs:
                   snake_case__				:      Tuple        =      kwargs['multi_label']
            return preprocess_params, {}, postprocess_params






     def __call__(			self    :    Dict,					_snake_case    :    Union[str, List[str]],					*_snake_case    :    List[Any],					**_snake_case    :    Optional[Any],					)  ->Any:
            if len(_snake_case   ) == 0:
                   pass
            elif len(_snake_case   ) == 1 and "candidate_labels" not in kwargs:
                   snake_case__				:      int        =      args[0]
            else:
                   raise ValueError(F'''Unable to understand extra arguments {args}'''   )

            return super().__call__(_snake_case,					**_snake_case   )






     def   lowercase_  (			self    :    Tuple,					_snake_case    :    str,					_snake_case    :    Union[str, Any]=None,					_snake_case    :    Optional[Any]="This example is {}."   )  ->int:
            snake_case__      ,							snake_case__				:      Optional[Any]        =      self._args_parser(_snake_case,					_snake_case,					_snake_case   )

            for i, (candidate_label, sequence_pair) in enumerate(zip(_snake_case,					_snake_case   )   ):
                   snake_case__				:      Union[str, Any]        =      self._parse_and_tokenize([sequence_pair]   )

                   yield {
                       "candidate_label": candidate_label,
                       "sequence": sequences[0],
                       "is_last": i == len(_snake_case   ) - 1,
                       **model_input,
                   }






     def   lowercase_  (			self    :    Optional[Any],					_snake_case    :    Optional[Any]   )  ->Union[str, Any]:
            snake_case__				:      List[str]        =      inputs['candidate_label']
            snake_case__				:      int        =      inputs['sequence']
            snake_case__				:      Tuple        =      {k: inputs[k] for k in self.tokenizer.model_input_names}
            snake_case__				:      Tuple        =      self.model(**_snake_case   )

            snake_case__				:      Optional[Any]        =      {
                'candidate_label': candidate_label,
                'sequence': sequence,
                'is_last': inputs['is_last'],
                **outputs,
            }
            return model_outputs






     def   lowercase_  (			self    :    int,					_snake_case    :    Optional[Any],					_snake_case    :    List[Any]=False   )  ->Any:
            snake_case__				:      Optional[int]        =      [outputs['candidate_label'] for outputs in model_outputs]
            snake_case__				:      Optional[Any]        =      [outputs['sequence'] for outputs in model_outputs]
            snake_case__				:      List[str]        =      np.concatenate([output['logits'].numpy() for output in model_outputs]   )
            snake_case__				:      List[Any]        =      logits.shape[0]
            snake_case__				:      List[Any]        =      len(_snake_case   )
            snake_case__				:      Optional[Any]        =      N // n
            snake_case__				:      Union[str, Any]        =      logits.reshape((num_sequences, n, -1)   )

            if multi_label or len(_snake_case   ) == 1:
                   # softmax over the entailment vs. contradiction dim for each label independently
                   snake_case__				:      List[Any]        =      self.entailment_id
                   snake_case__				:      int        =      -1 if entailment_id == 0 else 0
                   snake_case__				:      Any        =      reshaped_outputs[..., [contradiction_id, entailment_id]]
                   snake_case__				:      str        =      np.exp(_snake_case   ) / np.exp(_snake_case   ).sum(-1,					keepdims=_snake_case   )
                   snake_case__				:      Optional[int]        =      scores[..., 1]
            else:
                   # softmax the "entailment" logits over all candidate labels
                   snake_case__				:      Any        =      reshaped_outputs[..., self.entailment_id]
                   snake_case__				:      Optional[Any]        =      np.exp(_snake_case   ) / np.exp(_snake_case   ).sum(-1,					keepdims=_snake_case   )

            snake_case__				:      str        =      list(reversed(scores[0].argsort()   )   )
            return {
                "sequence": sequences[0],
                "labels": [candidate_labels[i] for i in top_inds],
                "scores": scores[0, top_inds].tolist(),
            }








"""simple docstring"""





import itertools
import json
import linecache
import os
import pickle
import re
import socket
import string
from collections import Counter
from logging import getLogger
from pathlib import Path
from typing import Callable, Dict, Iterable, List

import git
import torch
from torch.utils.data import Dataset

from transformers import BartTokenizer, RagTokenizer, TaTokenizer
def 				A_  (__a					, __a					, __a					, __a					, __a=True					, __a="pt"					):

					'''simple docstring'''





					A_								=    {"add_prefix_space": True} if isinstance(__a					, __a					) and not line.startswith(" "					) else {}
					A_								=    padding_side
					return tokenizer(
					    [line]					, max_length=__a					, padding="max_length" if pad_to_max_length else None					, truncation=__a					, return_tensors=__a					, add_special_tokens=__a					, **__a					, )
def 				A_  (__a					, __a					, __a=None					, ):

					'''simple docstring'''





					A_								=    input_ids.ne(__a					).any(dim=0					)
					if attention_mask is None:
										return input_ids[:, keep_column_mask]
					else:
										return (input_ids[:, keep_column_mask], attention_mask[:, keep_column_mask])



class __lowerCAmelCase					(       _lowercase   ):




			"""simple docstring"""







			def __init__(					self    :       Optional[Any]							, _snake_case    :       List[Any]							, _snake_case    :       List[Any]							, _snake_case    :       int							, _snake_case    :       Tuple							, _snake_case    :       Dict="train"							, _snake_case    :       List[Any]=None							, _snake_case    :       Optional[int]=None							, _snake_case    :       Optional[int]=None							, _snake_case    :       Any=""							, )   ->			List[str]:







								"""simple docstring"""
								super().__init__()
								A_								=    Path(_snake_case						).joinpath(type_path + ".source"						)
								A_								=    Path(_snake_case						).joinpath(type_path + ".target"						)
								A_								=    self.get_char_lens(self.src_file						)
								A_								=    max_source_length
								A_								=    max_target_length
								assert min(self.src_lens						) > 0, F'found empty line in {self.src_file}'
								A_								=    tokenizer
								A_								=    prefix
								if n_obs is not None:
													A_								=    self.src_lens[:n_obs]
								A_								=    src_lang
								A_								=    tgt_lang




			def __len__(					self    :       Optional[int]						)   ->			List[str]:







								"""simple docstring"""
								return len(self.src_lens						)




			def __getitem__(					self    :       str							, _snake_case    :       Optional[int]						)   ->			Dict[str, torch.Tensor]:







								"""simple docstring"""
								A_								=    index + 1  # linecache starts at 1
								A_								=    self.prefix + linecache.getline(str(self.src_file						)							, _snake_case						).rstrip("\n"						)
								A_								=    linecache.getline(str(self.tgt_file						)							, _snake_case						).rstrip("\n"						)
								assert source_line, F'empty source line for index {index}'
								assert tgt_line, F'empty tgt line for index {index}'

								# Need to add eos token manually for T5
								if isinstance(self.tokenizer							, _snake_case						):
													source_line += self.tokenizer.eos_token
													tgt_line += self.tokenizer.eos_token

								# Pad source and target to the right
								A_								=    (
								    self.tokenizer.question_encoder if isinstance(self.tokenizer							, _snake_case						) else self.tokenizer
								)
								A_								=    self.tokenizer.generator if isinstance(self.tokenizer							, _snake_case						) else self.tokenizer

								A_								=    encode_line(_snake_case							, _snake_case							, self.max_source_length							, "right"						)
								A_								=    encode_line(_snake_case							, _snake_case							, self.max_target_length							, "right"						)

								A_								=    source_inputs["input_ids"].squeeze()
								A_								=    target_inputs["input_ids"].squeeze()
								A_								=    source_inputs["attention_mask"].squeeze()
								return {
								    "input_ids": source_ids,
								    "attention_mask": src_mask,
								    "decoder_input_ids": target_ids,
								}




			@staticmethod
			def   lowerCamelCase__  (					_snake_case    :       List[Any]						)   ->			str:







								"""simple docstring"""
								return [len(_snake_case						) for x in Path(_snake_case						).open().readlines()]






			def   lowerCamelCase__  (					self    :       Any							, _snake_case    :       List[str]						)   ->			Dict[str, torch.Tensor]:







								"""simple docstring"""
								A_								=    torch.stack([x["input_ids"] for x in batch]						)
								A_								=    torch.stack([x["attention_mask"] for x in batch]						)
								A_								=    torch.stack([x["decoder_input_ids"] for x in batch]						)
								A_								=    (
								    self.tokenizer.generator.pad_token_id
								    if isinstance(self.tokenizer							, _snake_case						)
								    else self.tokenizer.pad_token_id
								)
								A_								=    (
								    self.tokenizer.question_encoder.pad_token_id
								    if isinstance(self.tokenizer							, _snake_case						)
								    else self.tokenizer.pad_token_id
								)
								A_								=    trim_batch(_snake_case							, _snake_case						)
								A_			,     A_								=    trim_batch(_snake_case							, _snake_case							, attention_mask=_snake_case						)
								A_								=    {
								    "input_ids": source_ids,
								    "attention_mask": source_mask,
								    "decoder_input_ids": y,
								}
								return batch





UpperCamelCase_       :    Any						      =			getLogger(__name__)
def 				A_  (__a					):

					'''simple docstring'''





					return list(itertools.chain.from_iterable(__a					)					)
def 				A_  (__a					):

					'''simple docstring'''





					A_								=    get_git_info()
					save_json(__a					, os.path.join(__a					, "git_log.json"					)					)
def 				A_  (__a					, __a					, __a=4					, **__a					):

					'''simple docstring'''





					with open(__a					, "w"					) as f:
										json.dump(__a					, __a					, indent=__a					, **__a					)
def 				A_  (__a					):

					'''simple docstring'''





					with open(__a					) as f:
										return json.load(__a					)
def 				A_  ():

					'''simple docstring'''





					A_								=    git.Repo(search_parent_directories=__a					)
					A_								=    {
					    "repo_id": str(__a					),
					    "repo_sha": str(repo.head.object.hexsha					),
					    "repo_branch": str(repo.active_branch					),
					    "hostname": str(socket.gethostname()					),
					}
					return repo_infos
def 				A_  (__a					, __a					):

					'''simple docstring'''





					return list(map(__a					, __a					)					)
def 				A_  (__a					, __a					):

					'''simple docstring'''





					with open(__a					, "wb"					) as f:
										return pickle.dump(__a					, __a					)
def 				A_  (__a					):

					'''simple docstring'''





					def remove_articles(__a					):
										return re.sub(R"\b(a|an|the)\b"					, " "					, __a					)

					def white_space_fix(__a					):
										return " ".join(text.split()					)

					def remove_punc(__a					):
										A_								=    set(string.punctuation					)
										return "".join(ch for ch in text if ch not in exclude					)

					def lower(__a					):
										return text.lower()

					return white_space_fix(remove_articles(remove_punc(lower(__a					)					)					)					)
def 				A_  (__a					, __a					):

					'''simple docstring'''





					A_								=    normalize_answer(__a					).split()
					A_								=    normalize_answer(__a					).split()
					A_								=    Counter(__a					) & Counter(__a					)
					A_								=    sum(common.values()					)
					if num_same == 0:
										return 0
					A_								=    1.0 * num_same / len(__a					)
					A_								=    1.0 * num_same / len(__a					)
					A_								=    (2 * precision * recall) / (precision + recall)
					return fa
def 				A_  (__a					, __a					):

					'''simple docstring'''





					return normalize_answer(__a					) == normalize_answer(__a					)
def 				A_  (__a					, __a					):

					'''simple docstring'''





					assert len(__a					) == len(__a					)
					A_								=    0
					for hypo, pred in zip(__a					, __a					):
										em += exact_match_score(__a					, __a					)
					if len(__a					) > 0:
										em /= len(__a					)
					return {"em": em}
def 				A_  (__a					):

					'''simple docstring'''





					return model_prefix.startswith("rag"					)




def 				A_  (__a					, __a					, __a					):

					'''simple docstring'''





					A_								=    {p: p for p in extra_params}
					# T5 models don't have `dropout` param, they have `dropout_rate` instead
					A_								=    "dropout_rate"
					for p in extra_params:
										if getattr(__a					, __a					, __a					):
															if not hasattr(__a					, __a					) and not hasattr(__a					, equivalent_param[p]					):
																				logger.info("config doesn't have a `{}` attribute".format(__a					)					)
																				delattr(__a					, __a					)
																				continue
															A_								=    p if hasattr(__a					, __a					) else equivalent_param[p]
															setattr(__a					, __a					, getattr(__a					, __a					)					)
															delattr(__a					, __a					)
					return hparams, config




