'''simple docstring'''






import logging
import os
from dataclasses import dataclass
from enum import Enum
from typing import List, Optional, Union

from filelock import FileLock

from transformers import PreTrainedTokenizer, is_tf_available, is_torch_available


__a									=   logging.getLogger(__name__)





@dataclass
class        A__				:





   """simple docstring"""





   UpperCamelCase_			:		str
   UpperCamelCase_			:		List[str]
   UpperCamelCase_			:		Optional[List[str]]





@dataclass
class        A__				:





   """simple docstring"""





   UpperCamelCase_			:		List[int]
   UpperCamelCase_			:		List[int]
   UpperCamelCase_			:		Optional[List[int]]       							=	None
   UpperCamelCase_			:		Optional[List[int]]       							=	None





class        A__				(			_SCREAMING_SNAKE_CASE  ):





   """simple docstring"""





   UpperCamelCase_			:		Optional[Any]       							=	"""train"""
   UpperCamelCase_			:		List[Any]       							=	"""dev"""
   UpperCamelCase_			:		Dict       							=	"""test"""





class        A__				:





   """simple docstring"""





   @staticmethod
   def      _lowerCAmelCase						(					lowerCAmelCase__	:						Any		,	lowerCAmelCase__	:						Union[str, Any]     )			->       Union[str, Any]:







      """simple docstring"""




      raise NotImplementedError




   @staticmethod
   def      _lowerCAmelCase						(					lowerCAmelCase__	:						Optional[int]     )			->       Any:







      """simple docstring"""




      raise NotImplementedError




   @staticmethod
   def      _lowerCAmelCase						(					lowerCAmelCase__	:						Optional[int]		,	lowerCAmelCase__	:						Tuple		,	lowerCAmelCase__	:						Any		,	lowerCAmelCase__	:						Union[str, Any]		,	lowerCAmelCase__	:						Optional[int]=False		,	lowerCAmelCase__	:						str="[CLS]"		,	lowerCAmelCase__	:						Tuple=1		,	lowerCAmelCase__	:						str="[SEP]"		,	lowerCAmelCase__	:						int=False		,	lowerCAmelCase__	:						str=False		,	lowerCAmelCase__	:						List[str]=0		,	lowerCAmelCase__	:						Optional[int]=0		,	lowerCAmelCase__	:						Tuple=-1_0_0		,	lowerCAmelCase__	:						int=0		,	lowerCAmelCase__	:						Tuple=True		,	)			->       Tuple:







      """simple docstring"""




      _UpperCAmelCase				:      int				=  {label: i for i, label in enumerate(lowerCAmelCase_     )}

      _UpperCAmelCase				:      int				=  []
      for ex_index, example in enumerate(lowerCAmelCase_     ):
         if ex_index % 1_0_0_0_0 == 0:
            logger.info("Writing example %d of %d"		,	lowerCAmelCase_		,	len(lowerCAmelCase_     )     )

         _UpperCAmelCase				:      List[str]				=  []
         _UpperCAmelCase				:      str				=  []
         for word, label in zip(example.words		,	example.labels     ):
            _UpperCAmelCase				:      Any				=  tokenizer.tokenize(lowerCAmelCase_     )

            # bert-base-multilingual-cased sometimes output "nothing ([]) when calling tokenize with just a space.
            if len(lowerCAmelCase_     ) > 0:
               tokens.extend(lowerCAmelCase_     )
               # Use the real label id for the first token of the word, and padding ids for the remaining tokens
               label_ids.extend([label_map[label]] + [pad_token_label_id] * (len(lowerCAmelCase_     ) - 1)     )

            # Account for [CLS] and [SEP] with "- 2" and with "- 3" for RoBERTa.
         _UpperCAmelCase				:      Optional[int]				=  tokenizer.num_special_tokens_to_add()
         if len(lowerCAmelCase_     ) > max_seq_length - special_tokens_count:
            _UpperCAmelCase				:      int				=  tokens[: (max_seq_length - special_tokens_count)]
            _UpperCAmelCase				:      Union[str, Any]				=  label_ids[: (max_seq_length - special_tokens_count)]

         # The convention in BERT is:
         # (a) For sequence pairs:
         #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
         #  type_ids:   0   0  0    0    0     0       0   0   1  1  1  1   1   1
         # (b) For single sequences:
         #  tokens:   [CLS] the dog is hairy . [SEP]
         #  type_ids:   0   0   0   0  0     0   0
         #
         # Where "type_ids" are used to indicate whether this is the first
         # sequence or the second sequence. The embedding vectors for `type=0` and
         # `type=1` were learned during pre-training and are added to the wordpiece
         # embedding vector (and position vector). This is not *strictly* necessary
         # since the [SEP] token unambiguously separates the sequences, but it makes
         # it easier for the model to learn the concept of sequences.
         #
         # For classification tasks, the first vector (corresponding to [CLS]) is
         # used as the "sentence vector". Note that this only makes sense because
         # the entire model is fine-tuned.
         tokens += [sep_token]
         label_ids += [pad_token_label_id]
         if sep_token_extra:
            # roberta uses an extra separator b/w pairs of sentences
            tokens += [sep_token]
            label_ids += [pad_token_label_id]
         _UpperCAmelCase				:      Any				=  [sequence_a_segment_id] * len(lowerCAmelCase_     )

         if cls_token_at_end:
            tokens += [cls_token]
            label_ids += [pad_token_label_id]
            segment_ids += [cls_token_segment_id]
         else:
            _UpperCAmelCase				:      str				=  [cls_token] + tokens
            _UpperCAmelCase				:      Dict				=  [pad_token_label_id] + label_ids
            _UpperCAmelCase				:      str				=  [cls_token_segment_id] + segment_ids

         _UpperCAmelCase				:      Optional[int]				=  tokenizer.convert_tokens_to_ids(lowerCAmelCase_     )

         # The mask has 1 for real tokens and 0 for padding tokens. Only real
         # tokens are attended to.
         _UpperCAmelCase				:      Any				=  [1 if mask_padding_with_zero else 0] * len(lowerCAmelCase_     )

         # Zero-pad up to the sequence length.
         _UpperCAmelCase				:      Tuple				=  max_seq_length - len(lowerCAmelCase_     )
         if pad_on_left:
            _UpperCAmelCase				:      Any				=  ([pad_token] * padding_length) + input_ids
            _UpperCAmelCase				:      Optional[Any]				=  ([0 if mask_padding_with_zero else 1] * padding_length) + input_mask
            _UpperCAmelCase				:      List[str]				=  ([pad_token_segment_id] * padding_length) + segment_ids
            _UpperCAmelCase				:      Optional[int]				=  ([pad_token_label_id] * padding_length) + label_ids
         else:
            input_ids += [pad_token] * padding_length
            input_mask += [0 if mask_padding_with_zero else 1] * padding_length
            segment_ids += [pad_token_segment_id] * padding_length
            label_ids += [pad_token_label_id] * padding_length

         assert len(lowerCAmelCase_     ) == max_seq_length
         assert len(lowerCAmelCase_     ) == max_seq_length
         assert len(lowerCAmelCase_     ) == max_seq_length
         assert len(lowerCAmelCase_     ) == max_seq_length

         if ex_index < 5:
            logger.info("*** Example ***"     )
            logger.info("guid: %s"		,	example.guid     )
            logger.info("tokens: %s"		,	" ".join([str(lowerCAmelCase_     ) for x in tokens]     )     )
            logger.info("input_ids: %s"		,	" ".join([str(lowerCAmelCase_     ) for x in input_ids]     )     )
            logger.info("input_mask: %s"		,	" ".join([str(lowerCAmelCase_     ) for x in input_mask]     )     )
            logger.info("segment_ids: %s"		,	" ".join([str(lowerCAmelCase_     ) for x in segment_ids]     )     )
            logger.info("label_ids: %s"		,	" ".join([str(lowerCAmelCase_     ) for x in label_ids]     )     )

         if "token_type_ids" not in tokenizer.model_input_names:
            _UpperCAmelCase				:      List[str]				=  None

         features.append(
             InputFeatures(
                 input_ids=lowerCAmelCase_		,	attention_mask=lowerCAmelCase_		,	token_type_ids=lowerCAmelCase_		,	label_ids=lowerCAmelCase_     )     )
      return features


if is_torch_available():
     import torch
     from torch import nn
     from torch.utils.data import Dataset





     class        A__				(			_SCREAMING_SNAKE_CASE  ):





        """simple docstring"""





        UpperCamelCase_			:		List[InputFeatures]
        UpperCamelCase_			:		int       							=	nn.CrossEntropyLoss().ignore_index




        def __init__(					self	:						Tuple		,	lowerCAmelCase__	:						Dict		,	lowerCAmelCase__	:						List[str]		,	lowerCAmelCase__	:						str		,	lowerCAmelCase__	:						str		,	lowerCAmelCase__	:						Optional[Any]		,	lowerCAmelCase__	:						Optional[int] = None		,	lowerCAmelCase__	:						Dict=False		,	lowerCAmelCase__	:						Dict = Split.train		,	)			->       str:







           """simple docstring"""




           _UpperCAmelCase				:      Any				=  os.path.join(
               lowerCAmelCase_		,	"cached_{}_{}_{}".format(mode.value		,	tokenizer.__class__.__name__		,	str(lowerCAmelCase_     )     )		,	)

           # Make sure only the first process in distributed training processes the dataset,
           # and the others will use the cache.
           _UpperCAmelCase				:      List[str]				=  cached_features_file + """.lock"""
           with FileLock(lowerCAmelCase_     ):
              if os.path.exists(lowerCAmelCase_     ) and not overwrite_cache:
                 logger.info(F"""Loading features from cached file {cached_features_file}"""     )
                 _UpperCAmelCase				:      Dict				=  torch.load(lowerCAmelCase_     )
              else:
                 logger.info(F"""Creating features from dataset file at {data_dir}"""     )
                 _UpperCAmelCase				:      int				=  token_classification_task.read_examples_from_file(lowerCAmelCase_		,	lowerCAmelCase_     )
                 # TODO clean up all this to leverage built-in features of tokenizers
                 _UpperCAmelCase				:      List[str]				=  token_classification_task.convert_examples_to_features(
                     lowerCAmelCase_		,	lowerCAmelCase_		,	lowerCAmelCase_		,	lowerCAmelCase_		,	cls_token_at_end=bool(model_type in ["xlnet"]     )		,	cls_token=tokenizer.cls_token		,	cls_token_segment_id=2 if model_type in ["xlnet"] else 0		,	sep_token=tokenizer.sep_token		,	sep_token_extra=lowerCAmelCase_		,	pad_on_left=bool(tokenizer.padding_side == "left"     )		,	pad_token=tokenizer.pad_token_id		,	pad_token_segment_id=tokenizer.pad_token_type_id		,	pad_token_label_id=self.pad_token_label_id		,	)
                 logger.info(F"""Saving features into cached file {cached_features_file}"""     )
                 torch.save(self.features		,	lowerCAmelCase_     )




        def __len__(					self	:						Any     )			->       List[Any]:







           """simple docstring"""




           return len(self.features     )




        def __getitem__(					self	:						Optional[Any]		,	lowerCAmelCase__	:						Optional[Any]     )			->       Optional[Any]:







           """simple docstring"""




           return self.features[i]


if is_tf_available():
     import tensorflow as tf





     class        A__				:





        """simple docstring"""





        UpperCamelCase_			:		List[InputFeatures]
        UpperCamelCase_			:		int       							=	-1_00




        def __init__(					self	:						Optional[int]		,	lowerCAmelCase__	:						Tuple		,	lowerCAmelCase__	:						Tuple		,	lowerCAmelCase__	:						List[Any]		,	lowerCAmelCase__	:						List[Any]		,	lowerCAmelCase__	:						Dict		,	lowerCAmelCase__	:						Dict = None		,	lowerCAmelCase__	:						Optional[int]=False		,	lowerCAmelCase__	:						Any = Split.train		,	)			->       Optional[Any]:







           """simple docstring"""




           _UpperCAmelCase				:      str				=  token_classification_task.read_examples_from_file(lowerCAmelCase_		,	lowerCAmelCase_     )
           # TODO clean up all this to leverage built-in features of tokenizers
           _UpperCAmelCase				:      Optional[int]				=  token_classification_task.convert_examples_to_features(
               lowerCAmelCase_		,	lowerCAmelCase_		,	lowerCAmelCase_		,	lowerCAmelCase_		,	cls_token_at_end=bool(model_type in ["xlnet"]     )		,	cls_token=tokenizer.cls_token		,	cls_token_segment_id=2 if model_type in ["xlnet"] else 0		,	sep_token=tokenizer.sep_token		,	sep_token_extra=lowerCAmelCase_		,	pad_on_left=bool(tokenizer.padding_side == "left"     )		,	pad_token=tokenizer.pad_token_id		,	pad_token_segment_id=tokenizer.pad_token_type_id		,	pad_token_label_id=self.pad_token_label_id		,	)

           def gen():
              for ex in self.features:
                 if ex.token_type_ids is None:
                    yield (
                        {"input_ids": ex.input_ids, "attention_mask": ex.attention_mask},
                        ex.label_ids,
                    )
                 else:
                    yield (
                        {
                            "input_ids": ex.input_ids,
                            "attention_mask": ex.attention_mask,
                            "token_type_ids": ex.token_type_ids,
                        },
                        ex.label_ids,
                    )

           if "token_type_ids" not in tokenizer.model_input_names:
              _UpperCAmelCase				:      List[Any]				=  tf.data.Dataset.from_generator(
                  lowerCAmelCase_		,	({"input_ids": tf.intaa, "attention_mask": tf.intaa}, tf.intaa)		,	(
                      {"input_ids": tf.TensorShape([None]     ), "attention_mask": tf.TensorShape([None]     )},
                      tf.TensorShape([None]     ),
                  )		,	)
           else:
              _UpperCAmelCase				:      Tuple				=  tf.data.Dataset.from_generator(
                  lowerCAmelCase_		,	({"input_ids": tf.intaa, "attention_mask": tf.intaa, "token_type_ids": tf.intaa}, tf.intaa)		,	(
                      {
                          "input_ids": tf.TensorShape([None]     ),
                          "attention_mask": tf.TensorShape([None]     ),
                          "token_type_ids": tf.TensorShape([None]     ),
                      },
                      tf.TensorShape([None]     ),
                  )		,	)




        def      _lowerCAmelCase						(					self	:						str     )			->       Dict:







           """simple docstring"""




           _UpperCAmelCase				:      Optional[int]				=  self.dataset.apply(tf.data.experimental.assert_cardinality(len(self.features     )     )     )

           return self.dataset




        def __len__(					self	:						str     )			->       Tuple:







           """simple docstring"""




           return len(self.features     )




        def __getitem__(					self	:						int		,	lowerCAmelCase__	:						Dict     )			->       str:







           """simple docstring"""




           return self.features[i]







"""simple docstring"""
import warnings
from typing import Dict

import numpy as np

from ..utils import ExplicitEnum, add_end_docstrings, is_tf_available, is_torch_available
from .base import PIPELINE_INIT_ARGS, GenericTensor, Pipeline


if is_tf_available():
							from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

if is_torch_available():
							from ..models.auto.modeling_auto import MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING






def       __UpperCamelCase						(		snake_case__       ):
	return 1.0 / (1.0 + np.exp(-_outputs       ))






def       __UpperCamelCase						(		snake_case__       ):
	A_		:    Union[str, Any]						     = np.max(_outputs , axis=-1 , keepdims=snake_case__       )
	A_		:    str						     = np.exp(_outputs - maxes       )
	return shifted_exp / shifted_exp.sum(axis=-1 , keepdims=snake_case__       )



class   SCREAMING_SNAKE_CASE	(					_SCREAMING_SNAKE_CASE    ):





	"""simple docstring"""




	_A							:					Any 			=					"""sigmoid"""
	_A							:					Any 			=					"""softmax"""
	_A							:					Union[str, Any] 			=					"""none"""






@add_end_docstrings(
    _SCREAMING_SNAKE_CASE							,							R"""
        return_all_scores (`bool`, *optional*, defaults to `False`):
            Whether to return all prediction scores or just the one of the predicted class.
        function_to_apply (`str`, *optional*, defaults to `\"default\"`):
            The function to apply to the model outputs in order to retrieve the scores. Accepts four different values:

            - `\"default\"`: if the model has a single label, will apply the sigmoid function on the output. If the model
              has several labels, will apply the softmax function on the output.
            - `\"sigmoid\"`: Applies the sigmoid function on the output.
            - `\"softmax\"`: Applies the softmax function on the output.
            - `\"none\"`: Does not apply any function on the output.
    """							,							)
class   SCREAMING_SNAKE_CASE	(					_SCREAMING_SNAKE_CASE    ):





	"""simple docstring"""




	_A							:					Optional[int] 			=					False
	_A							:					Dict 			=					ClassificationFunction.NONE





	def __init__(self  ,  **lowerCAmelCase_				):
		super().__init__(**lowerCAmelCase_				)

		self.check_model_type(
		    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
		    if self.framework == """tf"""
		    else MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_=None  ,  lowerCAmelCase_=None  ,  lowerCAmelCase_=""  ,  **lowerCAmelCase_				):
		# Using "" as default argument because we're going to use `top_k=None` in user code to declare
		# "No top_k"
		A_		:    Union[str, Any]						     = tokenizer_kwargs

		A_		:    List[str]						     = {}
		if hasattr(self.model.config  ,  """return_all_scores"""				) and return_all_scores is None:
			A_		:    Optional[int]						     = self.model.config.return_all_scores

		if isinstance(lowerCAmelCase_  ,  lowerCAmelCase_				) or top_k is None:
			A_		:    Dict						     = top_k
			A_		:    Any						     = False
		elif return_all_scores is not None:
			warnings.warn(
			    """`return_all_scores` is now deprecated,  if want a similar functionality use `top_k=None` instead of"""
			    """ `return_all_scores=True` or `top_k=1` instead of `return_all_scores=False`."""  ,  lowerCAmelCase_  ,  )
			if return_all_scores:
				A_		:    List[Any]						     = None
			else:
				A_		:    List[str]						     = 1

		if isinstance(lowerCAmelCase_  ,  lowerCAmelCase_				):
			A_		:    str						     = ClassificationFunction[function_to_apply.upper()]

		if function_to_apply is not None:
			A_		:    List[str]						     = function_to_apply
		return preprocess_params, {}, postprocess_params





	def __call__(self  ,  *lowerCAmelCase_  ,  **lowerCAmelCase_				):
		A_		:    List[str]						     = super().__call__(*lowerCAmelCase_  ,  **lowerCAmelCase_				)
		# TODO try and retrieve it in a nicer way from _sanitize_parameters.
		A_		:    Union[str, Any]						     = """top_k""" not in kwargs
		if isinstance(args[0]  ,  lowerCAmelCase_				) and _legacy:
			# This pipeline is odd, and return a list when single item is run
			return [result]
		else:
			return result





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  **lowerCAmelCase_				):
		A_		:    Union[str, Any]						     = self.framework
		if isinstance(lowerCAmelCase_  ,  lowerCAmelCase_				):
			return self.tokenizer(**lowerCAmelCase_  ,  return_tensors=lowerCAmelCase_  ,  **lowerCAmelCase_				)
		elif isinstance(lowerCAmelCase_  ,  lowerCAmelCase_				) and len(lowerCAmelCase_				) == 1 and isinstance(inputs[0]  ,  lowerCAmelCase_				) and len(inputs[0]				) == 2:
			# It used to be valid to use a list of list of list for text pairs, keeping this path for BC
			return self.tokenizer(
			    text=inputs[0][0]  ,  text_pair=inputs[0][1]  ,  return_tensors=lowerCAmelCase_  ,  **lowerCAmelCase_				)
		elif isinstance(lowerCAmelCase_  ,  lowerCAmelCase_				):
			# This is likely an invalid usage of the pipeline attempting to pass text pairs.
			raise ValueError(
			    """The pipeline received invalid inputs, if you are trying to send text pairs, you can try to send a"""
			    """ dictionary `{\"text\": \"My text\", \"text_pair\": \"My pair\"}` in order to send a text pair."""				)
		return self.tokenizer(lowerCAmelCase_  ,  return_tensors=lowerCAmelCase_  ,  **lowerCAmelCase_				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		return self.model(**lowerCAmelCase_				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_=None  ,  lowerCAmelCase_=1  ,  lowerCAmelCase_=True				):
		# `_legacy` is used to determine if we're running the naked pipeline and in backward
		# compatibility mode, or if running the pipeline with `pipeline(..., top_k=1)` we're running
		# the more natural result containing the list.
		# Default value before `set_parameters`
		if function_to_apply is None:
			if self.model.config.problem_type == "multi_label_classification" or self.model.config.num_labels == 1:
				A_		:    Optional[Any]						     = ClassificationFunction.SIGMOID
			elif self.model.config.problem_type == "single_label_classification" or self.model.config.num_labels > 1:
				A_		:    Tuple						     = ClassificationFunction.SOFTMAX
			elif hasattr(self.model.config  ,  """function_to_apply"""				) and function_to_apply is None:
				A_		:    Any						     = self.model.config.function_to_apply
			else:
				A_		:    Dict						     = ClassificationFunction.NONE

		A_		:    Optional[Any]						     = model_outputs["""logits"""][0]
		A_		:    Tuple						     = outputs.numpy()

		if function_to_apply == ClassificationFunction.SIGMOID:
			A_		:    str						     = sigmoid(lowerCAmelCase_				)
		elif function_to_apply == ClassificationFunction.SOFTMAX:
			A_		:    Dict						     = softmax(lowerCAmelCase_				)
		elif function_to_apply == ClassificationFunction.NONE:
			A_		:    Optional[int]						     = outputs
		else:
			raise ValueError(f"""Unrecognized `function_to_apply` argument: {function_to_apply}"""				)

		if top_k == 1 and _legacy:
			return {"label": self.model.config.idalabel[scores.argmax().item()], "score": scores.max().item()}

		A_		:    Optional[Any]						     = [
		    {"""label""": self.model.config.idalabel[i], """score""": score.item()} for i, score in enumerate(lowerCAmelCase_				)
		]
		if not _legacy:
			dict_scores.sort(key=lambda lowerCAmelCase_				: x["score"]  ,  reverse=lowerCAmelCase_				)
			if top_k is not None:
				A_		:    str						     = dict_scores[:top_k]
		return dict_scores

