





'''simple docstring'''



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__magic_name__   :						Any    =			logging.get_logger(__name__)

__magic_name__   :						Union[str, Any]    =			{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

__magic_name__   :						int    =			{
    """vocab_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/vocab.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/vocab.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/vocab.json""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/merges.txt""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/merges.txt""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/merges.txt""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/tokenizer.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/tokenizer.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json""",
        """roberta-base-openai-detector""": (
            """https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"""
        ),
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"""
        ),
    },
}

__magic_name__   :						List[str]    =			{
    """roberta-base""": 512,
    """roberta-large""": 512,
    """roberta-large-mnli""": 512,
    """distilroberta-base""": 512,
    """roberta-base-openai-detector""": 512,
    """roberta-large-openai-detector""": 512,
}


class       __SCREAMING_SNAKE_CASE	(						__UpperCamelCase			):



     '''simple docstring'''






     UpperCAmelCase__       :						Union[str, Any]      =   VOCAB_FILES_NAMES
     UpperCAmelCase__       :						str      =   PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase__       :						List[Any]      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase__       :						Dict      =   ['''input_ids''', '''attention_mask''']
     UpperCAmelCase__       :						Any      =   RobertaTokenizer

     def __init__(					self						,   lowerCamelCase=None						,   lowerCamelCase=None						,   lowerCamelCase=None						,   lowerCamelCase="replace"						,   lowerCamelCase="<s>"						,   lowerCamelCase="</s>"						,   lowerCamelCase="</s>"						,   lowerCamelCase="<s>"						,   lowerCamelCase="<unk>"						,   lowerCamelCase="<pad>"						,   lowerCamelCase="<mask>"						,   lowerCamelCase=False						,   lowerCamelCase=True						,   **lowerCamelCase						,   ):
       super().__init__(
           lowerCamelCase						,   lowerCamelCase						,   tokenizer_file=lowerCamelCase						,   errors=lowerCamelCase						,   bos_token=lowerCamelCase						,   eos_token=lowerCamelCase						,   sep_token=lowerCamelCase						,   cls_token=lowerCamelCase						,   unk_token=lowerCamelCase						,   pad_token=lowerCamelCase						,   mask_token=lowerCamelCase						,   add_prefix_space=lowerCamelCase						,   trim_offsets=lowerCamelCase						,   **lowerCamelCase						,   )

       _snake_case     		=				json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
       if pre_tok_state.get("add_prefix_space"						,   lowerCamelCase	) != add_prefix_space:
         _snake_case     		=				getattr(lowerCamelCase						,   pre_tok_state.pop("type"	)	)
         _snake_case     		=				add_prefix_space
         _snake_case     		=				pre_tok_class(**lowerCamelCase	)

       _snake_case     		=				add_prefix_space

       _snake_case     		=				"post_processor"
       _snake_case     		=				getattr(self.backend_tokenizer						,   lowerCamelCase						,   lowerCamelCase	)
       if tokenizer_component_instance:
         _snake_case     		=				json.loads(tokenizer_component_instance.__getstate__()	)

         # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
         if "sep" in state:
           _snake_case     		=				tuple(state["sep"]	)
         if "cls" in state:
           _snake_case     		=				tuple(state["cls"]	)

         _snake_case     		=				False

         if state.get("add_prefix_space"						,   lowerCamelCase	) != add_prefix_space:
           _snake_case     		=				add_prefix_space
           _snake_case     		=				True

         if state.get("trim_offsets"						,   lowerCamelCase	) != trim_offsets:
           _snake_case     		=				trim_offsets
           _snake_case     		=				True

         if changes_to_apply:
           _snake_case     		=				getattr(lowerCamelCase						,   state.pop("type"	)	)
           _snake_case     		=				component_class(**lowerCamelCase	)
           setattr(self.backend_tokenizer						,   lowerCamelCase						,   lowerCamelCase	)

     @property
     def 	UpperCamelCase(					self	):
       if self._mask_token is None:
         if self.verbose:
           logger.error("Using mask_token, but it is not set yet."	)
         return None
       return str(self._mask_token	)

     @mask_token.setter
     def 	UpperCamelCase(					self						,   lowerCamelCase	):
       _snake_case     		=				AddedToken(lowerCamelCase						,   lstrip=lowerCamelCase						,   rstrip=lowerCamelCase	) if isinstance(lowerCamelCase						,   lowerCamelCase	) else value
       _snake_case     		=				value

     def 	UpperCamelCase(					self						,   *lowerCamelCase						,   **lowerCamelCase	):
       _snake_case     		=				kwargs.get("is_split_into_words"						,   lowerCamelCase	)
       assert self.add_prefix_space or not is_split_into_words, (
           F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
           "to use it with pretokenized inputs."
       )

       return super()._batch_encode_plus(*lowerCamelCase						,   **lowerCamelCase	)

     def 	UpperCamelCase(					self						,   *lowerCamelCase						,   **lowerCamelCase	):
       _snake_case     		=				kwargs.get("is_split_into_words"						,   lowerCamelCase	)

       assert self.add_prefix_space or not is_split_into_words, (
           F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
           "to use it with pretokenized inputs."
       )

       return super()._encode_plus(*lowerCamelCase						,   **lowerCamelCase	)

     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase = None	):
       _snake_case     		=				self._tokenizer.model.save(lowerCamelCase						,   name=lowerCamelCase	)
       return tuple(lowerCamelCase	)

     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase=None	):
       _snake_case     		=				[self.bos_token_id] + token_ids_a + [self.eos_token_id]
       if token_ids_a is None:
         return output

       return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






     def 	UpperCamelCase(					self						,   lowerCamelCase						,   lowerCamelCase = None	):
       _snake_case     		=				[self.sep_token_id]
       _snake_case     		=				[self.cls_token_id]

       if token_ids_a is None:
         return len(cls + token_ids_a + sep	) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]









'''simple docstring'''



import warnings

from ...utils import is_sklearn_available, requires_backends


if is_sklearn_available():
 from scipy.stats import pearsonr, spearmanr
 from sklearn.metrics import fa_score, matthews_corrcoef


__magic_name__   :						List[str]    =			(
    """This metric will be removed from the library soon, metrics should be handled with the 🤗 Evaluate """
    """library. You can have a look at this example script for pointers: """
    """https://github.com/huggingface/transformers/blob/main/examples/pytorch/text-classification/run_glue.py"""
)




def 				snake_case_      (		SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       ):







  '''simple docstring'''



  warnings.warn(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )
  requires_backends(SCREAMING_SNAKE_CASE__					,	"sklearn"       )
  return (preds == labels).mean()




def 				snake_case_      (		SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       ):







  '''simple docstring'''



  warnings.warn(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )
  requires_backends(SCREAMING_SNAKE_CASE__					,	"sklearn"       )
  _snake_case     		=				simple_accuracy(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )
  _snake_case     		=				fa_score(y_true=SCREAMING_SNAKE_CASE__					,	y_pred=SCREAMING_SNAKE_CASE__       )
  return {
      "acc": acc,
      "f1": fa,
      "acc_and_f1": (acc + fa) / 2,
  }




def 				snake_case_      (		SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       ):







  '''simple docstring'''



  warnings.warn(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )
  requires_backends(SCREAMING_SNAKE_CASE__					,	"sklearn"       )
  _snake_case     		=				pearsonr(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )[0]
  _snake_case     		=				spearmanr(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )[0]
  return {
      "pearson": pearson_corr,
      "spearmanr": spearman_corr,
      "corr": (pearson_corr + spearman_corr) / 2,
  }




def 				snake_case_      (		SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       ):







  '''simple docstring'''



  warnings.warn(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )
  requires_backends(SCREAMING_SNAKE_CASE__					,	"sklearn"       )
  assert len(SCREAMING_SNAKE_CASE__       ) == len(SCREAMING_SNAKE_CASE__       ), f'''Predictions and labels have mismatched lengths {len(SCREAMING_SNAKE_CASE__       )} and {len(SCREAMING_SNAKE_CASE__       )}'''
  if task_name == "cola":
    return {"mcc": matthews_corrcoef(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )}
  elif task_name == "sst-2":
    return {"acc": simple_accuracy(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )}
  elif task_name == "mrpc":
    return acc_and_fa(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )
  elif task_name == "sts-b":
    return pearson_and_spearman(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )
  elif task_name == "qqp":
    return acc_and_fa(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )
  elif task_name == "mnli":
    return {"mnli/acc": simple_accuracy(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )}
  elif task_name == "mnli-mm":
    return {"mnli-mm/acc": simple_accuracy(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )}
  elif task_name == "qnli":
    return {"acc": simple_accuracy(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )}
  elif task_name == "rte":
    return {"acc": simple_accuracy(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )}
  elif task_name == "wnli":
    return {"acc": simple_accuracy(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )}
  elif task_name == "hans":
    return {"acc": simple_accuracy(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )}
  else:
    raise KeyError(SCREAMING_SNAKE_CASE__       )




def 				snake_case_      (		SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       ):







  '''simple docstring'''



  warnings.warn(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )
  requires_backends(SCREAMING_SNAKE_CASE__					,	"sklearn"       )
  if len(SCREAMING_SNAKE_CASE__       ) != len(SCREAMING_SNAKE_CASE__       ):
    raise ValueError(f'''Predictions and labels have mismatched lengths {len(SCREAMING_SNAKE_CASE__       )} and {len(SCREAMING_SNAKE_CASE__       )}'''       )
  if task_name == "xnli":
    return {"acc": simple_accuracy(SCREAMING_SNAKE_CASE__					,	SCREAMING_SNAKE_CASE__       )}
  else:
    raise KeyError(SCREAMING_SNAKE_CASE__       )


