




import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
							import torch


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


UpperCAmelCase_		:					Optional[int]          =  logging.get_logger(__name__)
UpperCAmelCase_		:					List[Any]          =  {"vocab_file": "spiece.model"}

UpperCAmelCase_		:					List[Any]          =  {
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

UpperCAmelCase_		:					Optional[Any]          =  {
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}




class 		__A   (      UpperCamelCase__      ):
						UpperCamelCase      =       VOCAB_FILES_NAMES
						UpperCamelCase      =       PRETRAINED_VOCAB_FILES_MAP
						UpperCamelCase      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCamelCase      =       ["""input_ids""", """attention_mask"""]

						def __init__(		self :List[Any]     ,			__snake_case :Union[str, Any]     ,			__snake_case :str=False     ,			__snake_case :List[str]=False     ,			__snake_case :str=False     ,			__snake_case :str=None     ,			__snake_case :List[str]=None     ,			__snake_case :Tuple=None     ,			__snake_case :Dict=None     ,			__snake_case :Optional[Dict[str, Any]] = None     ,			**__snake_case :Optional[Any]     ,			):
										'''simple docstring'''

										__magic_name__						:						Union[str, Any]										={} if sp_model_kwargs is None else sp_model_kwargs

										__magic_name__						:						List[Any]										=kwargs.get("""name_or_path"""			)
										if name_or_path is None:
														logger.warning(
														    """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
														    """ you are testing the model, this can safely be ignored"""			)
														__magic_name__						:						Any										="""None"""

										# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
										__magic_name__						:						Optional[int]										="""<|endoftext|>""" if eos_token is None else eos_token
										__magic_name__						:						List[Any]										="""<unk>""" if unk_token is None else unk_token
										if "gpt-sw3-7b" in name_or_path:
														__magic_name__						:						int										=unk_token if pad_token is None else pad_token
														__magic_name__						:						int										=eos_token if bos_token is None else bos_token
										else:
														__magic_name__						:						Optional[Any]										="""<pad>""" if pad_token is None else pad_token
														__magic_name__						:						Optional[int]										="""<s>""" if bos_token is None else bos_token

										super().__init__(
										    do_lower_case=__snake_case     ,			remove_space=__snake_case     ,			keep_accents=__snake_case     ,			bos_token=__snake_case     ,			eos_token=__snake_case     ,			unk_token=__snake_case     ,			pad_token=__snake_case     ,			sp_model_kwargs=self.sp_model_kwargs     ,			**__snake_case     ,			)

										__magic_name__						:						Tuple										=do_lower_case
										__magic_name__						:						List[str]										=remove_space
										__magic_name__						:						Optional[Any]										=keep_accents
										__magic_name__						:						int										=vocab_file

										__magic_name__						:						str										=spm.SentencePieceProcessor(**self.sp_model_kwargs			)
										self.sp_model.Load(__snake_case			)

										# Used for whitespace normalization in input texts
										# fmt : off
										__magic_name__						:						List[Any]										={""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
										# fmt : on

										# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
										__magic_name__						:						str										=re.compile(
										    f"[{''.join(map(__snake_case     ,			list(range(0     ,			9			)			) + list(range(11     ,			32			)			) + list(range(1_27     ,			1_60			)			) + [1_60, 1_73, 82_03]			)			)}]"			)

						def __getstate__(		self :Dict			):
										'''simple docstring'''

										__magic_name__						:						Optional[int]										=self.__dict__.copy()
										__magic_name__						:						List[str]										=None
										return state

						def __setstate__(		self :List[Any]     ,			__snake_case :List[str]			):
										'''simple docstring'''

										__magic_name__						:						List[Any]										=d

										# for backward compatibility
										if not hasattr(self     ,			"""sp_model_kwargs"""			):
														__magic_name__						:						Dict										={}

										__magic_name__						:						Optional[int]										=spm.SentencePieceProcessor(**self.sp_model_kwargs			)
										self.sp_model.Load(self.vocab_file			)

						@property
						# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
						def      A__    (		self :Any			):
										'''simple docstring'''

										return len(self.sp_model			)

						def      A__    (		self :Any     ,			__snake_case :str			):
										'''simple docstring'''

										__magic_name__						:						Optional[int]										=self.non_printing_characters_re.sub(""""""     ,			__snake_case			)

										# Normalize whitespaces
										__magic_name__						:						Union[str, Any]										="""""".join([char if char not in self.whitespaces else """ """ for char in text]			)

										# NFC Unicode normalization
										__magic_name__						:						List[str]										=unicodedata.normalize("""NFC"""     ,			__snake_case			)
										return text

						def      A__    (		self :Dict     ,			__snake_case :str     ,			**__snake_case :Optional[int]			):
										'''simple docstring'''

										__magic_name__						:						str										=self.preprocess_text(__snake_case			)
										return self.sp_model.encode(__snake_case     ,			out_type=__snake_case			)

						def      A__    (		self :Union[str, Any]     ,			__snake_case :str			):
										'''simple docstring'''

										return self.sp_model.PieceToId(__snake_case			)

						def      A__    (		self :Dict     ,			__snake_case :int			):
										'''simple docstring'''

										return self.sp_model.IdToPiece(__snake_case			)

						@staticmethod
						def      A__    (		__snake_case :str			):
										'''simple docstring'''

										return out_string

						def      A__    (		self :Optional[int]     ,			__snake_case :List[str]			):
										'''simple docstring'''

										__magic_name__						:						Optional[int]										=[]
										__magic_name__						:						List[Any]										=""""""
										__magic_name__						:						Union[str, Any]										=False
										for token in tokens:
														# make sure that special tokens are not decoded using sentencepiece model
														if token in self.all_special_tokens:
																		# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																		if not prev_is_special:
																						out_string += " "

																		out_string += self.sp_model.decode(__snake_case			) + token
																		__magic_name__						:						int										=True
																		__magic_name__						:						str										=[]
														else:
																		current_sub_tokens.append(__snake_case			)
																		__magic_name__						:						List[Any]										=False
										out_string += self.sp_model.decode(__snake_case			)

										return out_string

						def      A__    (		self :Any			):
										'''simple docstring'''

										__magic_name__						:						List[Any]										={self.convert_ids_to_tokens(__snake_case			): i for i in range(self.vocab_size			)}
										vocab.update(self.added_tokens_encoder			)
										return vocab

						def      A__    (		self :List[str]     ,			__snake_case :str     ,			__snake_case :Optional[str] = None			):
										'''simple docstring'''

										if not os.path.isdir(__snake_case			):
														logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
														return
										__magic_name__						:						Tuple										=os.path.join(
										    __snake_case     ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

										if os.path.abspath(self.vocab_file			) != os.path.abspath(__snake_case			) and os.path.isfile(self.vocab_file			):
														copyfile(self.vocab_file     ,			__snake_case			)
										elif not os.path.isfile(self.vocab_file			):
														with open(__snake_case     ,			"""wb"""			) as fi:
																		__magic_name__						:						Dict										=self.sp_model.serialized_model_proto()
																		fi.write(__snake_case			)

										return (out_vocab_file,)

						def      A__    (		self :Dict     ,			__snake_case :Union[str, List[str]]     ,			__snake_case :Union[str, bool] = False			):
										'''simple docstring'''

										if isinstance(__snake_case     ,			__snake_case			):
														__magic_name__						:						List[str]										=self.preprocess_text(__snake_case			)
														__magic_name__						:						Optional[int]										=self.sp_model.encode(__snake_case			)
										else:
														__magic_name__						:						str										=[self.preprocess_text(__snake_case			) for t in text]
														__magic_name__						:						Optional[Any]										=self.sp_model.encode(__snake_case			)

										if return_tensors is True or return_tensors == "pt":
														__magic_name__						:						Any										=torch.tensor(__snake_case			)

										return token_ids

						def      A__    (		self :Tuple     ,			__snake_case :Union[int, List[int]]			):
										'''simple docstring'''

										return self.sp_model.decode(__snake_case			)


						def      A__    (		self :Any     ,			__snake_case :"Conversation"			):
										'''simple docstring'''

										__magic_name__						:						List[str]										=[f"User: {text}" if is_user else f"Bot: {text}" for is_user, text in conversation.iter_texts()]
										__magic_name__						:						str										=(
										    f"{self.eos_token}{self.bos_token}" + f"{self.bos_token}".join(__snake_case			) + f"{self.bos_token}Bot:"
										)
										return self.encode(text=__snake_case			)










from sklearn.metrics import matthews_corrcoef

import datasets


UpperCAmelCase_		:					Dict          =  "\nCompute the Matthews correlation coefficient (MCC)\n\nThe Matthews correlation coefficient is used in machine learning as a\nmeasure of the quality of binary and multiclass classifications. It takes\ninto account true and false positives and negatives and is generally\nregarded as a balanced measure which can be used even if the classes are of\nvery different sizes. The MCC is in essence a correlation coefficient value\nbetween -1 and +1. A coefficient of +1 represents a perfect prediction, 0\nan average random prediction and -1 an inverse prediction.  The statistic\nis also known as the phi coefficient. [source: Wikipedia]\n"

UpperCAmelCase_		:					Any          =  "\nArgs:\n    predictions (list of int): Predicted labels, as returned by a model.\n    references (list of int): Ground truth labels.\n    sample_weight (list of int, float, or bool): Sample weights. Defaults to `None`.\nReturns:\n    matthews_correlation (dict containing float): Matthews correlation.\nExamples:\n    Example 1, a basic example with only predictions and references as inputs:\n        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")\n        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],\n        ...                                     predictions=[1, 2, 2, 0, 3, 3])\n        >>> print(round(results['matthews_correlation'], 2))\n        0.54\n\n    Example 2, the same example as above, but also including sample weights:\n        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")\n        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],\n        ...                                     predictions=[1, 2, 2, 0, 3, 3],\n        ...                                     sample_weight=[0.5, 3, 1, 1, 1, 2])\n        >>> print(round(results['matthews_correlation'], 2))\n        0.1\n\n    Example 3, the same example as above, but with sample weights that cause a negative correlation:\n        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")\n        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],\n        ...                                     predictions=[1, 2, 2, 0, 3, 3],\n        ...                                     sample_weight=[0.5, 1, 0, 0, 0, 1])\n        >>> print(round(results['matthews_correlation'], 2))\n        -0.25\n"

UpperCAmelCase_		:					Dict          =  "\\n@article{scikit-learn,\n  title={Scikit-learn: Machine Learning in {P}ython},\n  author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.\n         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.\n         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and\n         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},\n  journal={Journal of Machine Learning Research},\n  volume={12},\n  pages={2825--2830},\n  year={2011}\n}\n"




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION       ,    _KWARGS_DESCRIPTION      )
class 		__A   (      datasets.Metric      ):

						def      A__    (		self :List[str]			):
										'''simple docstring'''

										return datasets.MetricInfo(
										    description=_DESCRIPTION     ,			citation=_CITATION     ,			inputs_description=_KWARGS_DESCRIPTION     ,			features=datasets.Features(
										        {
										            """predictions""": datasets.Value("""int32"""			),
										            """references""": datasets.Value("""int32"""			),
										        }			)     ,			reference_urls=[
										        """https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html"""
										    ]     ,			)


						def      A__    (		self :Tuple     ,			__snake_case :str     ,			__snake_case :Tuple     ,			__snake_case :List[str]=None			):
										'''simple docstring'''

										return {
										    "matthews_correlation": float(matthews_corrcoef(__snake_case     ,			__snake_case     ,			sample_weight=__snake_case			)			),
										}




