




import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_		:					Dict          =  logging.get_logger(__name__)

UpperCAmelCase_		:					List[Any]          =  {
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

UpperCAmelCase_		:					str          =  {
    "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
    "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
}

UpperCAmelCase_		:					Union[str, Any]          =  {
    "ctrl": 256,
}

UpperCAmelCase_		:					Optional[Any]          =  {
    "Pregnancy": 168629,
    "Christianity": 7675,
    "Explain": 106423,
    "Fitness": 63440,
    "Saving": 63163,
    "Ask": 27171,
    "Ass": 95985,
    "Joke": 163509,
    "Questions": 45622,
    "Thoughts": 49605,
    "Retail": 52342,
    "Feminism": 164338,
    "Writing": 11992,
    "Atheism": 192263,
    "Netflix": 48616,
    "Computing": 39639,
    "Opinion": 43213,
    "Alone": 44967,
    "Funny": 58917,
    "Gaming": 40358,
    "Human": 4088,
    "India": 1331,
    "Joker": 77138,
    "Diet": 36206,
    "Legal": 11859,
    "Norman": 4939,
    "Tip": 72689,
    "Weight": 52343,
    "Movies": 46273,
    "Running": 23425,
    "Science": 2090,
    "Horror": 37793,
    "Confession": 60572,
    "Finance": 12250,
    "Politics": 16360,
    "Scary": 191985,
    "Support": 12654,
    "Technologies": 32516,
    "Teenage": 66160,
    "Event": 32769,
    "Learned": 67460,
    "Notion": 182770,
    "Wikipedia": 37583,
    "Books": 6665,
    "Extract": 76050,
    "Confessions": 102701,
    "Conspiracy": 75932,
    "Links": 63674,
    "Narcissus": 150425,
    "Relationship": 54766,
    "Relationships": 134796,
    "Reviews": 41671,
    "News": 4256,
    "Translation": 26820,
    "multilingual": 128406,
}




def 		lowerCAmelCase_ (						lowerCamelCase ):
				__magic_name__						:						Tuple										=set()
				__magic_name__						:						Dict										=word[0]
				for char in word[1:]:
								pairs.add((prev_char, char) )
								__magic_name__						:						List[str]										=char

				__magic_name__						:						List[Any]										=set(lowerCamelCase )
				return pairs




class 		__A   (      UpperCamelCase__      ):
						UpperCamelCase      =       VOCAB_FILES_NAMES
						UpperCamelCase      =       PRETRAINED_VOCAB_FILES_MAP
						UpperCamelCase      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCamelCase      =       CONTROL_CODES

						def __init__(		self :List[Any]     ,			__snake_case :Dict     ,			__snake_case :int     ,			__snake_case :Optional[int]="<unk>"     ,			**__snake_case :int			):
										'''simple docstring'''

										super().__init__(unk_token=__snake_case     ,			**__snake_case			)

										with open(__snake_case     ,			encoding="""utf-8"""			) as vocab_handle:
														__magic_name__						:						List[str]										=json.load(__snake_case			)
										__magic_name__						:						Any										={v: k for k, v in self.encoder.items()}
										with open(__snake_case     ,			encoding="""utf-8"""			) as merges_handle:
														__magic_name__						:						Optional[int]										=merges_handle.read().split("""\n"""			)[1:-1]
										__magic_name__						:						Tuple										=[tuple(merge.split()			) for merge in merges]
										__magic_name__						:						Union[str, Any]										=dict(zip(__snake_case     ,			range(len(__snake_case			)			)			)			)
										__magic_name__						:						str										={}

						@property
						def      A__    (		self :Dict			):
										'''simple docstring'''

										return len(self.encoder			)

						def      A__    (		self :Optional[int]			):
										'''simple docstring'''

										return dict(self.encoder     ,			**self.added_tokens_encoder			)

						def      A__    (		self :List[str]     ,			__snake_case :Tuple			):
										'''simple docstring'''

										if token in self.cache:
														return self.cache[token]
										__magic_name__						:						Any										=tuple(__snake_case			)
										__magic_name__						:						List[str]										=tuple(list(word[:-1]			) + [word[-1] + """</w>"""]			)
										__magic_name__						:						Tuple										=get_pairs(__snake_case			)

										if not pairs:
														return token

										while True:
														__magic_name__						:						List[Any]										=min(__snake_case     ,			key=lambda __snake_case			: self.bpe_ranks.get(__snake_case     ,			float("""inf"""			)			)			)
														if bigram not in self.bpe_ranks:
																		break
														__magic_name__					,			__magic_name__						:						List[Any]										=bigram
														__magic_name__						:						Any										=[]
														__magic_name__						:						Union[str, Any]										=0
														while i < len(__snake_case			):
																		try:
																						__magic_name__						:						Union[str, Any]										=word.index(__snake_case     ,			__snake_case			)
																		except ValueError:
																						new_word.extend(word[i:]			)
																						break
																		else:
																						new_word.extend(word[i:j]			)
																						__magic_name__						:						int										=j

																		if word[i] == first and i < len(__snake_case			) - 1 and word[i + 1] == second:
																						new_word.append(first + second			)
																						i += 2
																		else:
																						new_word.append(word[i]			)
																						i += 1
														__magic_name__						:						Tuple										=tuple(__snake_case			)
														__magic_name__						:						List[str]										=new_word
														if len(__snake_case			) == 1:
																		break
														else:
																		__magic_name__						:						Any										=get_pairs(__snake_case			)
										__magic_name__						:						Any										="""@@ """.join(__snake_case			)
										__magic_name__						:						Optional[Any]										=word[:-4]
										__magic_name__						:						Union[str, Any]										=word
										return word

						def      A__    (		self :Optional[int]     ,			__snake_case :Dict			):
										'''simple docstring'''

										__magic_name__						:						str										=[]

										__magic_name__						:						Union[str, Any]										=re.findall(r"""\S+\n?"""     ,			__snake_case			)

										for token in words:
														split_tokens.extend(list(self.bpe(__snake_case			).split(""" """			)			)			)
										return split_tokens

						def      A__    (		self :Tuple     ,			__snake_case :str			):
										'''simple docstring'''

										return self.encoder.get(__snake_case     ,			self.encoder.get(self.unk_token			)			)

						def      A__    (		self :Optional[int]     ,			__snake_case :Optional[Any]			):
										'''simple docstring'''

										return self.decoder.get(__snake_case     ,			self.unk_token			)

						def      A__    (		self :Dict     ,			__snake_case :List[str]			):
										'''simple docstring'''

										__magic_name__						:						List[str]										=""" """.join(__snake_case			).replace("""@@ """     ,			""""""			).strip()
										return out_string


						def      A__    (		self :Any     ,			__snake_case :str     ,			__snake_case :Optional[str] = None			):
										'''simple docstring'''

										if not os.path.isdir(__snake_case			):
														logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
														return
										__magic_name__						:						str										=os.path.join(
										    __snake_case     ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)
										__magic_name__						:						int										=os.path.join(
										    __snake_case     ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]			)

										with open(__snake_case     ,			"""w"""     ,			encoding="""utf-8"""			) as f:
														f.write(json.dumps(self.encoder     ,			indent=2     ,			sort_keys=__snake_case     ,			ensure_ascii=__snake_case			) + """\n"""			)

										__magic_name__						:						int										=0
										with open(__snake_case     ,			"""w"""     ,			encoding="""utf-8"""			) as writer:
														writer.write("""#version: 0.2\n"""			)
														for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,			key=lambda __snake_case			: kv[1]			):
																		if index != token_index:
																						logger.warning(
																						    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																						    """ Please check that the tokenizer is not corrupted!"""			)
																						__magic_name__						:						List[str]										=token_index
																		writer.write(""" """.join(__snake_case			) + """\n"""			)
																		index += 1

										return vocab_file, merge_file

						# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
						#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
						#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
						#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
						#     return ''.join(tokens_generated_so_far)










from sklearn.metrics import matthews_corrcoef

import datasets


UpperCAmelCase_		:					Dict          =  "\nCompute the Matthews correlation coefficient (MCC)\n\nThe Matthews correlation coefficient is used in machine learning as a\nmeasure of the quality of binary and multiclass classifications. It takes\ninto account true and false positives and negatives and is generally\nregarded as a balanced measure which can be used even if the classes are of\nvery different sizes. The MCC is in essence a correlation coefficient value\nbetween -1 and +1. A coefficient of +1 represents a perfect prediction, 0\nan average random prediction and -1 an inverse prediction.  The statistic\nis also known as the phi coefficient. [source: Wikipedia]\n"

UpperCAmelCase_		:					Any          =  "\nArgs:\n    predictions (list of int): Predicted labels, as returned by a model.\n    references (list of int): Ground truth labels.\n    sample_weight (list of int, float, or bool): Sample weights. Defaults to `None`.\nReturns:\n    matthews_correlation (dict containing float): Matthews correlation.\nExamples:\n    Example 1, a basic example with only predictions and references as inputs:\n        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")\n        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],\n        ...                                     predictions=[1, 2, 2, 0, 3, 3])\n        >>> print(round(results['matthews_correlation'], 2))\n        0.54\n\n    Example 2, the same example as above, but also including sample weights:\n        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")\n        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],\n        ...                                     predictions=[1, 2, 2, 0, 3, 3],\n        ...                                     sample_weight=[0.5, 3, 1, 1, 1, 2])\n        >>> print(round(results['matthews_correlation'], 2))\n        0.1\n\n    Example 3, the same example as above, but with sample weights that cause a negative correlation:\n        >>> matthews_metric = datasets.load_metric(\"matthews_correlation\")\n        >>> results = matthews_metric.compute(references=[1, 3, 2, 0, 3, 2],\n        ...                                     predictions=[1, 2, 2, 0, 3, 3],\n        ...                                     sample_weight=[0.5, 1, 0, 0, 0, 1])\n        >>> print(round(results['matthews_correlation'], 2))\n        -0.25\n"

UpperCAmelCase_		:					Dict          =  "\\n@article{scikit-learn,\n  title={Scikit-learn: Machine Learning in {P}ython},\n  author={Pedregosa, F. and Varoquaux, G. and Gramfort, A. and Michel, V.\n         and Thirion, B. and Grisel, O. and Blondel, M. and Prettenhofer, P.\n         and Weiss, R. and Dubourg, V. and Vanderplas, J. and Passos, A. and\n         Cournapeau, D. and Brucher, M. and Perrot, M. and Duchesnay, E.},\n  journal={Journal of Machine Learning Research},\n  volume={12},\n  pages={2825--2830},\n  year={2011}\n}\n"




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION       ,    _KWARGS_DESCRIPTION      )
class 		__A   (      datasets.Metric      ):

						def      A__    (		self :List[str]			):
										'''simple docstring'''

										return datasets.MetricInfo(
										    description=_DESCRIPTION     ,			citation=_CITATION     ,			inputs_description=_KWARGS_DESCRIPTION     ,			features=datasets.Features(
										        {
										            """predictions""": datasets.Value("""int32"""			),
										            """references""": datasets.Value("""int32"""			),
										        }			)     ,			reference_urls=[
										        """https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html"""
										    ]     ,			)


						def      A__    (		self :Tuple     ,			__snake_case :str     ,			__snake_case :Tuple     ,			__snake_case :List[str]=None			):
										'''simple docstring'''

										return {
										    "matthews_correlation": float(matthews_corrcoef(__snake_case     ,			__snake_case     ,			sample_weight=__snake_case			)			),
										}




