

from scipy.stats import spearmanr

import datasets


_a:    Optional[Any] 							=			'''
The Spearman rank-order correlation coefficient is a measure of the
relationship between two datasets. Like other correlation coefficients,
this one varies between -1 and +1 with 0 implying no correlation.
Positive correlations imply that as data in dataset x increases, so
does data in dataset y. Negative correlations imply that as x increases,
y decreases. Correlations of -1 or +1 imply an exact monotonic relationship.

Unlike the Pearson correlation, the Spearman correlation does not
assume that both datasets are normally distributed.

The p-value roughly indicates the probability of an uncorrelated system
producing datasets that have a Spearman correlation at least as extreme
as the one computed from these datasets. The p-values are not entirely
reliable but are probably reasonable for datasets larger than 500 or so.
'''

_a:    int 							=			'''
Args:
    predictions (`List[float]`): Predicted labels, as returned by a model.
    references (`List[float]`): Ground truth labels.
    return_pvalue (`bool`): If `True`, returns the p-value. If `False`, returns
            only the spearmanr score. Defaults to `False`.
Returns:
    spearmanr (`float`): Spearman correlation coefficient.
    p-value (`float`): p-value. **Note**: is only returned if `return_pvalue=True` is input.
Examples:
    Example 1:
        >>> spearmanr_metric = datasets.load_metric("spearmanr")
        >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4])
        >>> print(results)
        {\'spearmanr\': -0.7}

    Example 2:
        >>> spearmanr_metric = datasets.load_metric("spearmanr")
        >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5],
        ...                                     predictions=[10, 9, 2.5, 6, 4],
        ...                                     return_pvalue=True)
        >>> print(results[\'spearmanr\'])
        -0.7
        >>> print(round(results[\'spearmanr_pvalue\'], 2))
        0.19
'''

_a:    List[Any] 							=			R'''\
@book{kokoska2000crc,
  title={CRC standard probability and statistics tables and formulae},
  author={Kokoska, Stephen and Zwillinger, Daniel},
  year={2000},
  publisher={Crc Press}
}
@article{2020SciPy-NMeth,
  author  = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and
            Haberland, Matt and Reddy, Tyler and Cournapeau, David and
            Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and
            Bright, Jonathan and {van der Walt}, St{\'e}fan J. and
            Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and
            Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and
            Kern, Robert and Larson, Eric and Carey, C J and
            Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and
            {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and
            Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and
            Harris, Charles R. and Archibald, Anne M. and
            Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and
            {van Mulbregt}, Paul and {SciPy 1.0 Contributors}},
  title   = {{{SciPy} 1.0: Fundamental Algorithms for Scientific
            Computing in Python}},
  journal = {Nature Methods},
  year    = {2020},
  volume  = {17},
  pages   = {261--272},
  adsurl  = {https://rdcu.be/b08Wh},
  doi     = {10.1038/s41592-019-0686-2},
}
'''

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,     _KWARGS_DESCRIPTION						)
class 						__UpperCamelCase    (					datasets.Metric						):




	def       __A					(		self	:						str							):


		'''simple docstring'''







		return datasets.MetricInfo(
		    description=_DESCRIPTION		,	citation=_CITATION		,	inputs_description=_KWARGS_DESCRIPTION		,	features=datasets.Features(
		        {
		            "predictions": datasets.Value("float"							),
		            "references": datasets.Value("float"							),
		        }							)		,	reference_urls=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.spearmanr.html"]		,	)




	def       __A					(		self	:						Dict		,	lowerCAmelCase	:						Dict		,	lowerCAmelCase	:						Tuple		,	lowerCAmelCase	:						Dict=False							):


		'''simple docstring'''







		UpperCAmelCase_       				=			spearmanr(_lowercase		,	_lowercase							)
		if return_pvalue:
			return {"spearmanr": results[0], "spearmanr_pvalue": results[1]}
		else:
			return {"spearmanr": results[0]}


import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_a:    List[Any] 							=			logging.get_logger(__name__)

_a:    List[str] 							=			"""▁"""

_a:    Union[str, Any] 							=			{
    """vocab_file""": """vocab.json""",
    """spm_file""": """sentencepiece.bpe.model""",
}

_a:    Tuple 							=			{
    """vocab_file""": {
        """facebook/s2t-small-librispeech-asr""": (
            """https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json"""
        ),
    },
    """spm_file""": {
        """facebook/s2t-small-librispeech-asr""": (
            """https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model"""
        )
    },
}

_a:    Optional[int] 							=			{
    """facebook/s2t-small-librispeech-asr""": 1024,
}

_a:    List[Any] 							=			["""pt""", """fr""", """ru""", """nl""", """ro""", """it""", """es""", """de"""]

_a:    Tuple 							=			{"""mustc""": MUSTC_LANGS}

class 						__UpperCamelCase    (					lowercase						):
	SCREAMING_SNAKE_CASE__    				=     VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE__    				=     PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE__    				=     MAX_MODEL_INPUT_SIZES
	SCREAMING_SNAKE_CASE__    				=     ['input_ids', 'attention_mask']

	SCREAMING_SNAKE_CASE__    				=     []




	def __init__(		self	:						Union[str, Any]		,	lowerCAmelCase	:						str		,	lowerCAmelCase	:						Tuple		,	lowerCAmelCase	:						Tuple="<s>"		,	lowerCAmelCase	:						Optional[int]="</s>"		,	lowerCAmelCase	:						Any="<pad>"		,	lowerCAmelCase	:						Union[str, Any]="<unk>"		,	lowerCAmelCase	:						Tuple=False		,	lowerCAmelCase	:						List[Any]=False		,	lowerCAmelCase	:						Optional[int]=None		,	lowerCAmelCase	:						List[str]=None		,	lowerCAmelCase	:						Optional[Dict[str, Any]] = None		,	**lowerCAmelCase	:						Any		,	):


		'''simple docstring'''







		UpperCAmelCase_       				=			{} if sp_model_kwargs is None else sp_model_kwargs

		super().__init__(
		    bos_token=lowerCAmelCase		,	eos_token=lowerCAmelCase		,	unk_token=lowerCAmelCase		,	pad_token=lowerCAmelCase		,	do_upper_case=lowerCAmelCase		,	do_lower_case=lowerCAmelCase		,	tgt_lang=lowerCAmelCase		,	lang_codes=lowerCAmelCase		,	sp_model_kwargs=self.sp_model_kwargs		,	**lowerCAmelCase		,	)
		UpperCAmelCase_       				=			do_upper_case
		UpperCAmelCase_       				=			do_lower_case

		UpperCAmelCase_       				=			load_json(lowerCAmelCase							)
		UpperCAmelCase_       				=			{v: k for k, v in self.encoder.items()}
		UpperCAmelCase_       				=			spm_file
		UpperCAmelCase_       				=			load_spm(lowerCAmelCase		,	self.sp_model_kwargs							)

		if lang_codes is not None:
			UpperCAmelCase_       				=			lang_codes
			UpperCAmelCase_       				=			LANGUAGES[lang_codes]
			UpperCAmelCase_       				=			[F"<lang:{lang}>" for lang in self.langs]
			UpperCAmelCase_       				=			{lang: self.sp_model.PieceToId(F"<lang:{lang}>"							) for lang in self.langs}

			UpperCAmelCase_       				=			self.lang_tokens
			UpperCAmelCase_       				=			tgt_lang if tgt_lang is not None else self.langs[0]

			self.set_tgt_lang_special_tokens(self._tgt_lang							)
		else:
			UpperCAmelCase_       				=			{}




	@property
	def       __A					(		self	:						List[Any]							):


		'''simple docstring'''







		return len(self.encoder							)




	@property
	def       __A					(		self	:						Any							):


		'''simple docstring'''







		return self._tgt_lang




	@tgt_lang.setter
	def       __A					(		self	:						int		,	lowerCAmelCase	:						Optional[Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			new_tgt_lang
		self.set_tgt_lang_special_tokens(lowerCAmelCase							)




	def       __A					(		self	:						List[Any]		,	lowerCAmelCase	:						str							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.lang_code_to_id[tgt_lang]
		UpperCAmelCase_       				=			[lang_code_id]




	def       __A					(		self	:						List[str]		,	lowerCAmelCase	:						str							):


		'''simple docstring'''







		return self.sp_model.encode(lowerCAmelCase		,	out_type=lowerCAmelCase							)




	def       __A					(		self	:						str		,	lowerCAmelCase	:						int							):


		'''simple docstring'''







		return self.encoder.get(lowerCAmelCase		,	self.encoder[self.unk_token]							)




	def       __A					(		self	:						Optional[Any]		,	lowerCAmelCase	:						int							):


		'''simple docstring'''







		return self.decoder.get(lowerCAmelCase		,	self.unk_token							)




	def       __A					(		self	:						Dict		,	lowerCAmelCase	:						List[str]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			[]
		UpperCAmelCase_       				=			""
		for token in tokens:
			# make sure that special tokens are not decoded using sentencepiece model
			if token in self.all_special_tokens:
				UpperCAmelCase_       				=			self.sp_model.decode(lowerCAmelCase							)
				out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
				UpperCAmelCase_       				=			[]
			else:
				current_sub_tokens.append(lowerCAmelCase							)
		UpperCAmelCase_       				=			self.sp_model.decode(lowerCAmelCase							)
		out_string += decoded.upper() if self.do_upper_case else decoded
		return out_string.strip()




	def       __A					(		self	:						Optional[Any]		,	lowerCAmelCase	:						List[Any]		,	lowerCAmelCase	:						Optional[int]=None							):


		'''simple docstring'''







		if token_ids_a is None:
			return self.prefix_tokens + token_ids_a + [self.eos_token_id]
		# We don't expect to process pairs, but leave the pair logic for API consistency
		return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]




	def       __A					(		self	:						Any		,	lowerCAmelCase	:						List[int]		,	lowerCAmelCase	:						Optional[List[int]] = None		,	lowerCAmelCase	:						bool = False							):


		'''simple docstring'''







		if already_has_special_tokens:
			return super().get_special_tokens_mask(
			    token_ids_a=lowerCAmelCase		,	token_ids_a=lowerCAmelCase		,	already_has_special_tokens=lowerCAmelCase							)

		UpperCAmelCase_       				=			[1] * len(self.prefix_tokens							)
		UpperCAmelCase_       				=			[1]
		if token_ids_a is None:
			return prefix_ones + ([0] * len(lowerCAmelCase							)) + suffix_ones
		return prefix_ones + ([0] * len(lowerCAmelCase							)) + ([0] * len(lowerCAmelCase							)) + suffix_ones




	def       __A					(		self	:						Optional[Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.encoder.copy()
		vocab.update(self.added_tokens_encoder							)
		return vocab




	def __getstate__(		self	:						Union[str, Any]							):


		'''simple docstring'''







		UpperCAmelCase_       				=			self.__dict__.copy()
		UpperCAmelCase_       				=			None
		return state




	def __setstate__(		self	:						Tuple		,	lowerCAmelCase	:						Dict							):


		'''simple docstring'''







		UpperCAmelCase_       				=			d

		# for backward compatibility
		if not hasattr(self		,	"sp_model_kwargs"							):
			UpperCAmelCase_       				=			{}

		UpperCAmelCase_       				=			load_spm(self.spm_file		,	self.sp_model_kwargs							)




	def       __A					(		self	:						Optional[Any]		,	lowerCAmelCase	:						str		,	lowerCAmelCase	:						Optional[str] = None							):


		'''simple docstring'''







		UpperCAmelCase_       				=			Path(lowerCAmelCase							)
		assert save_dir.is_dir(), F"{save_directory} should be a directory"
		UpperCAmelCase_       				=			save_dir / (
		    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["vocab_file"]
		)
		UpperCAmelCase_       				=			save_dir / (
		    (filename_prefix + "-" if filename_prefix else "") + self.vocab_files_names["spm_file"]
		)

		save_json(self.encoder		,	lowerCAmelCase							)

		if os.path.abspath(self.spm_file							) != os.path.abspath(lowerCAmelCase							) and os.path.isfile(self.spm_file							):
			copyfile(self.spm_file		,	lowerCAmelCase							)
		elif not os.path.isfile(self.spm_file							):
			with open(lowerCAmelCase		,	"wb"							) as fi:
				UpperCAmelCase_       				=			self.sp_model.serialized_model_proto()
				fi.write(lowerCAmelCase							)

		return (str(lowerCAmelCase							), str(lowerCAmelCase							))



def       __lowerCAmelCase      (						A       ,     A    ):
	UpperCAmelCase_       				=			sentencepiece.SentencePieceProcessor(**A    )
	spm.Load(str(A    )    )
	return spm



def       __lowerCAmelCase      (						A    ):
	with open(A       ,     "r"    ) as f:
		return json.load(A    )




def       __lowerCAmelCase      (						A       ,     A    ):
	with open(A       ,     "w"    ) as f:
		json.dump(A       ,     A       ,     indent=2    )