




import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device


if is_torch_available():
 import torch

 from transformers import XLMProphetNetForConditionalGeneration, XLMProphetNetTokenizer



@require_torch
class 			a     (   unittest.TestCase			):

     '''simple docstring'''





     @slow
     def   lowerCamelCase_   (	self   :			Any				):
         UpperCAmelCase_							 =     XLMProphetNetForConditionalGeneration.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased'''				)
         model.to(__snake_case				)

         # encoder-decoder outputs
         UpperCAmelCase_							 =     torch.tensor([[17, 9_62_08, 10_34_71, 2]]				).to(__snake_case				)
         UpperCAmelCase_							 =     torch.tensor(
             [[2, 2_50, 99_53, 34, 6_94_89, 16_20, 32, 11_84_24, 6_24, 2_10, 1_05, 29_13, 10_32, 3_51]]				).to(__snake_case				)
         UpperCAmelCase_							 =     model(
             input_ids=__snake_case					,    attention_mask=__snake_case					,    encoder_outputs=__snake_case					,    decoder_input_ids=__snake_case				)
         UpperCAmelCase_							 =     output[0]
         UpperCAmelCase_							 =     torch.Size((1, 14, 25_00_12)				)
         self.assertEqual(output_predited_logis.shape					,    __snake_case				)
         UpperCAmelCase_							 =     torch.tensor(
             [[[-6.3_986, -8.2_391, 12.5_189], [-6.3_289, -8.0_864, 12.6_211], [-6.2_418, -8.0_445, 12.7_968]]]				).to(__snake_case				)
         self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]					,    __snake_case					,    atol=1E-4				)				)

         # encoder outputs
         UpperCAmelCase_							 =     model.prophetnet.encoder(__snake_case				)[0]
         UpperCAmelCase_							 =     torch.tensor(
             [[[-1.4_260, -0.7_628, 0.8_453], [-1.4_719, -0.1_391, 0.7_807], [-1.7_678, 0.0_114, 0.4_646]]]				).to(__snake_case				)
         UpperCAmelCase_							 =     torch.Size((1, 4, 10_24)				)
         self.assertEqual(encoder_outputs.shape					,    __snake_case				)
         self.assertTrue(torch.allclose(encoder_outputs[:, :3, :3]					,    __snake_case					,    atol=1E-4				)				)

         # decoder outputs
         UpperCAmelCase_							 =     model.prophetnet.decoder(
             __snake_case					,    encoder_hidden_states=__snake_case					,    )
         UpperCAmelCase_							 =     decoder_outputs[1].view(1					,    model.config.ngram					,    14					,    -1				)
         UpperCAmelCase_							 =     model.lm_head(__snake_case				)
         UpperCAmelCase_							 =     predicting_streams_logits[:, 0]
         self.assertTrue(torch.allclose(next_first_stream_logits[:, :3, :3]					,    __snake_case					,    atol=1E-4				)				)
     @slow
     def   lowerCamelCase_   (	self   :			str				):
         UpperCAmelCase_							 =     XLMProphetNetForConditionalGeneration.from_pretrained(
             '''microsoft/xprophetnet-large-wiki100-cased-xglue-ntg'''				)
         model.to(__snake_case				)

         UpperCAmelCase_							 =     torch.tensor([[17, 9_62_08, 10_34_71, 2]]				).to(__snake_case				)
         UpperCAmelCase_							 =     torch.tensor(
             [[2, 2_50, 99_53, 34, 6_94_89, 16_20, 32, 11_84_24, 6_24, 2_10, 1_05, 29_13, 10_32, 3_51]]				).to(__snake_case				)
         UpperCAmelCase_							 =     model(
             input_ids=__snake_case					,    attention_mask=__snake_case					,    encoder_outputs=__snake_case					,    decoder_input_ids=__snake_case				)
         UpperCAmelCase_							 =     output[0]
         UpperCAmelCase_							 =     torch.Size((1, 14, 25_00_12)				)
         self.assertEqual(output_predited_logis.shape					,    __snake_case				)
         # compare the actual values for a slice.
         UpperCAmelCase_							 =     torch.tensor(
             [[[-9.2_253, -9.7_173, -6.3_529], [-7.6_701, -9.0_145, -1.9_382], [-8.0_195, -7.0_004, -0.1_523]]]				).to(__snake_case				)

         self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]					,    __snake_case					,    atol=1E-4				)				)




     @slow
     def   lowerCamelCase_   (	self   :			List[str]				):
         UpperCAmelCase_							 =     XLMProphetNetForConditionalGeneration.from_pretrained(
             '''microsoft/xprophetnet-large-wiki100-cased-xglue-ntg'''				)
         model.to(__snake_case				)
         UpperCAmelCase_							 =     5_12

         UpperCAmelCase_							 =     XLMProphetNetTokenizer.from_pretrained('''microsoft/xprophetnet-large-wiki100-cased-xglue-ntg'''				)

         UpperCAmelCase_							 =     (
             '''Microsoft Corporation intends to officially end free support for the Windows 7 operating system after'''
             ''' January 14, 2020, according to the official portal of the organization. From that day, users of this'''
             ''' system will not be able to receive security updates, which could make their computers vulnerable to'''
             ''' cyber attacks.'''
         )
         UpperCAmelCase_							 =     (
             '''орпорация Microsoft намерена официально прекратить бесплатную поддержку операционной системы Windows 7'''
             ''' после 14 января 2020 года, сообщается на официальном портале организации . С указанного дня пользователи'''
             ''' этой системы не смогут получать обновления безопасности, из-за чего их компьютеры могут стать уязвимыми'''
             ''' к кибератакам.'''
         )
         UpperCAmelCase_							 =     (
             '''根据该组织的官方门户网站，微软公司打算在2020年1月14日之后正式终止对Windows 7操作系统的免费支持。从那时起，该系统的用户将无法接收安全更新，这可能会使他们的计算机容易受到网络攻击。'''
         )

         UpperCAmelCase_							 =     tokenizer(
             [EN_SENTENCE, RU_SENTENCE, ZH_SENTENCE]					,    padding=__snake_case					,    max_length=2_55					,    return_tensors='''pt'''				).input_ids
         UpperCAmelCase_							 =     input_ids.to(__snake_case				)

         UpperCAmelCase_							 =     model.generate(
             __snake_case					,    num_beams=10					,    length_penalty=1.0					,    no_repeat_ngram_size=3					,    early_stopping=__snake_case				)
         UpperCAmelCase_							 =     [tokenizer.decode(__snake_case					,    skip_special_tokens=__snake_case				) for g in summary_ids]
         UpperCAmelCase_							 =     '''Microsoft to end Windows 7 free support after January 14, 2020'''
         UpperCAmelCase_							 =     '''Microsoft намерена прекратить бесплатную поддержку Windows 7 после 14 января 2020 года'''
         UpperCAmelCase_							 =     '''微软打算终止对Windows 7操作系统的免费支持'''
         self.assertListEqual(
             [EXPECTED_TITLE_EN, EXPECTED_TITLE_RU, EXPECTED_TITLE_ZH]					,    __snake_case					,    )

         UpperCAmelCase_							 =     model.generate(
             __snake_case					,    num_beams=1					,    length_penalty=1.0					,    no_repeat_ngram_size=3					,    early_stopping=__snake_case				)
         UpperCAmelCase_							 =     [
             tokenizer.convert_ids_to_tokens(__snake_case					,    skip_special_tokens=__snake_case				) for g in summary_ids_beama
         ]
         UpperCAmelCase_							 =     '''▁Microsoft ▁to ▁end ▁free ▁support ▁for ▁Windows ▁7'''.split(''' '''				)
         UpperCAmelCase_							 =     '''▁Microsoft ▁намерен а ▁прекрати ть ▁бес плат ную ▁поддержку ▁Windows ▁7 ▁после ▁14 ▁января ▁2020 ▁года'''.split(
             ''' '''				)
         UpperCAmelCase_							 =     '''微软 公司 打算 终止 对 Windows ▁7 操作 系统的 免费 支持'''.split(''' '''				)
         self.assertListEqual(
             [EXPECTED_TITLE_EN_BEAM1_TOK, EXPECTED_TITLE_RU_BEAM1_TOK, EXPECTED_TITLE_ZH_BEAM1_TOK]					,    __snake_case					,    )













from typing import List

from packaging import version
from sklearn.metrics import fa_score

import datasets
from datasets.config import PY_VERSION


if PY_VERSION < version.parse("""3.8"""):
   import importlib_metadata
else:
   import importlib.metadata as importlib_metadata


# TODO(Patrick/Anton)
snake_case			 =						"""\
"""

snake_case			 =						"""\
    XTREME-S is a benchmark to evaluate universal cross-lingual speech representations in many languages.
    XTREME-S covers four task families: speech recognition, classification, speech-to-text translation and retrieval.
"""

snake_case			 =						"""
Compute XTREME-S evaluation metric associated to each XTREME-S dataset.
Args:
    predictions: list of predictions to score.
        Each translation should be tokenized into a list of tokens.
    references: list of lists of references for each translation.
        Each reference should be tokenized into a list of tokens.
    bleu_kwargs: optional dict of keywords to be passed when computing 'bleu'.
        Keywords include Dict can be one of 'smooth_method', 'smooth_value', 'force', 'lowercase',
        'tokenize', 'use_effective_order'.
    wer_kwargs: optional dict of keywords to be passed when computing 'wer' and 'cer'.
        Keywords include 'concatenate_texts'.
Returns: depending on the XTREME-S task, one or several of:
    \"accuracy\": Accuracy - for 'fleurs-lang_id', 'minds14'
    \"f1\": F1 score - for 'minds14'
    \"wer\": Word error rate - for 'mls', 'fleurs-asr', 'voxpopuli', 'babel'
    \"cer\": Character error rate - for 'mls', 'fleurs-asr', 'voxpopuli', 'babel'
    \"bleu\": BLEU score according to the `sacrebleu` metric - for 'covost2'
Examples:

    >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'mls')  # 'mls', 'voxpopuli', 'fleurs-asr' or 'babel'
    >>> references = [\"it is sunny here\", \"paper and pen are essentials\"]
    >>> predictions = [\"it's sunny\", \"paper pen are essential\"]
    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)
    >>> print({k: round(v, 2) for k, v in results.items()})
    {'wer': 0.56, 'cer': 0.27}

    >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'covost2')
    >>> references = [\"bonjour paris\", \"il est necessaire de faire du sport de temps en temp\"]
    >>> predictions = [\"bonjour paris\", \"il est important de faire du sport souvent\"]
    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)
    >>> print({k: round(v, 2) for k, v in results.items()})
    {'bleu': 31.65}

    >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'fleurs-lang_id')
    >>> references = [0, 1, 0, 0, 1]
    >>> predictions = [0, 1, 1, 0, 0]
    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)
    >>> print({k: round(v, 2) for k, v in results.items()})
    {'accuracy': 0.6}

    >>> xtreme_s_metric = datasets.load_metric('xtreme_s', 'minds14')
    >>> references = [0, 1, 0, 0, 1]
    >>> predictions = [0, 1, 1, 0, 0]
    >>> results = xtreme_s_metric.compute(predictions=predictions, references=references)
    >>> print({k: round(v, 2) for k, v in results.items()})
    {'f1': 0.58, 'accuracy': 0.6}
"""

snake_case			 =						["""fleurs-asr""", """mls""", """voxpopuli""", """babel""", """covost2""", """fleurs-lang_id""", """minds14"""]
snake_case			 =						""""""

try:
   from jiwer import transforms as tr

   snake_case			 =						True
except ImportError:
   snake_case			 =						False

if _jiwer_available and version.parse(importlib_metadata.version("""jiwer""")) < version.parse("""2.3.0"""):


   class        SCREAMING_SNAKE_CASE	(	tr.AbstractTransform    ):
        '''simple docstring'''


        def __init__( self				:		Dict    ,       UpperCAmelCase_				:		str = " "					):
          SCREAMING_SNAKE_CASE				:					List[Any]		  =			sentence_delimiter


        def  _A				( self				:		Optional[int]    ,       UpperCAmelCase_				:		str					):
          return list(UpperCAmelCase_					)






        def  _A				( self				:		str    ,       UpperCAmelCase_				:		List[str]					):
          SCREAMING_SNAKE_CASE				:					Tuple		  =			[]
          for sent_idx, sentence in enumerate(UpperCAmelCase_					):
            chars.extend(self.process_string(UpperCAmelCase_					)					)
            if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(UpperCAmelCase_					) - 1:
              chars.append(self.sentence_delimiter					)
          return chars





   snake_case			 =						tr.Compose(
       [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)]
   )
elif _jiwer_available:
   snake_case			 =						tr.Compose(
       [
           tr.RemoveMultipleSpaces(),
           tr.Strip(),
           tr.ReduceToSingleSentence(SENTENCE_DELIMITER),
           tr.ReduceToListOfListOfChars(),
       ]
   )
else:
   snake_case			 =						None


def      lowerCamelCase__	(   lowercase  ,   lowercase ):

  """simple docstring"""






  return float((preds == labels).mean() )


def      lowerCamelCase__	(   lowercase  ,   lowercase ):

  """simple docstring"""






  return {
      "f1": float(fa_score(y_true=lowercase  ,   y_pred=lowercase  ,   average="macro" ) ),
      "accuracy": simple_accuracy(lowercase  ,   lowercase ),
  }


def      lowerCamelCase__	(   lowercase  ,   lowercase  ,   lowercase="exp"  ,   lowercase=None  ,   lowercase=False  ,   lowercase=False  ,   lowercase=None  ,   lowercase=False  ,   ):

  """simple docstring"""






  SCREAMING_SNAKE_CASE				:					str		  =			[[label] for label in labels]
  SCREAMING_SNAKE_CASE				:					Tuple		  =			list(lowercase )
  try:
    import sacrebleu as scb
  except ImportError:
    raise ValueError(
        "sacrebleu has to be installed in order to apply the bleu metric for covost2."
        "You can install it via `pip install sacrebleu`." )

  if version.parse(scb.__version__ ) < version.parse("1.4.12" ):
    raise ImportWarning(
        "To use `sacrebleu`, the module `sacrebleu>=1.4.12` is required, and the current version of `sacrebleu` doesn't match this condition.\n"
        "You can install it with `pip install \"sacrebleu>=1.4.12\"`." )

  SCREAMING_SNAKE_CASE				:					Tuple		  =			len(labels[0] )
  if any(len(lowercase ) != references_per_prediction for refs in labels ):
    raise ValueError("Sacrebleu requires the same number of references for each prediction" )
  SCREAMING_SNAKE_CASE				:					Dict		  =			[[refs[i] for refs in labels] for i in range(lowercase )]
  SCREAMING_SNAKE_CASE				:					List[str]		  =			scb.corpus_bleu(
      lowercase  ,   lowercase  ,   smooth_method=lowercase  ,   smooth_value=lowercase  ,   force=lowercase  ,   lowercase=lowercase  ,   use_effective_order=lowercase  ,   **({"tokenize": tokenize} if tokenize else {})  ,   )
  return {"bleu": output.score}


def      lowerCamelCase__	(   lowercase  ,   lowercase  ,   lowercase  ,   lowercase ):

  """simple docstring"""






  try:
    from jiwer import compute_measures
  except ImportError:
    raise ValueError(
        F'''jiwer has to be installed in order to apply the wer metric for {config_name}.'''
        "You can install it via `pip install jiwer`." )

  if concatenate_texts:
    SCREAMING_SNAKE_CASE				:					Tuple		  =			compute_measures(lowercase  ,   lowercase )["wer"]

    SCREAMING_SNAKE_CASE				:					int		  =			compute_measures(lowercase  ,   lowercase  ,   truth_transform=lowercase  ,   hypothesis_transform=lowercase )["wer"]
    return {"wer": wer, "cer": cer}
  else:

    def compute_score(lowercase  ,   lowercase  ,   lowercase="wer" ):
      SCREAMING_SNAKE_CASE				:					Tuple		  =			0
      SCREAMING_SNAKE_CASE				:					str		  =			0
      for prediction, reference in zip(lowercase  ,   lowercase ):
        if score_type == "wer":
          SCREAMING_SNAKE_CASE				:					int		  =			compute_measures(lowercase  ,   lowercase )
        elif score_type == "cer":
          SCREAMING_SNAKE_CASE				:					int		  =			compute_measures(
              lowercase  ,   lowercase  ,   truth_transform=lowercase  ,   hypothesis_transform=lowercase )
        incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
        total += measures["substitutions"] + measures["deletions"] + measures["hits"]
      return incorrect / total

    return {"wer": compute_score(lowercase  ,   lowercase  ,   "wer" ), "cer": compute_score(lowercase  ,   lowercase  ,   "cer" )}


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION					,						_KWARGS_DESCRIPTION    )
class        SCREAMING_SNAKE_CASE	(	datasets.Metric    ):
     '''simple docstring'''


     def  _A				( self				:		str					):
       if self.config_name not in _CONFIG_NAMES:
         raise KeyError(f'''You should supply a configuration name selected in {_CONFIG_NAMES}'''					)

       SCREAMING_SNAKE_CASE				:					Tuple		  =			"int64" if self.config_name in ["fleurs-lang_id", "minds14"] else "string"

       return datasets.MetricInfo(
           description=_DESCRIPTION    ,       citation=_CITATION    ,       inputs_description=_KWARGS_DESCRIPTION    ,       features=datasets.Features(
               {"predictions": datasets.Value(UpperCAmelCase_					), "references": datasets.Value(UpperCAmelCase_					)}					)    ,       codebase_urls=[]    ,       reference_urls=[]    ,       format="numpy"    ,       )






     def  _A				( self				:		Optional[Any]    ,       UpperCAmelCase_				:		int    ,       UpperCAmelCase_				:		Union[str, Any]    ,       UpperCAmelCase_				:		str=None    ,       UpperCAmelCase_				:		Dict=None					):
       SCREAMING_SNAKE_CASE				:					str		  =			bleu_kwargs if bleu_kwargs is not None else {}
       SCREAMING_SNAKE_CASE				:					Optional[Any]		  =			wer_kwargs if wer_kwargs is not None else {}

       if self.config_name == "fleurs-lang_id":
         return {"accuracy": simple_accuracy(UpperCAmelCase_    ,       UpperCAmelCase_					)}
       elif self.config_name == "minds14":
         return fa_and_simple_accuracy(UpperCAmelCase_    ,       UpperCAmelCase_					)
       elif self.config_name == "covost2":
         SCREAMING_SNAKE_CASE				:					int		  =			bleu_kwargs.pop("smooth_method"    ,       "exp"					)
         SCREAMING_SNAKE_CASE				:					Dict		  =			bleu_kwargs.pop("smooth_value"    ,       UpperCAmelCase_					)
         SCREAMING_SNAKE_CASE				:					List[Any]		  =			bleu_kwargs.pop("force"    ,       UpperCAmelCase_					)
         SCREAMING_SNAKE_CASE				:					Optional[Any]		  =			bleu_kwargs.pop("lowercase"    ,       UpperCAmelCase_					)
         SCREAMING_SNAKE_CASE				:					Optional[Any]		  =			bleu_kwargs.pop("tokenize"    ,       UpperCAmelCase_					)
         SCREAMING_SNAKE_CASE				:					Dict		  =			bleu_kwargs.pop("use_effective_order"    ,       UpperCAmelCase_					)
         return bleu(
             preds=UpperCAmelCase_    ,       labels=UpperCAmelCase_    ,       smooth_method=UpperCAmelCase_    ,       smooth_value=UpperCAmelCase_    ,       force=UpperCAmelCase_    ,       lowercase=UpperCAmelCase_    ,       tokenize=UpperCAmelCase_    ,       use_effective_order=UpperCAmelCase_    ,       )
       elif self.config_name in ["fleurs-asr", "mls", "voxpopuli", "babel"]:
         SCREAMING_SNAKE_CASE				:					Tuple		  =			wer_kwargs.pop("concatenate_texts"    ,       UpperCAmelCase_					)
         return wer_and_cer(UpperCAmelCase_    ,       UpperCAmelCase_    ,       UpperCAmelCase_    ,       self.config_name					)
       else:
         raise KeyError(f'''You should supply a configuration name selected in {_CONFIG_NAMES}'''					)






