'''simple docstring'''






import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, BatchEncoding, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase    :     Dict         =       logging.get_logger(__name__)

_UpperCAmelCase    :     Dict         =       '''▁'''

_UpperCAmelCase    :     Optional[Any]         =       {'''vocab_file''': '''sentencepiece.bpe.model'''}

_UpperCAmelCase    :     Any         =       {
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

_UpperCAmelCase    :     Dict         =       {
    '''facebook/mbart-large-50-one-to-many-mmt''': 10_24,
}

# fmt: off
_UpperCAmelCase    :     Tuple         =       ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']







class lowercase_       (		_UpperCamelCase			):







   """simple docstring"""

   __lowerCAmelCase         =   VOCAB_FILES_NAMES
   __lowerCAmelCase         =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __lowerCAmelCase         =   PRETRAINED_VOCAB_FILES_MAP
   __lowerCAmelCase         =   ["input_ids", "attention_mask"]

   __lowerCAmelCase         =   []
   __lowerCAmelCase         =   []
   def __init__(      self      :			List[str], UpperCamelCase__      :			Union[str, Any], UpperCamelCase__      :			List[str]=None, UpperCamelCase__      :			Optional[int]=None, UpperCamelCase__      :			int="</s>", UpperCamelCase__      :			List[Any]="</s>", UpperCamelCase__      :			Optional[int]="<s>", UpperCamelCase__      :			Tuple="<unk>", UpperCamelCase__      :			Union[str, Any]="<pad>", UpperCamelCase__      :			str="<mask>", UpperCamelCase__      :			Optional[Dict[str, Any]] = None, **UpperCamelCase__      :			Optional[int], )  ->			None:
        # Mask token behave like a normal word, i.e. include the space before it
        _A    		=  AddedToken(UpperCamelCase__, lstrip=UpperCamelCase__, rstrip=UpperCamelCase__     ) if isinstance(UpperCamelCase__, UpperCamelCase__     ) else mask_token

        _A    		=  {} if sp_model_kwargs is None else sp_model_kwargs

        _A    		=  kwargs.get('additional_special_tokens', []     )
        kwargs["additional_special_tokens"] += [
            code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
        ]

        super().__init__(
            src_lang=UpperCamelCase__, tgt_lang=UpperCamelCase__, eos_token=UpperCamelCase__, unk_token=UpperCamelCase__, sep_token=UpperCamelCase__, cls_token=UpperCamelCase__, pad_token=UpperCamelCase__, mask_token=UpperCamelCase__, sp_model_kwargs=self.sp_model_kwargs, **UpperCamelCase__, )

        _A    		=  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
        self.sp_model.Load(str(UpperCamelCase__     )     )
        _A    		=  vocab_file

        # Original fairseq vocab and spm vocab must be "aligned":
        # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
        # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
        # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
        # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

        # Mimic fairseq token-to-id alignment for the first 4 token
        _A    		=  {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

        # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
        _A    		=  1

        _A    		=  len(self.sp_model     )
        _A    		=  {
            code: self.sp_model_size + i + self.fairseq_offset for i, code in enumerate(UpperCamelCase__     )
        }
        _A    		=  {v: k for k, v in self.lang_code_to_id.items()}
        _A    		=  len(self.sp_model     ) + len(self.lang_code_to_id     ) + self.fairseq_offset

        self.fairseq_tokens_to_ids.update(self.lang_code_to_id     )
        _A    		=  {v: k for k, v in self.fairseq_tokens_to_ids.items()}

        _A    		=  src_lang if src_lang is not None else 'en_XX'
        _A    		=  self.lang_code_to_id[self._src_lang]
        _A    		=  tgt_lang
        self.set_src_lang_special_tokens(self._src_lang     )
   @property
   def 	__UpperCAmelCase   (      self      :			List[Any]     )  ->			int:
        return len(self.sp_model     ) + len(self.lang_code_to_id     ) + self.fairseq_offset + 1  # Plus 1 for the mask token
   @property
   def 	__UpperCAmelCase   (      self      :			int     )  ->			str:
        return self._src_lang
   @src_lang.setter
   def 	__UpperCAmelCase   (      self      :			List[str], UpperCamelCase__      :			str     )  ->			None:
        _A    		=  new_src_lang
        self.set_src_lang_special_tokens(self._src_lang     )
   def __getstate__(      self      :			Tuple     )  ->			Dict:
        _A    		=  self.__dict__.copy()
        _A    		=  None
        return state
   def __setstate__(      self      :			int, UpperCamelCase__      :			Dict     )  ->			None:
        _A    		=  d

        # for backward compatibility
        if not hasattr(self, 'sp_model_kwargs'     ):
             _A    		=  {}

        _A    		=  spm.SentencePieceProcessor(**self.sp_model_kwargs     )
        self.sp_model.Load(self.vocab_file     )
   def 	__UpperCAmelCase   (      self      :			Union[str, Any]     )  ->			Dict:
        _A    		=  {self.convert_ids_to_tokens(UpperCamelCase__     ): i for i in range(self.vocab_size     )}
        vocab.update(self.added_tokens_encoder     )
        return vocab
   def 	__UpperCAmelCase   (      self      :			str, UpperCamelCase__      :			str     )  ->			List[str]:
        return self.sp_model.encode(UpperCamelCase__, out_type=UpperCamelCase__     )
   def 	__UpperCAmelCase   (      self      :			Tuple, UpperCamelCase__      :			str     )  ->			int:
        if token in self.fairseq_tokens_to_ids:
             return self.fairseq_tokens_to_ids[token]
        _A    		=  self.sp_model.PieceToId(UpperCamelCase__     )

        # Need to return unknown token if the SP model returned 0
        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id
   def 	__UpperCAmelCase   (      self      :			Optional[int], UpperCamelCase__      :			int     )  ->			str:
        if index in self.fairseq_ids_to_tokens:
             return self.fairseq_ids_to_tokens[index]
        return self.sp_model.IdToPiece(index - self.fairseq_offset     )
   def 	__UpperCAmelCase   (      self      :			List[str], UpperCamelCase__      :			Optional[Any]     )  ->			Optional[Any]:
        _A    		=  []
        _A    		=  ''
        _A    		=  False
        for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                  if not prev_is_special:
                       out_string += " "
                  out_string += self.sp_model.decode(UpperCamelCase__     ) + token
                  _A    		=  True
                  _A    		=  []
             else:
                  current_sub_tokens.append(UpperCamelCase__     )
                  _A    		=  False
        out_string += self.sp_model.decode(UpperCamelCase__     )
        return out_string.strip()
   def 	__UpperCAmelCase   (      self      :			Optional[int], UpperCamelCase__      :			str, UpperCamelCase__      :			Optional[str] = None     )  ->			Tuple[str]:
        if not os.path.isdir(UpperCamelCase__     ):
             logger.error(f'Vocabulary path ({save_directory}) should be a directory'     )
             return
        _A    		=  os.path.join(
            UpperCamelCase__, (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']     )

        if os.path.abspath(self.vocab_file     ) != os.path.abspath(UpperCamelCase__     ) and os.path.isfile(self.vocab_file     ):
             copyfile(self.vocab_file, UpperCamelCase__     )
        elif not os.path.isfile(self.vocab_file     ):
             with open(UpperCamelCase__, 'wb'     ) as fi:
                  _A    		=  self.sp_model.serialized_model_proto()
                  fi.write(UpperCamelCase__     )

        return (out_vocab_file,)
   def 	__UpperCAmelCase   (      self      :			List[Any], UpperCamelCase__      :			List[int], UpperCamelCase__      :			Optional[List[int]] = None, UpperCamelCase__      :			bool = False     )  ->			List[int]:

        if already_has_special_tokens:
             return super().get_special_tokens_mask(
                 token_ids_a=UpperCamelCase__, token_ids_a=UpperCamelCase__, already_has_special_tokens=UpperCamelCase__     )

        _A    		=  [1] * len(self.prefix_tokens     )
        _A    		=  [1] * len(self.suffix_tokens     )
        if token_ids_a is None:
             return prefix_ones + ([0] * len(UpperCamelCase__     )) + suffix_ones
        return prefix_ones + ([0] * len(UpperCamelCase__     )) + ([0] * len(UpperCamelCase__     )) + suffix_ones
   def 	__UpperCAmelCase   (      self      :			List[str], UpperCamelCase__      :			List[int], UpperCamelCase__      :			Optional[List[int]] = None     )  ->			List[int]:
        if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + self.suffix_tokens
        # We don't expect to process pairs, but leave the pair logic for API consistency
        return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
   def 	__UpperCAmelCase   (      self      :			Optional[Any], UpperCamelCase__      :			Union[str, Any], UpperCamelCase__      :			str, UpperCamelCase__      :			Optional[str], UpperCamelCase__      :			Optional[str], **UpperCamelCase__      :			Any     )  ->			str:
        if src_lang is None or tgt_lang is None:
             raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'     )
        _A    		=  src_lang
        _A    		=  self(UpperCamelCase__, add_special_tokens=UpperCamelCase__, return_tensors=UpperCamelCase__, **UpperCamelCase__     )
        _A    		=  self.convert_tokens_to_ids(UpperCamelCase__     )
        _A    		=  tgt_lang_id
        return inputs
   def 	__UpperCAmelCase   (      self      :			str, UpperCamelCase__      :			List[str], UpperCamelCase__      :			str = "en_XX", UpperCamelCase__      :			Optional[List[str]] = None, UpperCamelCase__      :			str = "ro_RO", **UpperCamelCase__      :			Dict, )  ->			BatchEncoding:
        _A    		=  src_lang
        _A    		=  tgt_lang
        return super().prepare_seqaseq_batch(UpperCamelCase__, UpperCamelCase__, **UpperCamelCase__     )
   def 	__UpperCAmelCase   (      self      :			Any     )  ->			int:
        return self.set_src_lang_special_tokens(self.src_lang     )
   def 	__UpperCAmelCase   (      self      :			Any     )  ->			Any:
        return self.set_tgt_lang_special_tokens(self.tgt_lang     )
   def 	__UpperCAmelCase   (      self      :			str, UpperCamelCase__      :			str     )  ->			None:
        _A    		=  self.lang_code_to_id[src_lang]
        _A    		=  [self.cur_lang_code_id]
        _A    		=  [self.eos_token_id]

   def 	__UpperCAmelCase   (      self      :			List[str], UpperCamelCase__      :			str     )  ->			None:
        _A    		=  self.lang_code_to_id[tgt_lang]
        _A    		=  [self.cur_lang_code_id]
        _A    		=  [self.eos_token_id]




'''simple docstring'''






from typing import Dict, List

from nltk.translate import gleu_score

import datasets
from datasets import MetricInfo


_UpperCAmelCase    :     Any         =       '''\
@misc{wu2016googles,
      title={Google\'s Neural Machine Translation System: Bridging the Gap between Human and Machine Translation},
      author={Yonghui Wu and Mike Schuster and Zhifeng Chen and Quoc V. Le and Mohammad Norouzi and Wolfgang Macherey
              and Maxim Krikun and Yuan Cao and Qin Gao and Klaus Macherey and Jeff Klingner and Apurva Shah and Melvin
              Johnson and Xiaobing Liu and Łukasz Kaiser and Stephan Gouws and Yoshikiyo Kato and Taku Kudo and Hideto
              Kazawa and Keith Stevens and George Kurian and Nishant Patil and Wei Wang and Cliff Young and
              Jason Smith and Jason Riesa and Alex Rudnick and Oriol Vinyals and Greg Corrado and Macduff Hughes
              and Jeffrey Dean},
      year={2016},
      eprint={1609.08144},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}
'''

_UpperCAmelCase    :     str         =       '''\
The BLEU score has some undesirable properties when used for single
sentences, as it was designed to be a corpus measure. We therefore
use a slightly different score for our RL experiments which we call
the \'GLEU score\'. For the GLEU score, we record all sub-sequences of
1, 2, 3 or 4 tokens in output and target sequence (n-grams). We then
compute a recall, which is the ratio of the number of matching n-grams
to the number of total n-grams in the target (ground truth) sequence,
and a precision, which is the ratio of the number of matching n-grams
to the number of total n-grams in the generated output sequence. Then
GLEU score is simply the minimum of recall and precision. This GLEU
score\'s range is always between 0 (no matches) and 1 (all match) and
it is symmetrical when switching output and target. According to
our experiments, GLEU score correlates quite well with the BLEU
metric on a corpus level but does not have its drawbacks for our per
sentence reward objective.
'''

_UpperCAmelCase    :     Union[str, Any]         =       '''\
Computes corpus-level Google BLEU (GLEU) score of translated segments against one or more references.
Instead of averaging the sentence level GLEU scores (i.e. macro-average precision), Wu et al. (2016) sum up the matching
tokens and the max of hypothesis and reference tokens for each sentence, then compute using the aggregate values.

Args:
    predictions (list of str): list of translations to score.
        Each translation should be tokenized into a list of tokens.
    references (list of list of str): list of lists of references for each translation.
        Each reference should be tokenized into a list of tokens.
    min_len (int): The minimum order of n-gram this function should extract. Defaults to 1.
    max_len (int): The maximum order of n-gram this function should extract. Defaults to 4.

Returns:
    \'google_bleu\': google_bleu score

Examples:
    Example 1:
        >>> hyp1 = [\'It\', \'is\', \'a\', \'guide\', \'to\', \'action\', \'which\',
        ...         \'ensures\', \'that\', \'the\', \'rubber\', \'duck\', \'always\',
        ...         \'disobeys\', \'the\', \'commands\', \'of\', \'the\', \'cat\']
        >>> ref1a = [\'It\', \'is\', \'the\', \'guiding\', \'principle\', \'which\',
        ...          \'guarantees\', \'the\', \'rubber\', \'duck\', \'forces\', \'never\',
        ...          \'being\', \'under\', \'the\', \'command\', \'of\', \'the\', \'cat\']

        >>> hyp2 = [\'he\', \'read\', \'the\', \'book\', \'because\', \'he\', \'was\',
        ...         \'interested\', \'in\', \'world\', \'history\']
        >>> ref2a = [\'he\', \'was\', \'interested\', \'in\', \'world\', \'history\',
        ...          \'because\', \'he\', \'read\', \'the\', \'book\']

        >>> list_of_references = [[ref1a], [ref2a]]
        >>> hypotheses = [hyp1, hyp2]
        >>> google_bleu = datasets.load_metric("google_bleu")
        >>> results = google_bleu.compute(predictions=hypotheses, references=list_of_references)
        >>> print(round(results["google_bleu"], 2))
        0.44

    Example 2:
        >>> hyp1 = [\'It\', \'is\', \'a\', \'guide\', \'to\', \'action\', \'which\',
        ...         \'ensures\', \'that\', \'the\', \'rubber\', \'duck\', \'always\',
        ...         \'disobeys\', \'the\', \'commands\', \'of\', \'the\', \'cat\']
        >>> ref1a = [\'It\', \'is\', \'the\', \'guiding\', \'principle\', \'which\',
        ...          \'guarantees\', \'the\', \'rubber\', \'duck\', \'forces\', \'never\',
        ...          \'being\', \'under\', \'the\', \'command\', \'of\', \'the\', \'cat\']
        >>> ref1b = [\'It\', \'is\', \'a\', \'guide\', \'to\', \'action\', \'that\',
        ...          \'ensures\', \'that\', \'the\', \'rubber\', \'duck\', \'will\', \'never\',
        ...          \'heed\', \'the\', \'cat\', \'commands\']
        >>> ref1c = [\'It\', \'is\', \'the\', \'practical\', \'guide\', \'for\', \'the\',
        ...          \'rubber\', \'duck\', \'army\', \'never\', \'to\', \'heed\', \'the\', \'directions\',
        ...          \'of\', \'the\', \'cat\']

        >>> hyp2 = [\'he\', \'read\', \'the\', \'book\', \'because\', \'he\', \'was\',
        ...         \'interested\', \'in\', \'world\', \'history\']
        >>> ref2a = [\'he\', \'was\', \'interested\', \'in\', \'world\', \'history\',
        ...          \'because\', \'he\', \'read\', \'the\', \'book\']

        >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]
        >>> hypotheses = [hyp1, hyp2]
        >>> google_bleu = datasets.load_metric("google_bleu")
        >>> results = google_bleu.compute(predictions=hypotheses, references=list_of_references)
        >>> print(round(results["google_bleu"], 2))
        0.61

    Example 3:
        >>> hyp1 = [\'It\', \'is\', \'a\', \'guide\', \'to\', \'action\', \'which\',
        ...         \'ensures\', \'that\', \'the\', \'rubber\', \'duck\', \'always\',
        ...         \'disobeys\', \'the\', \'commands\', \'of\', \'the\', \'cat\']
        >>> ref1a = [\'It\', \'is\', \'the\', \'guiding\', \'principle\', \'which\',
        ...          \'guarantees\', \'the\', \'rubber\', \'duck\', \'forces\', \'never\',
        ...          \'being\', \'under\', \'the\', \'command\', \'of\', \'the\', \'cat\']
        >>> ref1b = [\'It\', \'is\', \'a\', \'guide\', \'to\', \'action\', \'that\',
        ...          \'ensures\', \'that\', \'the\', \'rubber\', \'duck\', \'will\', \'never\',
        ...          \'heed\', \'the\', \'cat\', \'commands\']
        >>> ref1c = [\'It\', \'is\', \'the\', \'practical\', \'guide\', \'for\', \'the\',
        ...          \'rubber\', \'duck\', \'army\', \'never\', \'to\', \'heed\', \'the\', \'directions\',
        ...          \'of\', \'the\', \'cat\']

        >>> hyp2 = [\'he\', \'read\', \'the\', \'book\', \'because\', \'he\', \'was\',
        ...         \'interested\', \'in\', \'world\', \'history\']
        >>> ref2a = [\'he\', \'was\', \'interested\', \'in\', \'world\', \'history\',
        ...          \'because\', \'he\', \'read\', \'the\', \'book\']

        >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]
        >>> hypotheses = [hyp1, hyp2]
        >>> google_bleu = datasets.load_metric("google_bleu")
        >>> results = google_bleu.compute(predictions=hypotheses, references=list_of_references, min_len=2)
        >>> print(round(results["google_bleu"], 2))
        0.53

    Example 4:
        >>> hyp1 = [\'It\', \'is\', \'a\', \'guide\', \'to\', \'action\', \'which\',
        ...         \'ensures\', \'that\', \'the\', \'rubber\', \'duck\', \'always\',
        ...         \'disobeys\', \'the\', \'commands\', \'of\', \'the\', \'cat\']
        >>> ref1a = [\'It\', \'is\', \'the\', \'guiding\', \'principle\', \'which\',
        ...          \'guarantees\', \'the\', \'rubber\', \'duck\', \'forces\', \'never\',
        ...          \'being\', \'under\', \'the\', \'command\', \'of\', \'the\', \'cat\']
        >>> ref1b = [\'It\', \'is\', \'a\', \'guide\', \'to\', \'action\', \'that\',
        ...          \'ensures\', \'that\', \'the\', \'rubber\', \'duck\', \'will\', \'never\',
        ...          \'heed\', \'the\', \'cat\', \'commands\']
        >>> ref1c = [\'It\', \'is\', \'the\', \'practical\', \'guide\', \'for\', \'the\',
        ...          \'rubber\', \'duck\', \'army\', \'never\', \'to\', \'heed\', \'the\', \'directions\',
        ...          \'of\', \'the\', \'cat\']

        >>> hyp2 = [\'he\', \'read\', \'the\', \'book\', \'because\', \'he\', \'was\',
        ...         \'interested\', \'in\', \'world\', \'history\']
        >>> ref2a = [\'he\', \'was\', \'interested\', \'in\', \'world\', \'history\',
        ...          \'because\', \'he\', \'read\', \'the\', \'book\']

        >>> list_of_references = [[ref1a, ref1b, ref1c], [ref2a]]
        >>> hypotheses = [hyp1, hyp2]
        >>> google_bleu = datasets.load_metric("google_bleu")
        >>> results = google_bleu.compute(predictions=hypotheses,references=list_of_references, min_len=2, max_len=6)
        >>> print(round(results["google_bleu"], 2))
        0.4
'''







@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION    ,    _KWARGS_DESCRIPTION			)
class lowercase_       (		datasets.Metric			):







   """simple docstring"""

   def 	__UpperCAmelCase   (      self      :			List[Any]     )  ->			MetricInfo:
        return datasets.MetricInfo(
            description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features(
                {
                    'predictions': datasets.Sequence(datasets.Value('string', id='token'     ), id='sequence'     ),
                    'references': datasets.Sequence(
                        datasets.Sequence(datasets.Value('string', id='token'     ), id='sequence'     ), id='references'     ),
                }     ), )

   def 	__UpperCAmelCase   (      self      :			Dict, UpperCamelCase__      :			List[List[List[str]]], UpperCamelCase__      :			List[List[str]], UpperCamelCase__      :			int = 1, UpperCamelCase__      :			int = 4, )  ->			Dict[str, float]:
        return {
            "google_bleu": gleu_score.corpus_gleu(
                list_of_references=UpperCamelCase__, hypotheses=UpperCamelCase__, min_len=UpperCamelCase__, max_len=UpperCamelCase__     )
        }



