

import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_albert import AlbertTokenizer
else:
   _lowerCamelCase      :					Any					    =      None

_lowerCamelCase      :					Optional[Any]					    =      logging.get_logger(__name__)
_lowerCamelCase      :					List[Any]					    =      {'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

_lowerCamelCase      :					str					    =      {
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json''',
    },
}

_lowerCamelCase      :					str					    =      {
    '''albert-base-v1''': 512,
    '''albert-large-v1''': 512,
    '''albert-xlarge-v1''': 512,
    '''albert-xxlarge-v1''': 512,
    '''albert-base-v2''': 512,
    '''albert-large-v2''': 512,
    '''albert-xlarge-v2''': 512,
    '''albert-xxlarge-v2''': 512,
}

_lowerCamelCase      :					str					    =      '''▁'''





class        lowerCAmelCase__  (					__magic_name__			):



       '''simple docstring'''







       lowercase_						=    VOCAB_FILES_NAMES
       lowercase_						=    PRETRAINED_VOCAB_FILES_MAP
       lowercase_						=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowercase_						=    AlbertTokenizer

       def __init__(							self ,   lowercase__=None ,   lowercase__=None ,   lowercase__=True ,   lowercase__=True ,   lowercase__=False ,   lowercase__="[CLS]" ,   lowercase__="[SEP]" ,   lowercase__="<unk>" ,   lowercase__="[SEP]" ,   lowercase__="<pad>" ,   lowercase__="[CLS]" ,   lowercase__="[MASK]" ,   **lowercase__ ,   ):







             '''simple docstring'''



             __A    =(
                 AddedToken(lowercase__ ,   lstrip=lowercase__ ,   rstrip=lowercase__ ,   normalized=lowercase__     )
                 if isinstance(lowercase__ ,   lowercase__     )
                 else mask_token
             )

             super().__init__(
                 lowercase__ ,   tokenizer_file=lowercase__ ,   do_lower_case=lowercase__ ,   remove_space=lowercase__ ,   keep_accents=lowercase__ ,   bos_token=lowercase__ ,   eos_token=lowercase__ ,   unk_token=lowercase__ ,   sep_token=lowercase__ ,   pad_token=lowercase__ ,   cls_token=lowercase__ ,   mask_token=lowercase__ ,   **lowercase__ ,   )

             __A    =do_lower_case
             __A    =remove_space
             __A    =keep_accents
             __A    =vocab_file
             __A    =False if not self.vocab_file else True

       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__ = None     ):







             '''simple docstring'''



             __A    =[self.sep_token_id]
             __A    =[self.cls_token_id]
             if token_ids_a is None:
                   return cls + token_ids_a + sep
             return cls + token_ids_a + sep + token_ids_a + sep

       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__ = None     ):







             '''simple docstring'''



             __A    =[self.sep_token_id]
             __A    =[self.cls_token_id]

             if token_ids_a is None:
                   return len(cls + token_ids_a + sep     ) * [0]
             return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]


       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__ = None     ):







             '''simple docstring'''



             if not self.can_save_slow_tokenizer:
                   raise ValueError(
                       '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                       '''tokenizer.'''     )

             if not os.path.isdir(lowercase__     ):
                   logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''     )
                   return
             __A    =os.path.join(
                 lowercase__ ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']     )

             if os.path.abspath(self.vocab_file     ) != os.path.abspath(lowercase__     ):
                   copyfile(self.vocab_file ,   lowercase__     )

             return (out_vocab_file,)





import numpy as np
from nltk.translate import meteor_score

import datasets
from datasets.config import importlib_metadata, version


_lowerCamelCase      :					List[str]					    =      version.parse(importlib_metadata.version('''nltk'''))
if NLTK_VERSION >= version.Version('''3.6.4'''):
   from nltk import word_tokenize


_lowerCamelCase      :					Dict					    =      '''\
@inproceedings{banarjee2005,
  title     = {{METEOR}: An Automatic Metric for {MT} Evaluation with Improved Correlation with Human Judgments},
  author    = {Banerjee, Satanjeev  and Lavie, Alon},
  booktitle = {Proceedings of the {ACL} Workshop on Intrinsic and Extrinsic Evaluation Measures for Machine Translation and/or Summarization},
  month     = jun,
  year      = {2005},
  address   = {Ann Arbor, Michigan},
  publisher = {Association for Computational Linguistics},
  url       = {https://www.aclweb.org/anthology/W05-0909},
  pages     = {65--72},
}
'''

_lowerCamelCase      :					str					    =      '''\
METEOR, an automatic metric for machine translation evaluation
that is based on a generalized concept of unigram matching between the
machine-produced translation and human-produced reference translations.
Unigrams can be matched based on their surface forms, stemmed forms,
and meanings; furthermore, METEOR can be easily extended to include more
advanced matching strategies. Once all generalized unigram matches
between the two strings have been found, METEOR computes a score for
this matching using a combination of unigram-precision, unigram-recall, and
a measure of fragmentation that is designed to directly capture how
well-ordered the matched words in the machine translation are in relation
to the reference.

METEOR gets an R correlation value of 0.347 with human evaluation on the Arabic
data and 0.331 on the Chinese data. This is shown to be an improvement on
using simply unigram-precision, unigram-recall and their harmonic F1
combination.
'''

_lowerCamelCase      :					Union[str, Any]					    =      '''
Computes METEOR score of translated segments against one or more references.
Args:
    predictions: list of predictions to score. Each prediction
        should be a string with tokens separated by spaces.
    references: list of reference for each prediction. Each
        reference should be a string with tokens separated by spaces.
    alpha: Parameter for controlling relative weights of precision and recall. default: 0.9
    beta: Parameter for controlling shape of penalty as a function of fragmentation. default: 3
    gamma: Relative weight assigned to fragmentation penalty. default: 0.5
Returns:
    \'meteor\': meteor score.
Examples:

    >>> meteor = datasets.load_metric(\'meteor\')
    >>> predictions = ["It is a guide to action which ensures that the military always obeys the commands of the party"]
    >>> references = ["It is a guide to action that ensures that the military will forever heed Party commands"]
    >>> results = meteor.compute(predictions=predictions, references=references)
    >>> print(round(results["meteor"], 4))
    0.6944
'''





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION				,							_KWARGS_DESCRIPTION			)
class        lowerCAmelCase__  (					datasets.Metric			):



       '''simple docstring'''







       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             return datasets.MetricInfo(
                 description=_DESCRIPTION ,   citation=_CITATION ,   inputs_description=_KWARGS_DESCRIPTION ,   features=datasets.Features(
                     {
                         '''predictions''': datasets.Value('''string''' ,   id='''sequence'''     ),
                         '''references''': datasets.Value('''string''' ,   id='''sequence'''     ),
                     }     ) ,   codebase_urls=['''https://github.com/nltk/nltk/blob/develop/nltk/translate/meteor_score.py'''] ,   reference_urls=[
                     '''https://www.nltk.org/api/nltk.translate.html#module-nltk.translate.meteor_score''',
                     '''https://en.wikipedia.org/wiki/METEOR''',
                 ] ,   )

       def     __UpperCamelCase     (							self ,   lowercase__     ):







             '''simple docstring'''



             import nltk

             nltk.download('''wordnet'''     )
             if NLTK_VERSION >= version.Version('''3.6.5'''     ):
                   nltk.download('''punkt'''     )
             if NLTK_VERSION >= version.Version('''3.6.6'''     ):
                   nltk.download('''omw-1.4'''     )


       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__ ,   lowercase__=0.9 ,   lowercase__=3 ,   lowercase__=0.5     ):







             '''simple docstring'''



             if NLTK_VERSION >= version.Version('''3.6.5'''     ):
                   __A    =[
                       meteor_score.single_meteor_score(
                           word_tokenize(lowercase__     ) ,   word_tokenize(lowercase__     ) ,   alpha=lowercase__ ,   beta=lowercase__ ,   gamma=lowercase__     )
                       for ref, pred in zip(lowercase__ ,   lowercase__     )
                   ]
             else:
                   __A    =[
                       meteor_score.single_meteor_score(lowercase__ ,   lowercase__ ,   alpha=lowercase__ ,   beta=lowercase__ ,   gamma=lowercase__     )
                       for ref, pred in zip(lowercase__ ,   lowercase__     )
                   ]

             return {"meteor": np.mean(lowercase__     )}


