






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
	import torch


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


a__       =     logging.get_logger(__name__)
a__       =     {"""vocab_file""": """spiece.model"""}

a__       =     {
    """vocab_file""": {
        """AI-Sweden/gpt-sw3-126m""": """https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-350m""": """https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-1.6b""": """https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-6.7b""": """https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-20b""": """https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model""",
    }
}

a__       =     {
    """AI-Sweden/gpt-sw3-126m""": 2_0_4_8,
    """AI-Sweden/gpt-sw3-350m""": 2_0_4_8,
    """AI-Sweden/gpt-sw3-1.6b""": 2_0_4_8,
    """AI-Sweden/gpt-sw3-6.7b""": 2_0_4_8,
    """AI-Sweden/gpt-sw3-20b""": 2_0_4_8,
}


class    _lowerCAmelCase			(		lowercase_      ):



				"""simple docstring"""



				_lowercase				:   Tuple      =  VOCAB_FILES_NAMES
				_lowercase				:   Union[str, Any]      =  PRETRAINED_VOCAB_FILES_MAP
				_lowercase				:   Tuple      =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				_lowercase				:   str      =  ['''input_ids''', '''attention_mask''']



				def __init__(      self    :    Optional[int]			,		UpperCamelCase__    :    List[Any]			,		UpperCamelCase__    :    int=False			,		UpperCamelCase__    :    Tuple=False			,		UpperCamelCase__    :    List[Any]=False			,		UpperCamelCase__    :    Union[str, Any]=None			,		UpperCamelCase__    :    Any=None			,		UpperCamelCase__    :    List[Any]=None			,		UpperCamelCase__    :    str=None			,		UpperCamelCase__    :    Optional[Dict[str, Any]] = None			,		**UpperCamelCase__    :    List[str]			,		):






											'''simple docstring'''







											snake_case__	  =       {} if sp_model_kwargs is None else sp_model_kwargs

											snake_case__	  =       kwargs.get("""name_or_path""")
											if name_or_path is None:
																		logger.warning(
																		    """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
																		    """ you are testing the model, this can safely be ignored""")
																		snake_case__	  =       """None"""

											# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
											snake_case__	  =       """<|endoftext|>""" if eos_token is None else eos_token
											snake_case__	  =       """<unk>""" if unk_token is None else unk_token
											if "gpt-sw3-7b" in name_or_path:
																		snake_case__	  =       unk_token if pad_token is None else pad_token
																		snake_case__	  =       eos_token if bos_token is None else bos_token
											else:
																		snake_case__	  =       """<pad>""" if pad_token is None else pad_token
																		snake_case__	  =       """<s>""" if bos_token is None else bos_token

											super().__init__(
											    do_lower_case=UpperCamelCase__			,		remove_space=UpperCamelCase__			,		keep_accents=UpperCamelCase__			,		bos_token=UpperCamelCase__			,		eos_token=UpperCamelCase__			,		unk_token=UpperCamelCase__			,		pad_token=UpperCamelCase__			,		sp_model_kwargs=self.sp_model_kwargs			,		**UpperCamelCase__			,		)

											snake_case__	  =       do_lower_case
											snake_case__	  =       remove_space
											snake_case__	  =       keep_accents
											snake_case__	  =       vocab_file

											snake_case__	  =       spm.SentencePieceProcessor(**self.sp_model_kwargs)
											self.sp_model.Load(UpperCamelCase__)

											# Used for whitespace normalization in input texts
											# fmt : off
											snake_case__	  =       {""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
											# fmt : on

											# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
											snake_case__	  =       re.compile(
											    F'''[{''.join(map(UpperCamelCase__			,		list(range(0			,		9)) + list(range(1_1			,		3_2)) + list(range(1_2_7			,		1_6_0)) + [1_6_0, 1_7_3, 8_2_0_3]))}]''')



				def __getstate__(      self    :    Tuple):






											'''simple docstring'''







											snake_case__	  =       self.__dict__.copy()
											snake_case__	  =       None
											return state



				def __setstate__(      self    :    Dict			,		UpperCamelCase__    :    Optional[Any]):






											'''simple docstring'''







											snake_case__	  =       d

											# for backward compatibility
											if not hasattr(self			,		"""sp_model_kwargs"""):
																		snake_case__	  =       {}

											snake_case__	  =       spm.SentencePieceProcessor(**self.sp_model_kwargs)
											self.sp_model.Load(self.vocab_file)



				@property
				# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
				def 					__magic_name__							(      self    :    int):






											'''simple docstring'''







											return len(self.sp_model)



				def 					__magic_name__							(      self    :    List[str]			,		UpperCamelCase__    :    str):






											'''simple docstring'''







											snake_case__	  =       self.non_printing_characters_re.sub(""""""			,		UpperCamelCase__)

											# Normalize whitespaces
											snake_case__	  =       """""".join([char if char not in self.whitespaces else """ """ for char in text])

											# NFC Unicode normalization
											snake_case__	  =       unicodedata.normalize("""NFC"""			,		UpperCamelCase__)
											return text



				def 					__magic_name__							(      self    :    Any			,		UpperCamelCase__    :    str			,		**UpperCamelCase__    :    List[str]):






											'''simple docstring'''







											snake_case__	  =       self.preprocess_text(UpperCamelCase__)
											return self.sp_model.encode(UpperCamelCase__			,		out_type=UpperCamelCase__)



				def 					__magic_name__							(      self    :    Union[str, Any]			,		UpperCamelCase__    :    str):






											'''simple docstring'''







											return self.sp_model.PieceToId(UpperCamelCase__)



				def 					__magic_name__							(      self    :    Tuple			,		UpperCamelCase__    :    int):






											'''simple docstring'''







											return self.sp_model.IdToPiece(UpperCamelCase__)



				@staticmethod
				def 					__magic_name__							(      UpperCamelCase__    :    str):






											'''simple docstring'''







											return out_string



				def 					__magic_name__							(      self    :    Union[str, Any]			,		UpperCamelCase__    :    List[str]):






											'''simple docstring'''







											snake_case__	  =       []
											snake_case__	  =       """"""
											snake_case__	  =       False
											for token in tokens:
																		# make sure that special tokens are not decoded using sentencepiece model
																		if token in self.all_special_tokens:
																									# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																									if not prev_is_special:
																																out_string += " "

																									out_string += self.sp_model.decode(UpperCamelCase__) + token
																									snake_case__	  =       True
																									snake_case__	  =       []
																		else:
																									current_sub_tokens.append(UpperCamelCase__)
																									snake_case__	  =       False
											out_string += self.sp_model.decode(UpperCamelCase__)

											return out_string



				def 					__magic_name__							(      self    :    Any):






											'''simple docstring'''







											snake_case__	  =       {self.convert_ids_to_tokens(UpperCamelCase__): i for i in range(self.vocab_size)}
											vocab.update(self.added_tokens_encoder)
											return vocab



				def 					__magic_name__							(      self    :    Optional[Any]			,		UpperCamelCase__    :    str			,		UpperCamelCase__    :    Optional[str] = None):






											'''simple docstring'''







											if not os.path.isdir(UpperCamelCase__):
																		logger.error(F'''Vocabulary path ({save_directory}) should be a directory''')
																		return
											snake_case__	  =       os.path.join(
											    UpperCamelCase__			,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""])

											if os.path.abspath(self.vocab_file) != os.path.abspath(UpperCamelCase__) and os.path.isfile(self.vocab_file):
																		copyfile(self.vocab_file			,		UpperCamelCase__)
											elif not os.path.isfile(self.vocab_file):
																		with open(UpperCamelCase__			,		"""wb""") as fi:
																									snake_case__	  =       self.sp_model.serialized_model_proto()
																									fi.write(UpperCamelCase__)

											return (out_vocab_file,)



				def 					__magic_name__							(      self    :    str			,		UpperCamelCase__    :    Union[str, List[str]]			,		UpperCamelCase__    :    Union[str, bool] = False):






											'''simple docstring'''







											if isinstance(UpperCamelCase__			,		UpperCamelCase__):
																		snake_case__	  =       self.preprocess_text(UpperCamelCase__)
																		snake_case__	  =       self.sp_model.encode(UpperCamelCase__)
											else:
																		snake_case__	  =       [self.preprocess_text(UpperCamelCase__) for t in text]
																		snake_case__	  =       self.sp_model.encode(UpperCamelCase__)

											if return_tensors is True or return_tensors == "pt":
																		snake_case__	  =       torch.tensor(UpperCamelCase__)

											return token_ids



				def 					__magic_name__							(      self    :    str			,		UpperCamelCase__    :    Union[int, List[int]]):






											'''simple docstring'''







											return self.sp_model.decode(UpperCamelCase__)



				def 					__magic_name__							(      self    :    Union[str, Any]			,		UpperCamelCase__    :    "Conversation"):






											'''simple docstring'''







											snake_case__	  =       [F'''User: {text}''' if is_user else F'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
											snake_case__	  =       (
											    F'''{self.eos_token}{self.bos_token}''' + F'''{self.bos_token}'''.join(UpperCamelCase__) + F'''{self.bos_token}Bot:'''
											)
											return self.encode(text=UpperCamelCase__)








import comet  # From: unbabel-comet
import torch

import datasets


a__       =     datasets.logging.get_logger(__name__)

a__       =     """\
@inproceedings{rei-EtAl:2020:WMT,
   author    = {Rei, Ricardo  and  Stewart, Craig  and  Farinha, Ana C  and  Lavie, Alon},
   title     = {Unbabel's Participation in the WMT20 Metrics Shared Task},
   booktitle      = {Proceedings of the Fifth Conference on Machine Translation},
   month          = {November},
   year           = {2020},
   address        = {Online},
   publisher      = {Association for Computational Linguistics},
   pages     = {909--918},
}
@inproceedings{rei-etal-2020-comet,
   title = \"{COMET}: A Neural Framework for {MT} Evaluation\",
   author = \"Rei, Ricardo  and
      Stewart, Craig  and
      Farinha, Ana C  and
      Lavie, Alon\",
   booktitle = \"Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)\",
   month = nov,
   year = \"2020\",
   address = \"Online\",
   publisher = \"Association for Computational Linguistics\",
   url = \"https://www.aclweb.org/anthology/2020.emnlp-main.213\",
   pages = \"2685--2702\",
}
"""

a__       =     """\
Crosslingual Optimized Metric for Evaluation of Translation (COMET) is an open-source framework used to train Machine Translation metrics that achieve high levels of correlation with different types of human judgments (HTER, DA's or MQM).
With the release of the framework the authors also released fully trained models that were used to compete in the WMT20 Metrics Shared Task achieving SOTA in that years competition.

See the [README.md] file at https://unbabel.github.io/COMET/html/models.html for more information.
"""

a__       =     """
COMET score.

Args:

`sources` (list of str): Source sentences
`predictions` (list of str): candidate translations
`references` (list of str): reference translations
`cuda` (bool): If set to True, runs COMET using GPU
`show_progress` (bool): Shows progress
`model`: COMET model to be used. Will default to `wmt-large-da-estimator-1719` if None.

Returns:
    `samples`: List of dictionaries with `src`, `mt`, `ref` and `score`.
    `scores`: List of scores.

Examples:

    >>> comet_metric = datasets.load_metric('comet')
    >>> # comet_metric = load_metric('comet', 'wmt20-comet-da')  # you can also choose which model to use
    >>> source = [\"Dem Feuer konnte Einhalt geboten werden\", \"Schulen und Kindergärten wurden eröffnet.\"]
    >>> hypothesis = [\"The fire could be stopped\", \"Schools and kindergartens were open\"]
    >>> reference = [\"They were able to control the fire.\", \"Schools and kindergartens opened\"]
    >>> results = comet_metric.compute(predictions=hypothesis, references=reference, sources=source)
    >>> print([round(v, 2) for v in results[\"scores\"]])
    [0.19, 0.92]
"""


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,      _KWARGS_DESCRIPTION      )
class    _lowerCAmelCase			(		datasets.Metric      ):



				"""simple docstring"""



				def 					__magic_name__							(      self    :    Optional[Any]):






											'''simple docstring'''







											return datasets.MetricInfo(
											    description=_DESCRIPTION			,		citation=_CITATION			,		homepage="""https://unbabel.github.io/COMET/html/index.html"""			,		inputs_description=_KWARGS_DESCRIPTION			,		features=datasets.Features(
											        {
											            """sources""": datasets.Value("""string"""			,		id="""sequence"""),
											            """predictions""": datasets.Value("""string"""			,		id="""sequence"""),
											            """references""": datasets.Value("""string"""			,		id="""sequence"""),
											        })			,		codebase_urls=["""https://github.com/Unbabel/COMET"""]			,		reference_urls=[
											        """https://github.com/Unbabel/COMET""",
											        """https://www.aclweb.org/anthology/2020.emnlp-main.213/""",
											        """http://www.statmt.org/wmt20/pdf/2020.wmt-1.101.pdf6""",
											    ]			,		)



				def 					__magic_name__							(      self    :    str			,		UpperCamelCase__    :    Dict):






											'''simple docstring'''







											if self.config_name == "default":
																		snake_case__	  =       comet.load_from_checkpoint(comet.download_model("""wmt20-comet-da"""))
											else:
																		snake_case__	  =       comet.load_from_checkpoint(comet.download_model(self.config_name))



				def 					__magic_name__							(      self    :    List[str]			,		UpperCamelCase__    :    str			,		UpperCamelCase__    :    Tuple			,		UpperCamelCase__    :    List[Any]			,		UpperCamelCase__    :    int=None			,		UpperCamelCase__    :    List[Any]=False):






											'''simple docstring'''







											if gpus is None:
																		snake_case__	  =       1 if torch.cuda.is_available() else 0
											snake_case__	  =       {"""src""": sources, """mt""": predictions, """ref""": references}
											snake_case__	  =       [dict(zip(UpperCamelCase__			,		UpperCamelCase__)) for t in zip(*data.values())]
											snake_case__  , snake_case__	  =       self.scorer.predict(UpperCamelCase__			,		gpus=UpperCamelCase__			,		progress_bar=UpperCamelCase__)
											return {"mean_score": mean_score, "scores": scores}
