




'''simple docstring'''


import json
import os
import unittest

from transformers import CLIPTokenizer, CLIPTokenizerFast
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class     A					(	_a  ,unittest.TestCase					):
							lowercase_           =       CLIPTokenizer
							lowercase_           =       CLIPTokenizerFast
							lowercase_           =       True
							lowercase_           =       {}
							lowercase_           =       False







							def      __lowerCAmelCase    (						self :						str      )    ->							Union[str, Any]:


														"""simple docstring"""


														super().setUp()

														# fmt: off
														_a			  =       ['''l''', '''o''', '''w''', '''e''', '''r''', '''s''', '''t''', '''i''', '''d''', '''n''', '''lo''', '''l</w>''', '''w</w>''', '''r</w>''', '''t</w>''', '''low</w>''', '''er</w>''', '''lowest</w>''', '''newer</w>''', '''wider''', '''<unk>''', '''<|startoftext|>''', '''<|endoftext|>''']
														# fmt: on
														_a			  =       dict(zip(lowerCAmelCase_							,    range(len(lowerCAmelCase_      )      )      )      )
														_a			  =       ['''#version: 0.2''', '''l o''', '''lo w</w>''', '''e r</w>''']
														_a			  =       {'''unk_token''': '''<unk>'''}

														_a			  =       os.path.join(self.tmpdirname							,    VOCAB_FILES_NAMES['''vocab_file''']      )
														_a			  =       os.path.join(self.tmpdirname							,    VOCAB_FILES_NAMES['''merges_file''']      )
														with open(self.vocab_file							,    '''w'''							,    encoding='''utf-8'''      ) as fp:
																					fp.write(json.dumps(lowerCAmelCase_      ) + '''\n'''      )
														with open(self.merges_file							,    '''w'''							,    encoding='''utf-8'''      ) as fp:
																					fp.write('''\n'''.join(lowerCAmelCase_      )      )







							def      __lowerCAmelCase    (						self :						str							,    **lowerCAmelCase_ :						str      )    ->							List[str]:


														"""simple docstring"""


														kwargs.update(self.special_tokens_map      )
														return CLIPTokenizer.from_pretrained(self.tmpdirname							,    **lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						Dict							,    **lowerCAmelCase_ :						Optional[Any]      )    ->							int:


														"""simple docstring"""


														kwargs.update(self.special_tokens_map      )
														return CLIPTokenizerFast.from_pretrained(self.tmpdirname							,    **lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						Optional[Any]							,    lowerCAmelCase_ :						List[Any]      )    ->							Union[str, Any]:


														"""simple docstring"""


														_a			  =       '''lower newer'''
														_a			  =       '''lower newer'''
														return input_text, output_text







							def      __lowerCAmelCase    (						self :						Union[str, Any]      )    ->							Optional[Any]:


														"""simple docstring"""


														_a			  =       CLIPTokenizer(self.vocab_file							,    self.merges_file							,    **self.special_tokens_map      )
														_a			  =       '''lower newer'''
														_a			  =       ['''lo''', '''w''', '''er</w>''', '''n''', '''e''', '''w''', '''er</w>''']
														_a			  =       tokenizer.tokenize(lowerCAmelCase_      )
														self.assertListEqual(lowerCAmelCase_							,    lowerCAmelCase_      )

														_a			  =       tokens + [tokenizer.unk_token]
														_a			  =       [10, 2, 16, 9, 3, 2, 16, 20]
														self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase_      )							,    lowerCAmelCase_      )







							@require_ftfy
							def      __lowerCAmelCase    (						self :						Union[str, Any]      )    ->							Optional[int]:


														"""simple docstring"""


														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																												_a			  =       self.tokenizer_class.from_pretrained(lowerCAmelCase_							,    **lowerCAmelCase_      )
																												_a			  =       self.rust_tokenizer_class.from_pretrained(lowerCAmelCase_							,    **lowerCAmelCase_      )

																												_a			  =       '''A\n\'ll 11p223RF☆ho!!to?\'d\'d\'\'d of a cat to-$\'\'d.'''
																												_a			  =       tokenizer_s.tokenize(lowerCAmelCase_      )
																												_a			  =       tokenizer_r.tokenize(lowerCAmelCase_      )

																												self.assertListEqual(lowerCAmelCase_							,    lowerCAmelCase_      )

																												# Test that the tokenization is identical on an example containing a character (Latin Small Letter A
																												# with Tilde) encoded in 2 different ways
																												_a			  =       '''xa\u0303y''' + ''' ''' + '''x\xe3y'''
																												_a			  =       tokenizer_s.tokenize(lowerCAmelCase_      )
																												_a			  =       tokenizer_r.tokenize(lowerCAmelCase_      )

																												self.assertListEqual(lowerCAmelCase_							,    lowerCAmelCase_      )

																												# Test that the tokenization is identical on unicode of space type
																												_a			  =       [
																												    '''\u0009''',  # (horizontal tab, '\t')
																												    '''\u000B''',  # (vertical tab)
																												    '''\u000C''',  # (form feed)
																												    '''\u0020''',  # (space, ' ')
																												    '''\u200E''',  # (left-to-right mark):w
																												    '''\u200F''',  # (right-to-left mark)
																												]
																												for unicode_seq in spaces_unicodes:
																																			_a			  =       tokenizer_s.tokenize(lowerCAmelCase_      )
																																			_a			  =       tokenizer_r.tokenize(lowerCAmelCase_      )

																																			self.assertListEqual(lowerCAmelCase_							,    lowerCAmelCase_      )

																												# Test that the tokenization is identical on unicode of line break type
																												_a			  =       [
																												    '''\u000A''',  # (line feed, '\n')
																												    '''\r\n''',  # (carriage return and line feed, '\r\n')
																												    '''\u000D''',  # (carriage return, '\r')
																												    '''\r''',  # (carriage return, '\r')
																												    '''\u000D''',  # (carriage return, '\r')
																												    '''\u2028''',  # (line separator)
																												    '''\u2029''',  # (paragraph separator)
																												    # "\u0085", # (next line)
																												]

																												# The tokenization is not identical for the character "\u0085" (next line). The slow version using ftfy transforms
																												# it into the Horizontal Ellipsis character "…" ("\u2026") while the fast version transforms it into a
																												# space (and thus into an empty list).

																												for unicode_seq in line_break_unicodes:
																																			_a			  =       tokenizer_s.tokenize(lowerCAmelCase_      )
																																			_a			  =       tokenizer_r.tokenize(lowerCAmelCase_      )

																																			self.assertListEqual(lowerCAmelCase_							,    lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						Dict      )    ->							Dict:


														"""simple docstring"""


														for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																					with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																												_a			  =       '''hello'''  # `hello` is a token in the vocabulary of `pretrained_name`
																												_a			  =       F'{text_of_1_token} {text_of_1_token}'

																												_a			  =       self.rust_tokenizer_class.from_pretrained(
																												    lowerCAmelCase_							,    use_fast=lowerCAmelCase_							,    )
																												_a			  =       tokenizer_r(lowerCAmelCase_							,    return_offsets_mapping=lowerCAmelCase_							,    add_special_tokens=lowerCAmelCase_      )
																												self.assertEqual(encoding.offset_mapping[0]							,    (0, len(lowerCAmelCase_      ))      )
																												self.assertEqual(
																												    encoding.offset_mapping[1]							,    (len(lowerCAmelCase_      ) + 1, len(lowerCAmelCase_      ) + 1 + len(lowerCAmelCase_      ))							,    )

																												_a			  =       F' {text}'

																												_a			  =       self.rust_tokenizer_class.from_pretrained(
																												    lowerCAmelCase_							,    use_fast=lowerCAmelCase_							,    )
																												_a			  =       tokenizer_r(lowerCAmelCase_							,    return_offsets_mapping=lowerCAmelCase_							,    add_special_tokens=lowerCAmelCase_      )
																												self.assertEqual(encoding.offset_mapping[0]							,    (1, 1 + len(lowerCAmelCase_      ))      )
																												self.assertEqual(
																												    encoding.offset_mapping[1]							,    (1 + len(lowerCAmelCase_      ) + 1, 1 + len(lowerCAmelCase_      ) + 1 + len(lowerCAmelCase_      ))							,    )







							def      __lowerCAmelCase    (						self :						Union[str, Any]      )    ->							Union[str, Any]:


														"""simple docstring"""


														with self.assertRaises(lowerCAmelCase_      ) as context:
																					self.rust_tokenizer_class.from_pretrained('''robot-test/old-clip-tokenizer'''      )

														self.assertTrue(
														    context.exception.args[0].startswith(
														        '''The `backend_tokenizer` provided does not match the expected format.'''      )      )







							@require_ftfy
							def      __lowerCAmelCase    (						self :						Optional[int]      )    ->							List[Any]:


														"""simple docstring"""


														super().test_tokenization_python_rust_equals()







							def      __lowerCAmelCase    (						self :						Dict      )    ->							int:


														"""simple docstring"""


														pass







'''simple docstring'''


import os

from bleurt import score  # From: git+https://github.com/google-research/bleurt.git

import datasets


_snake_case   :   Any     							=	datasets.logging.get_logger(__name__)


_snake_case   :   Tuple     							=	'\\n@inproceedings{bleurt,\n  title={BLEURT: Learning Robust Metrics for Text Generation},\n  author={Thibault Sellam and Dipanjan Das and Ankur P. Parikh},\n  booktitle={ACL},\n  year={2020},\n  url={https://arxiv.org/abs/2004.04696}\n}\n'

_snake_case   :   Optional[Any]     							=	'\\nBLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018)\nand then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations. You may run BLEURT out-of-the-box or fine-tune\nit for your specific application (the latter is expected to perform better).\n\nSee the project\'s README at https://github.com/google-research/bleurt#readme for more information.\n'

_snake_case   :   Optional[int]     							=	'\nBLEURT score.\n\nArgs:\n    `predictions` (list of str): prediction/candidate sentences\n    `references` (list of str): reference sentences\n    `checkpoint` BLEURT checkpoint. Will default to BLEURT-tiny if None.\n\nReturns:\n    \'scores\': List of scores.\nExamples:\n\n    >>> predictions = ["hello there", "general kenobi"]\n    >>> references = ["hello there", "general kenobi"]\n    >>> bleurt = datasets.load_metric("bleurt")\n    >>> results = bleurt.compute(predictions=predictions, references=references)\n    >>> print([round(v, 2) for v in results["scores"]])\n    [1.03, 1.04]\n'

_snake_case   :   Dict     							=	{
    'bleurt-tiny-128': 'https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip',
    'bleurt-tiny-512': 'https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip',
    'bleurt-base-128': 'https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip',
    'bleurt-base-512': 'https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip',
    'bleurt-large-128': 'https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip',
    'bleurt-large-512': 'https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip',
    'BLEURT-20-D3': 'https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D3.zip',
    'BLEURT-20-D6': 'https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D6.zip',
    'BLEURT-20-D12': 'https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D12.zip',
    'BLEURT-20': 'https://storage.googleapis.com/bleurt-oss-21/BLEURT-20.zip',
}

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION  ,_KWARGS_DESCRIPTION					)
class     A					(	datasets.Metric					):







							def      __lowerCAmelCase    (						self :						int      )    ->							Dict:


														"""simple docstring"""


														return datasets.MetricInfo(
														    description=_DESCRIPTION							,    citation=_CITATION							,    homepage='''https://github.com/google-research/bleurt'''							,    inputs_description=_KWARGS_DESCRIPTION							,    features=datasets.Features(
														        {
														            '''predictions''': datasets.Value('''string'''							,    id='''sequence'''      ),
														            '''references''': datasets.Value('''string'''							,    id='''sequence'''      ),
														        }      )							,    codebase_urls=['''https://github.com/google-research/bleurt''']							,    reference_urls=['''https://github.com/google-research/bleurt''', '''https://arxiv.org/abs/2004.04696''']							,    )







							def      __lowerCAmelCase    (						self :						Dict							,    lowerCAmelCase_ :						str      )    ->							Tuple:


														"""simple docstring"""


														if self.config_name == "default":
																					logger.warning(
																					    '''Using default BLEURT-Base checkpoint for sequence maximum length 128. '''
																					    '''You can use a bigger model for better results with e.g.: datasets.load_metric(\'bleurt\', \'bleurt-large-512\').'''      )
																					_a			  =       '''bleurt-base-128'''

														if self.config_name.lower() in CHECKPOINT_URLS:
																					_a			  =       self.config_name.lower()

														elif self.config_name.upper() in CHECKPOINT_URLS:
																					_a			  =       self.config_name.upper()

														else:
																					raise KeyError(
																					    F'{self.config_name} model not found. You should supply the name of a model checkpoint for bleurt in {CHECKPOINT_URLS.keys()}'      )

														# download the model checkpoint specified by self.config_name and set up the scorer
														_a			  =       dl_manager.download_and_extract(CHECKPOINT_URLS[checkpoint_name]      )
														_a			  =       score.BleurtScorer(os.path.join(lowerCAmelCase_							,    lowerCAmelCase_      )      )







							def      __lowerCAmelCase    (						self :						str							,    lowerCAmelCase_ :						int							,    lowerCAmelCase_ :						Union[str, Any]      )    ->							Optional[int]:


														"""simple docstring"""


														_a			  =       self.scorer.score(references=lowerCAmelCase_							,    candidates=lowerCAmelCase_      )
														return {"scores": scores}

