

"""simple docstring"""




import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation


_lowercase   :	int												=						logging.get_logger(__name__)

_lowercase   :	Tuple												=						{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

_lowercase   :	List[str]												=						{
    'tokenizer_file': {
        'EleutherAI/gpt-neox-20b': 'https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/tokenizer.json',
    },
}

_lowercase   :	Optional[int]												=						{
    'gpt-neox-20b': 20_48,
}
class      _UpperCAmelCase   (							_lowerCAmelCase       ):
	a__		: List[str]					=					VOCAB_FILES_NAMES
	a__		: List[Any]					=					PRETRAINED_VOCAB_FILES_MAP
	a__		: Union[str, Any]					=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	a__		: str					=					["input_ids", "attention_mask"]



	def __init__(  self     :       Tuple			,    _lowercase     :       Optional[int]=None			,    _lowercase     :       Optional[Any]=None			,    _lowercase     :       str=None			,    _lowercase     :       List[Any]="<|endoftext|>"			,    _lowercase     :       List[Any]="<|endoftext|>"			,    _lowercase     :       Optional[int]="<|endoftext|>"			,    _lowercase     :       List[Any]=False			,    **_lowercase     :       Optional[int]			,    ):
		super().__init__(
		    _lowercase			,    _lowercase			,    tokenizer_file=_lowercase			,    unk_token=_lowercase			,    bos_token=_lowercase			,    eos_token=_lowercase			,    add_prefix_space=_lowercase			,    **_lowercase			,    )

		__UpperCAmelCase      		= json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
		if pre_tok_state.get('''add_prefix_space'''			,    _lowercase						) != add_prefix_space:
			__UpperCAmelCase      		= getattr(_lowercase			,    pre_tok_state.pop('''type'''						)						)
			__UpperCAmelCase      		= add_prefix_space
			__UpperCAmelCase      		= pre_tok_class(**_lowercase						)

		__UpperCAmelCase      		= add_prefix_space



	def        a  (  self     :       int			,    _lowercase     :       str			,    _lowercase     :       Optional[str] = None						):
		__UpperCAmelCase      		= self._tokenizer.model.save(_lowercase			,    name=_lowercase						)
		return tuple(_lowercase						)



	def        a  (  self     :       Any			,    _lowercase     :       "Conversation"						):
		__UpperCAmelCase      		= []
		for is_user, text in conversation.iter_texts():
			input_ids.extend(self.encode(_lowercase			,    add_special_tokens=_lowercase						) + [self.eos_token_id]						)

		if len(_lowercase						) > self.model_max_length:
			__UpperCAmelCase      		= input_ids[-self.model_max_length :]
		return input_ids

import datasets


__a    :	Optional[Any]         =    """\
@InProceedings{conneau2018xnli,
  author = \"Conneau, Alexis
                 and Rinott, Ruty
                 and Lample, Guillaume
                 and Williams, Adina
                 and Bowman, Samuel R.
                 and Schwenk, Holger
                 and Stoyanov, Veselin\",
  title = \"XNLI: Evaluating Cross-lingual Sentence Representations\",
  booktitle = \"Proceedings of the 2018 Conference on Empirical Methods
               in Natural Language Processing\",
  year = \"2018\",
  publisher = \"Association for Computational Linguistics\",
  location = \"Brussels, Belgium\",
}
"""

__a    :	Optional[Any]         =    """\
XNLI is a subset of a few thousand examples from MNLI which has been translated
into a 14 different languages (some low-ish resource). As with MNLI, the goal is
to predict textual entailment (does sentence A imply/contradict/neither sentence
B) and is a classification task (given two sentences, predict one of three
labels).
"""

__a    :	str         =    """
Computes XNLI score which is just simple accuracy.
Args:
    predictions: Predicted labels.
    references: Ground truth labels.
Returns:
    'accuracy': accuracy
Examples:

    >>> predictions = [0, 1]
    >>> references = [0, 1]
    >>> xnli_metric = datasets.load_metric(\"xnli\")
    >>> results = xnli_metric.compute(predictions=predictions, references=references)
    >>> print(results)
    {'accuracy': 1.0}
"""





def 							UpperCAmelCase		(       lowercase				,     lowercase				):







				"""simple docstring"""



				return (preds == labels).mean()




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION		,_KWARGS_DESCRIPTION					)
class   _UpperCamelCase (		datasets.Metric					):


			"""simple docstring"""






			def 			_SCREAMING_SNAKE_CASE     (  self		)	->   str:

							'''simple docstring'''


							return datasets.MetricInfo(
							    description=_DESCRIPTION ,  citation=_CITATION ,  inputs_description=_KWARGS_DESCRIPTION ,  features=datasets.Features(
							        {
							            '''predictions''': datasets.Value('''int64''' if self.config_name != '''sts-b''' else '''float32'''		),
							            '''references''': datasets.Value('''int64''' if self.config_name != '''sts-b''' else '''float32'''		),
							        }		) ,  codebase_urls=[] ,  reference_urls=[] ,  format='''numpy''' ,  )





			def 			_SCREAMING_SNAKE_CASE     (  self ,  lowerCAmelCase__ ,  lowerCAmelCase__		)	->   Dict:

							'''simple docstring'''


							return {"accuracy": simple_accuracy(lowerCAmelCase__ ,  lowerCAmelCase__		)}