






"""simple docstring"""
import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
							import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase       =				logging.get_logger(__name__)

_lowerCAmelCase       =				{"vocab_file": "vocab.txt"}

_lowerCAmelCase       =				{
    "vocab_file": {
        "openbmb/cpm-ant-10b": "https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt",
    },
}

_lowerCAmelCase       =				{
    "openbmb/cpm-ant-10b": 1_024,
}






def       __UpperCamelCase						(		snake_case__       ):
	A_		:    Optional[int]						     = collections.OrderedDict()
	with open(snake_case__ , """r""" , encoding="""utf-8"""       ) as reader:
		A_		:    Tuple						     = reader.readlines()
	for index, token in enumerate(snake_case__       ):
		A_		:    Optional[Any]						     = token.rstrip("""\n"""       )
		A_		:    List[str]						     = index
	return vocab



class   SCREAMING_SNAKE_CASE	(					_SCREAMING_SNAKE_CASE    ):





	"""simple docstring"""




	def __init__(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_="<unk>"  ,  lowerCAmelCase_=200				):
		A_		:    int						     = vocab
		A_		:    Optional[Any]						     = unk_token
		A_		:    str						     = max_input_chars_per_word





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		A_		:    Optional[Any]						     = list(lowerCAmelCase_				)
		if len(lowerCAmelCase_				) > self.max_input_chars_per_word:
			return [self.unk_token]

		A_		:    Optional[int]						     = 0
		A_		:    Dict						     = []
		while start < len(lowerCAmelCase_				):
			A_		:    Optional[int]						     = len(lowerCAmelCase_				)
			A_		:    int						     = None
			while start < end:
				A_		:    List[str]						     = """""".join(chars[start:end]				)
				if substr in self.vocab:
					A_		:    Dict						     = substr
					break
				end -= 1
			if cur_substr is None:
				sub_tokens.append(self.unk_token				)
				start += 1
			else:
				sub_tokens.append(lowerCAmelCase_				)
				A_		:    Dict						     = end

		return sub_tokens






class   SCREAMING_SNAKE_CASE	(					_SCREAMING_SNAKE_CASE    ):





	"""simple docstring"""




	_A							:					Tuple 			=					VOCAB_FILES_NAMES
	_A							:					Optional[int] 			=					PRETRAINED_VOCAB_FILES_MAP
	_A							:					str 			=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_A							:					Optional[Any] 			=					["""input_ids""", """attention_mask"""]
	_A							:					Optional[Any] 			=					False





	def __init__(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_="<d>"  ,  lowerCAmelCase_="</d>"  ,  lowerCAmelCase_="<s>"  ,  lowerCAmelCase_="</s>"  ,  lowerCAmelCase_="<pad>"  ,  lowerCAmelCase_="<unk>"  ,  lowerCAmelCase_="</n>"  ,  lowerCAmelCase_="</_>"  ,  lowerCAmelCase_="left"  ,  **lowerCAmelCase_  ,  ):
		requires_backends(self  ,  ["""jieba"""]				)
		super().__init__(
		    bod_token=lowerCAmelCase_  ,  eod_token=lowerCAmelCase_  ,  bos_token=lowerCAmelCase_  ,  eos_token=lowerCAmelCase_  ,  pad_token=lowerCAmelCase_  ,  unk_token=lowerCAmelCase_  ,  line_token=lowerCAmelCase_  ,  space_token=lowerCAmelCase_  ,  padding_side=lowerCAmelCase_  ,  **lowerCAmelCase_  ,  )
		A_		:    Optional[Any]						     = bod_token
		A_		:    Union[str, Any]						     = eod_token
		A_		:    List[Any]						     = load_vocab(lowerCAmelCase_				)
		A_		:    Optional[int]						     = self.encoder[space_token]
		A_		:    Any						     = self.encoder[line_token]

		del self.encoder[space_token]
		del self.encoder[line_token]

		A_		:    str						     = collections.OrderedDict(sorted(self.encoder.items()  ,  key=lambda lowerCAmelCase_				: x[1]				)				)
		A_		:    List[Any]						     = {v: k for k, v in self.encoder.items()}

		A_		:    int						     = WordpieceTokenizer(vocab=self.encoder  ,  unk_token=self.unk_token				)





	@property
	def  lowerCamelCase(self				):
		return self.encoder[self.bod_token]





	@property
	def  lowerCamelCase(self				):
		return self.encoder[self.eod_token]





	@property
	def  lowerCamelCase(self				):
		return self.encoder["\n"]





	@property
	def  lowerCamelCase(self				):
		return len(self.encoder				)





	def  lowerCamelCase(self				):
		return dict(self.encoder  ,  **self.added_tokens_encoder				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		A_		:    int						     = []
		for x in jieba.cut(lowerCAmelCase_  ,  cut_all=lowerCAmelCase_				):
			output_tokens.extend(self.wordpiece_tokenizer.tokenize(lowerCAmelCase_				)				)
		return output_tokens





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  **lowerCAmelCase_				):
		A_		:    Dict						     = [i for i in token_ids if i >= 0]
		A_		:    str						     = [
		    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
		]
		return super()._decode(lowerCAmelCase_  ,  **lowerCAmelCase_				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		return token in self.encoder





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		return "".join(lowerCAmelCase_				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		return self.encoder.get(lowerCAmelCase_  ,  self.encoder.get(self.unk_token				)				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		return self.decoder.get(lowerCAmelCase_  ,  self.unk_token				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_ = None				):
		if os.path.isdir(lowerCAmelCase_				):
			A_		:    List[str]						     = os.path.join(
			    lowerCAmelCase_  ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)
		else:
			A_		:    Dict						     = (filename_prefix + """-""" if filename_prefix else """""") + save_directory
		A_		:    Tuple						     = 0
		if " " in self.encoder:
			A_		:    int						     = self.encoder[""" """]
			del self.encoder[" "]
		if "\n" in self.encoder:
			A_		:    Tuple						     = self.encoder["""\n"""]
			del self.encoder["\n"]
		A_		:    Dict						     = collections.OrderedDict(sorted(self.encoder.items()  ,  key=lambda lowerCAmelCase_				: x[1]				)				)
		with open(lowerCAmelCase_  ,  """w"""  ,  encoding="""utf-8"""				) as writer:
			for token, token_index in self.encoder.items():
				if index != token_index:
					logger.warning(
					    f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
					    """ Please check that the vocabulary is not corrupted!"""				)
					A_		:    Dict						     = token_index
				writer.write(token + """\n"""				)
				index += 1
		return (vocab_file,)





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_ = None				):
		if token_ids_a is None:
			return [self.bos_token_id] + token_ids_a
		return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_ = None  ,  lowerCAmelCase_ = False				):

		if already_has_special_tokens:
			return super().get_special_tokens_mask(
			    token_ids_a=lowerCAmelCase_  ,  token_ids_a=lowerCAmelCase_  ,  already_has_special_tokens=lowerCAmelCase_				)

		if token_ids_a is not None:
			return [1] + ([0] * len(lowerCAmelCase_				)) + [1] + ([0] * len(lowerCAmelCase_				))
		return [1] + ([0] * len(lowerCAmelCase_				))









"""simple docstring"""
import math_equivalence  # From: git+https://github.com/hendrycks/math.git

import datasets


_lowerCAmelCase       =				"\\n@article{hendrycksmath2021,\n  title={Measuring Mathematical Problem Solving With the MATH Dataset},\n  author={Dan Hendrycks\n    and Collin Burns\n    and Saurav Kadavath\n    and Akul Arora\n    and Steven Basart\n    and Eric Tang\n    and Dawn Song\n    and Jacob Steinhardt},\n  journal={arXiv preprint arXiv:2103.03874},\n  year={2021}\n}\n"


_lowerCAmelCase       =				"\\nThis metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset.\nIt first canonicalizes the inputs (e.g., converting \"1/2\" to \"\\frac{1}{2}\") and then computes accuracy.\n"


_lowerCAmelCase       =				r"\nCalculates accuracy after canonicalizing inputs.\n\nArgs:\n    predictions: list of predictions to score. Each prediction\n        is a string that contains natural language and LaTex.\n    references: list of reference for each prediction. Each\n        reference is a string that contains natural language\n        and LaTex.\nReturns:\n    accuracy: accuracy after canonicalizing inputs\n        (e.g., converting \"1/2\" to \"\\frac{1}{2}\")\n\nExamples:\n    >>> metric = datasets.load_metric(\"competition_math\")\n    >>> results = metric.compute(references=[\"\\frac{1}{2}\"], predictions=[\"1/2\"])\n    >>> print(results)\n    {'accuracy': 1.0}\n"



@datasets.utils.file_utils.add_end_docstrings(_DESCRIPTION							,							_KWARGS_DESCRIPTION    )
class   SCREAMING_SNAKE_CASE	(					datasets.Metric    ):





	"""simple docstring"""




	def  lowerCamelCase(self				):
		return datasets.MetricInfo(
		    description=_DESCRIPTION  ,  citation=_CITATION  ,  inputs_description=_KWARGS_DESCRIPTION  ,  features=datasets.Features(
		        {
		            """predictions""": datasets.Value("""string"""				),
		            """references""": datasets.Value("""string"""				),
		        }				)  ,  homepage="""https://github.com/hendrycks/math"""  ,  codebase_urls=["""https://github.com/hendrycks/math"""]  ,  )





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_				):
		A_		:    Optional[Any]						     = 0.0
		for i, j in zip(lowerCAmelCase_  ,  lowerCAmelCase_				):
			n_correct += 1.0 if math_equivalence.is_equiv(lowerCAmelCase_  ,  lowerCAmelCase_				) else 0.0
		A_		:    List[str]						     = n_correct / len(lowerCAmelCase_				)
		return {
		    "accuracy": accuracy,
		}

