
import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


lowercase        =  logging.getLogger(__name__)





@dataclass(frozen=UpperCAmelCase       )
class       __A:
					SCREAMING_SNAKE_CASE			    =    4_2
					SCREAMING_SNAKE_CASE			    =    4_2
					SCREAMING_SNAKE_CASE			    =    None
					SCREAMING_SNAKE_CASE			    =    None
					SCREAMING_SNAKE_CASE			    =    None





@dataclass(frozen=UpperCAmelCase       )
class       __A:
					SCREAMING_SNAKE_CASE			    =    4_2
					SCREAMING_SNAKE_CASE			    =    None
					SCREAMING_SNAKE_CASE			    =    None
					SCREAMING_SNAKE_CASE			    =    None
					SCREAMING_SNAKE_CASE			    =    None


if is_torch_available():
		import torch
		from torch.utils.data import Dataset





		class       __A(				UpperCAmelCase       ):
							SCREAMING_SNAKE_CASE			    =    4_2

							def __init__(	self					:     Any   ,					__UpperCamelCase					:     str   ,					__UpperCamelCase					:     PreTrainedTokenizer   ,					__UpperCamelCase					:     str   ,					__UpperCamelCase					:     Optional[int] = None   ,					__UpperCamelCase					:     Dict=False   ,					__UpperCamelCase					:     bool = False   ,					):
														lowerCamelCase_  =					hans_processors[task]()

														lowerCamelCase_  =					os.path.join(
														    UpperCamelCase_   ,					"""cached_{}_{}_{}_{}""".format(
														        """dev""" if evaluate else """train"""   ,					tokenizer.__class__.__name__   ,					str(UpperCamelCase_   )   ,					UpperCamelCase_   ,					)   ,					)
														lowerCamelCase_  =					processor.get_labels()
														if tokenizer.__class__ in (
														    RobertaTokenizer,
														    RobertaTokenizerFast,
														    XLMRobertaTokenizer,
														    BartTokenizer,
														    BartTokenizerFast,
														):
																					# HACK(label indices are swapped in RoBERTa pretrained model)
																					lowerCamelCase_  =					label_list[2], label_list[1]
														lowerCamelCase_  =					label_list

														# Make sure only the first process in distributed training processes the dataset,
														# and the others will use the cache.
														lowerCamelCase_  =					cached_features_file + '.lock'
														with FileLock(UpperCamelCase_   ):
																					if os.path.exists(UpperCamelCase_   ) and not overwrite_cache:
																												logger.info(F'''Loading features from cached file {cached_features_file}'''   )
																												lowerCamelCase_  =					torch.load(UpperCamelCase_   )
																					else:
																												logger.info(F'''Creating features from dataset file at {data_dir}'''   )

																												lowerCamelCase_  =					(
																												    processor.get_dev_examples(UpperCamelCase_   ) if evaluate else processor.get_train_examples(UpperCamelCase_   )
																												)

																												logger.info("""Training examples: %s"""   ,					len(UpperCamelCase_   )   )
																												lowerCamelCase_  =					hans_convert_examples_to_features(UpperCamelCase_   ,					UpperCamelCase_   ,					UpperCamelCase_   ,					UpperCamelCase_   )
																												logger.info("""Saving features into cached file %s"""   ,					UpperCamelCase_   )
																												torch.save(self.features   ,					UpperCamelCase_   )

							def __len__(	self					:     Tuple   ):
														return len(self.features   )

							def __getitem__(	self					:     List[Any]   ,					__UpperCamelCase					:     Tuple   ):
														return self.features[i]



							def    lowercase__      (	self					:     Optional[int]   ):
														return self.label_list


if is_tf_available():
		import tensorflow as tf





		class       __A:
							SCREAMING_SNAKE_CASE			    =    4_2

							def __init__(	self					:     int   ,					__UpperCamelCase					:     str   ,					__UpperCamelCase					:     PreTrainedTokenizer   ,					__UpperCamelCase					:     str   ,					__UpperCamelCase					:     Optional[int] = 1_2_8   ,					__UpperCamelCase					:     Tuple=False   ,					__UpperCamelCase					:     bool = False   ,					):
														lowerCamelCase_  =					hans_processors[task]()
														lowerCamelCase_  =					processor.get_labels()
														if tokenizer.__class__ in (
														    RobertaTokenizer,
														    RobertaTokenizerFast,
														    XLMRobertaTokenizer,
														    BartTokenizer,
														    BartTokenizerFast,
														):
																					# HACK(label indices are swapped in RoBERTa pretrained model)
																					lowerCamelCase_  =					label_list[2], label_list[1]
														lowerCamelCase_  =					label_list

														lowerCamelCase_  =					processor.get_dev_examples(UpperCamelCase_   ) if evaluate else processor.get_train_examples(UpperCamelCase_   )
														lowerCamelCase_  =					hans_convert_examples_to_features(UpperCamelCase_   ,					UpperCamelCase_   ,					UpperCamelCase_   ,					UpperCamelCase_   )

														def gen():
																					for ex_index, ex in tqdm.tqdm(enumerate(self.features   )   ,					desc="""convert examples to features"""   ):
																												if ex_index % 1_0_0_0_0 == 0:
																																			logger.info("""Writing example %d of %d""" % (ex_index, len(UpperCamelCase_   ))   )

																												yield (
																												    {
																												        "example_id": 0,
																												        "input_ids": ex.input_ids,
																												        "attention_mask": ex.attention_mask,
																												        "token_type_ids": ex.token_type_ids,
																												    },
																												    ex.label,
																												)

														lowerCamelCase_  =					tf.data.Dataset.from_generator(
														    UpperCamelCase_   ,					(
														        {
														            """example_id""": tf.intaa,
														            """input_ids""": tf.intaa,
														            """attention_mask""": tf.intaa,
														            """token_type_ids""": tf.intaa,
														        },
														        tf.intaa,
														    )   ,					(
														        {
														            """example_id""": tf.TensorShape([]   ),
														            """input_ids""": tf.TensorShape([None, None]   ),
														            """attention_mask""": tf.TensorShape([None, None]   ),
														            """token_type_ids""": tf.TensorShape([None, None]   ),
														        },
														        tf.TensorShape([]   ),
														    )   ,					)

							def    lowercase__      (	self					:     Optional[int]   ):
														return self.dataset

							def __len__(	self					:     str   ):
														return len(self.features   )

							def __getitem__(	self					:     Dict   ,					__UpperCamelCase					:     Union[str, Any]   ):
														return self.features[i]



							def    lowercase__      (	self					:     Union[str, Any]   ):
														return self.label_list





class       __A(				UpperCAmelCase       ):

					def    lowercase__      (	self					:     Union[str, Any]   ,					__UpperCamelCase					:     int   ):
												return self._create_examples(self._read_tsv(os.path.join(UpperCamelCase_   ,					"""heuristics_train_set.txt"""   )   )   ,					"""train"""   )

					def    lowercase__      (	self					:     Any   ,					__UpperCamelCase					:     Dict   ):
												return self._create_examples(self._read_tsv(os.path.join(UpperCamelCase_   ,					"""heuristics_evaluation_set.txt"""   )   )   ,					"""dev"""   )

					def    lowercase__      (	self					:     str   ):
												return ["contradiction", "entailment", "neutral"]



					def    lowercase__      (	self					:     List[str]   ,					__UpperCamelCase					:     str   ,					__UpperCamelCase					:     Tuple   ):
												lowerCamelCase_  =					[]
												for i, line in enumerate(UpperCamelCase_   ):
																			if i == 0:
																										continue
																			lowerCamelCase_  =					'%s-%s' % (set_type, line[0])
																			lowerCamelCase_  =					line[5]
																			lowerCamelCase_  =					line[6]
																			lowerCamelCase_  =					line[7][2:] if line[7].startswith("""ex"""   ) else line[7]
																			lowerCamelCase_  =					line[0]
																			examples.append(InputExample(guid=UpperCamelCase_   ,					text_a=UpperCamelCase_   ,					text_b=UpperCamelCase_   ,					label=UpperCamelCase_   ,					pairID=UpperCamelCase_   )   )
												return examples



def      __lowerCAmelCase						(							UpperCAmelCase__				:      Optional[int]			,				UpperCAmelCase__				:      Tuple			,				UpperCAmelCase__				:      str			,				UpperCAmelCase__				:      int			,				)  ->    Any:
							lowerCamelCase_  =					{label: i for i, label in enumerate(_lowercase       )}

							lowerCamelCase_  =					[]
							for ex_index, example in tqdm.tqdm(enumerate(_lowercase       )			,				desc="""convert examples to features"""       ):
														if ex_index % 1_0_0_0_0 == 0:
																					logger.info("""Writing example %d""" % (ex_index)       )

														lowerCamelCase_  =					tokenizer(
														    example.text_a			,				example.text_b			,				add_special_tokens=_lowercase			,				max_length=_lowercase			,				padding="""max_length"""			,				truncation=_lowercase			,				return_overflowing_tokens=_lowercase			,				)

														lowerCamelCase_  =					label_map[example.label] if example.label in label_map else 0

														lowerCamelCase_  =					int(example.pairID       )

														features.append(InputFeatures(**_lowercase			,				label=_lowercase			,				pairID=_lowercase       )       )

							for i, example in enumerate(examples[:5]       ):
														logger.info("""*** Example ***"""       )
														logger.info(F'''guid: {example}'''       )
														logger.info(F'''features: {features[i]}'''       )

							return features


lowercase        =  {
    '''hans''': 3,
}


lowercase        =  {
    '''hans''': HansProcessor,
}







import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
		import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase        =  logging.get_logger(__name__)

lowercase        =  {'''vocab_file''': '''vocab.txt'''}

lowercase        =  {
    '''vocab_file''': {
        '''openbmb/cpm-ant-10b''': '''https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt''',
    },
}

lowercase        =  {
    '''openbmb/cpm-ant-10b''': 1_0_2_4,
}



def      __lowerCAmelCase						(							UpperCAmelCase__				:      List[str]       )  ->    Dict:
							lowerCamelCase_							  =					collections.OrderedDict()
							with open(UpperCAmelCase__			,				"""r"""			,				encoding="""utf-8"""       ) as reader:
														lowerCamelCase_							  =					reader.readlines()
							for index, token in enumerate(UpperCAmelCase__       ):
														lowerCamelCase_							  =					token.rstrip("""\n"""       )
														lowerCamelCase_							  =					index
							return vocab





class       __A(				UpperCAmelCase       ):

					def __init__(	self					:     Optional[Any]   ,					__UpperCamelCase					:     Tuple   ,					__UpperCamelCase					:     List[str]="<unk>"   ,					__UpperCamelCase					:     List[str]=2_0_0   ):
												lowerCamelCase_							  =					vocab
												lowerCamelCase_							  =					unk_token
												lowerCamelCase_							  =					max_input_chars_per_word



					def    lowercase__      (	self					:     int   ,					__UpperCamelCase					:     int   ):
												lowerCamelCase_							  =					list(__UpperCamelCase   )
												if len(__UpperCamelCase   ) > self.max_input_chars_per_word:
																			return [self.unk_token]

												lowerCamelCase_							  =					0
												lowerCamelCase_							  =					[]
												while start < len(__UpperCamelCase   ):
																			lowerCamelCase_							  =					len(__UpperCamelCase   )
																			lowerCamelCase_							  =					None
																			while start < end:
																										lowerCamelCase_							  =					"""""".join(chars[start:end]   )
																										if substr in self.vocab:
																																	lowerCamelCase_							  =					substr
																																	break
																										end -= 1
																			if cur_substr is None:
																										sub_tokens.append(self.unk_token   )
																										start += 1
																			else:
																										sub_tokens.append(__UpperCamelCase   )
																										lowerCamelCase_							  =					end

												return sub_tokens





class       __A(				UpperCAmelCase       ):
					SCREAMING_SNAKE_CASE			    =    VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE			    =    PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE			    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE			    =    ['''input_ids''', '''attention_mask''']
					SCREAMING_SNAKE_CASE			    =    False

					def __init__(	self					:     int   ,					__UpperCamelCase					:     Optional[int]   ,					__UpperCamelCase					:     Optional[int]="<d>"   ,					__UpperCamelCase					:     List[str]="</d>"   ,					__UpperCamelCase					:     Optional[Any]="<s>"   ,					__UpperCamelCase					:     Union[str, Any]="</s>"   ,					__UpperCamelCase					:     List[str]="<pad>"   ,					__UpperCamelCase					:     Union[str, Any]="<unk>"   ,					__UpperCamelCase					:     List[Any]="</n>"   ,					__UpperCamelCase					:     Tuple="</_>"   ,					__UpperCamelCase					:     Optional[Any]="left"   ,					**__UpperCamelCase					:     List[str]   ,					):
												requires_backends(self   ,					["""jieba"""]   )
												super().__init__(
												    bod_token=__UpperCamelCase   ,					eod_token=__UpperCamelCase   ,					bos_token=__UpperCamelCase   ,					eos_token=__UpperCamelCase   ,					pad_token=__UpperCamelCase   ,					unk_token=__UpperCamelCase   ,					line_token=__UpperCamelCase   ,					space_token=__UpperCamelCase   ,					padding_side=__UpperCamelCase   ,					**__UpperCamelCase   ,					)
												lowerCamelCase_							  =					bod_token
												lowerCamelCase_							  =					eod_token
												lowerCamelCase_							  =					load_vocab(__UpperCamelCase   )
												lowerCamelCase_							  =					self.encoder[space_token]
												lowerCamelCase_							  =					self.encoder[line_token]

												del self.encoder[space_token]
												del self.encoder[line_token]

												lowerCamelCase_							  =					collections.OrderedDict(sorted(self.encoder.items()   ,					key=lambda __UpperCamelCase   : x[1]   )   )
												lowerCamelCase_							  =					{v: k for k, v in self.encoder.items()}

												lowerCamelCase_							  =					WordpieceTokenizer(vocab=self.encoder   ,					unk_token=self.unk_token   )

					@property
					def    lowercase__      (	self					:     Any   ):
												return self.encoder[self.bod_token]

					@property
					def    lowercase__      (	self					:     Any   ):
												return self.encoder[self.eod_token]

					@property
					def    lowercase__      (	self					:     int   ):
												return self.encoder["\n"]

					@property
					def    lowercase__      (	self					:     str   ):
												return len(self.encoder   )

					def    lowercase__      (	self					:     Any   ):
												return dict(self.encoder   ,					**self.added_tokens_encoder   )

					def    lowercase__      (	self					:     Any   ,					__UpperCamelCase					:     Optional[Any]   ):
												lowerCamelCase_							  =					[]
												for x in jieba.cut(__UpperCamelCase   ,					cut_all=__UpperCamelCase   ):
																			output_tokens.extend(self.wordpiece_tokenizer.tokenize(__UpperCamelCase   )   )
												return output_tokens

					def    lowercase__      (	self					:     Optional[Any]   ,					__UpperCamelCase					:     Dict   ,					**__UpperCamelCase					:     str   ):
												lowerCamelCase_							  =					[i for i in token_ids if i >= 0]
												lowerCamelCase_							  =					[
												    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
												]
												return super()._decode(__UpperCamelCase   ,					**__UpperCamelCase   )

					def    lowercase__      (	self					:     Dict   ,					__UpperCamelCase					:     Optional[Any]   ):
												return token in self.encoder

					def    lowercase__      (	self					:     int   ,					__UpperCamelCase					:     List[str]   ):
												return "".join(__UpperCamelCase   )

					def    lowercase__      (	self					:     List[str]   ,					__UpperCamelCase					:     int   ):
												return self.encoder.get(__UpperCamelCase   ,					self.encoder.get(self.unk_token   )   )

					def    lowercase__      (	self					:     Any   ,					__UpperCamelCase					:     Union[str, Any]   ):
												return self.decoder.get(__UpperCamelCase   ,					self.unk_token   )

					def    lowercase__      (	self					:     Union[str, Any]   ,					__UpperCamelCase					:     str   ,					__UpperCamelCase					:     Optional[str] = None   ):
												if os.path.isdir(__UpperCamelCase   ):
																			lowerCamelCase_							  =					os.path.join(
																			    __UpperCamelCase   ,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]   )
												else:
																			lowerCamelCase_							  =					(filename_prefix + """-""" if filename_prefix else """""") + save_directory
												lowerCamelCase_							  =					0
												if " " in self.encoder:
																			lowerCamelCase_							  =					self.encoder[""" """]
																			del self.encoder[" "]
												if "\n" in self.encoder:
																			lowerCamelCase_							  =					self.encoder["""\n"""]
																			del self.encoder["\n"]
												lowerCamelCase_							  =					collections.OrderedDict(sorted(self.encoder.items()   ,					key=lambda __UpperCamelCase   : x[1]   )   )
												with open(__UpperCamelCase   ,					"""w"""   ,					encoding="""utf-8"""   ) as writer:
																			for token, token_index in self.encoder.items():
																										if index != token_index:
																																	logger.warning(
																																	    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																																	    """ Please check that the vocabulary is not corrupted!"""   )
																																	lowerCamelCase_							  =					token_index
																										writer.write(token + """\n"""   )
																										index += 1
												return (vocab_file,)

					def    lowercase__      (	self					:     Any   ,					__UpperCamelCase					:     List[int]   ,					__UpperCamelCase					:     List[int] = None   ):
												if token_ids_a is None:
																			return [self.bos_token_id] + token_ids_a
												return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a



					def    lowercase__      (	self					:     int   ,					__UpperCamelCase					:     List[int]   ,					__UpperCamelCase					:     Optional[List[int]] = None   ,					__UpperCamelCase					:     bool = False   ):

												if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=__UpperCamelCase   ,					token_ids_a=__UpperCamelCase   ,					already_has_special_tokens=__UpperCamelCase   )

												if token_ids_a is not None:
																			return [1] + ([0] * len(__UpperCamelCase   )) + [1] + ([0] * len(__UpperCamelCase   ))
												return [1] + ([0] * len(__UpperCamelCase   ))





