






import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


__A  			=					logging.getLogger(__name__)





@dataclass(frozen=snake_case       )
class     SCREAMING_SNAKE_CASE					:





							"""simple docstring"""

							A_    	=       42
							A_    	=       42
							A_    	=       None
							A_    	=       None
							A_    	=       None





@dataclass(frozen=snake_case       )
class     SCREAMING_SNAKE_CASE					:





							"""simple docstring"""

							A_    	=       42
							A_    	=       None
							A_    	=       None
							A_    	=       None
							A_    	=       None


if is_torch_available():
				import torch
				from torch.utils.data import Dataset





				class     SCREAMING_SNAKE_CASE					(			snake_case       ):





											"""simple docstring"""

											A_    	=       42

											def __init__(	self: Optional[Any]		,  __A: int		,  __A: Any		,  __A: Dict		,  __A: int = None		,  __A: int=False		,  __A: List[str] = False		,  )     ->					Any:
																_A										=			hans_processors[task]()

																_A										=			os.path.join(
																    a_		,  '''cached_{}_{}_{}_{}'''.format(
																        '''dev''' if evaluate else '''train'''		,  tokenizer.__class__.__name__		,  str(a_ )		,  a_		,  )		,  )
																_A										=			processor.get_labels()
																if tokenizer.__class__ in (
																    RobertaTokenizer,
																    RobertaTokenizerFast,
																    XLMRobertaTokenizer,
																    BartTokenizer,
																    BartTokenizerFast,
																):
																					# HACK(label indices are swapped in RoBERTa pretrained model)
																					_A				,_A										=			label_list[2], label_list[1]
																_A										=			label_list

																# Make sure only the first process in distributed training processes the dataset,
																# and the others will use the cache.
																_A										=			cached_features_file + '''.lock'''
																with FileLock(a_ ):
																					if os.path.exists(a_ ) and not overwrite_cache:
																										logger.info(f"""Loading features from cached file {cached_features_file}""" )
																										_A										=			torch.load(a_ )
																					else:
																										logger.info(f"""Creating features from dataset file at {data_dir}""" )

																										_A										=			(
																										    processor.get_dev_examples(a_ ) if evaluate else processor.get_train_examples(a_ )
																										)

																										logger.info('''Training examples: %s'''		,  len(a_ ) )
																										_A										=			hans_convert_examples_to_features(a_		,  a_		,  a_		,  a_ )
																										logger.info('''Saving features into cached file %s'''		,  a_ )
																										torch.save(self.features		,  a_ )

											def __len__(	self: List[Any] )     ->					Any:
																return len(self.features )

											def __getitem__(	self: Union[str, Any]		,  __A: List[str] )     ->					InputFeatures:
																return self.features[i]



											def    __A			(	self: Tuple )     ->					Tuple:
																return self.label_list


if is_tf_available():
				import tensorflow as tf





				class     SCREAMING_SNAKE_CASE					:





											"""simple docstring"""

											A_    	=       42

											def __init__(	self: str		,  __A: List[str]		,  __A: Optional[Any]		,  __A: List[str]		,  __A: List[Any] = 1_28		,  __A: List[str]=False		,  __A: Optional[int] = False		,  )     ->					Optional[Any]:
																_A										=			hans_processors[task]()
																_A										=			processor.get_labels()
																if tokenizer.__class__ in (
																    RobertaTokenizer,
																    RobertaTokenizerFast,
																    XLMRobertaTokenizer,
																    BartTokenizer,
																    BartTokenizerFast,
																):
																					# HACK(label indices are swapped in RoBERTa pretrained model)
																					_A				,_A										=			label_list[2], label_list[1]
																_A										=			label_list

																_A										=			processor.get_dev_examples(a_ ) if evaluate else processor.get_train_examples(a_ )
																_A										=			hans_convert_examples_to_features(a_		,  a_		,  a_		,  a_ )

																def gen():
																					for ex_index, ex in tqdm.tqdm(enumerate(self.features )		,  desc='''convert examples to features''' ):
																										if ex_index % 1_00_00 == 0:
																															logger.info('''Writing example %d of %d''' % (ex_index, len(a_ )) )

																										yield (
																										    {
																										        "example_id": 0,
																										        "input_ids": ex.input_ids,
																										        "attention_mask": ex.attention_mask,
																										        "token_type_ids": ex.token_type_ids,
																										    },
																										    ex.label,
																										)

																_A										=			tf.data.Dataset.from_generator(
																    a_		,  (
																        {
																            '''example_id''': tf.intaa,
																            '''input_ids''': tf.intaa,
																            '''attention_mask''': tf.intaa,
																            '''token_type_ids''': tf.intaa,
																        },
																        tf.intaa,
																    )		,  (
																        {
																            '''example_id''': tf.TensorShape([] ),
																            '''input_ids''': tf.TensorShape([None, None] ),
																            '''attention_mask''': tf.TensorShape([None, None] ),
																            '''token_type_ids''': tf.TensorShape([None, None] ),
																        },
																        tf.TensorShape([] ),
																    )		,  )

											def    __A			(	self: List[Any] )     ->					Optional[Any]:
																return self.dataset

											def __len__(	self: str )     ->					Tuple:
																return len(self.features )

											def __getitem__(	self: List[str]		,  __A: Optional[Any] )     ->					InputFeatures:
																return self.features[i]



											def    __A			(	self: List[Any] )     ->					Dict:
																return self.label_list





class     SCREAMING_SNAKE_CASE					(			snake_case       ):





							"""simple docstring"""

							def    __A			(	self: Union[str, Any]		,  __A: int )     ->					Dict:
												return self._create_examples(self._read_tsv(os.path.join(a_		,  '''heuristics_train_set.txt''' ) )		,  '''train''' )

							def    __A			(	self: Dict		,  __A: int )     ->					str:
												return self._create_examples(self._read_tsv(os.path.join(a_		,  '''heuristics_evaluation_set.txt''' ) )		,  '''dev''' )

							def    __A			(	self: List[Any] )     ->					Tuple:
												return ["contradiction", "entailment", "neutral"]



							def    __A			(	self: Tuple		,  __A: List[str]		,  __A: Tuple )     ->					Dict:
												_A										=			[]
												for i, line in enumerate(a_ ):
																	if i == 0:
																						continue
																	_A										=			'''%s-%s''' % (set_type, line[0])
																	_A										=			line[5]
																	_A										=			line[6]
																	_A										=			line[7][2:] if line[7].startswith('''ex''' ) else line[7]
																	_A										=			line[0]
																	examples.append(InputExample(guid=a_		,  text_a=a_		,  text_b=a_		,  label=a_		,  pairID=a_ ) )
												return examples




def   __A       (  _lowercase       ,      _lowercase       ,      _lowercase       ,      _lowercase       ,      ):

					'''simple docstring'''







					_A										=			{label: i for i, label in enumerate(_lowercase			)}

					_A										=			[]
					for ex_index, example in tqdm.tqdm(enumerate(_lowercase			)       ,      desc='''convert examples to features'''			):
										if ex_index % 1_00_00 == 0:
															logger.info('''Writing example %d''' % (ex_index)			)

										_A										=			tokenizer(
										    example.text_a       ,      example.text_b       ,      add_special_tokens=_lowercase       ,      max_length=_lowercase       ,      padding='''max_length'''       ,      truncation=_lowercase       ,      return_overflowing_tokens=_lowercase       ,      )

										_A										=			label_map[example.label] if example.label in label_map else 0

										_A										=			int(example.pairID			)

										features.append(InputFeatures(**_lowercase       ,      label=_lowercase       ,      pairID=_lowercase			)			)

					for i, example in enumerate(examples[:5]			):
										logger.info('''*** Example ***'''			)
										logger.info(f"""guid: {example}"""			)
										logger.info(f"""features: {features[i]}"""			)

					return features


__A  			=					{
    'hans': 3,
}


__A  			=					{
    'hans': HansProcessor,
}






'''simple docstring'''




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_a					:			Tuple							=    logging.get_logger(__name__)

_a					:			Optional[int]							=    '▁'

_a					:			str							=    {'vocab_file': 'sentencepiece.bpe.model', 'monolingual_vocab_file': 'dict.txt'}

_a					:			Optional[Any]							=    {
    'vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model',
    },
    'monolingual_vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt',
    },
}

_a					:			Union[str, Any]							=    {'vinai/bartpho-syllable': 1_024}



class 							lowercase_						(		a			):

  '''simple docstring'''





  __lowerCAmelCase       :			Dict     	= VOCAB_FILES_NAMES
  __lowerCAmelCase       :			str     	= PRETRAINED_VOCAB_FILES_MAP
  __lowerCAmelCase       :			Any     	= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowerCAmelCase       :			int     	= ["input_ids", "attention_mask"]
  def __init__(  self ,			a_ ,			a_ ,			a_="<s>" ,			a_="</s>" ,			a_="</s>" ,			a_="<s>" ,			a_="<unk>" ,			a_="<pad>" ,			a_="<mask>" ,			a_ = None ,			**a_ ,			)				->					None:


    """simple docstring"""
    # Mask token behave like a normal word, i.e. include the space before it
    UpperCAmelCase    				=						AddedToken(a_ ,			lstrip=a_ ,			rstrip=a_  ) if isinstance(a_ ,			a_  ) else mask_token

    UpperCAmelCase    				=						{} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        bos_token=a_ ,			eos_token=a_ ,			unk_token=a_ ,			sep_token=a_ ,			cls_token=a_ ,			pad_token=a_ ,			mask_token=a_ ,			sp_model_kwargs=self.sp_model_kwargs ,			**a_ ,			)

    UpperCAmelCase    				=						vocab_file
    UpperCAmelCase    				=						monolingual_vocab_file
    UpperCAmelCase    				=						spm.SentencePieceProcessor(**self.sp_model_kwargs  )
    self.sp_model.Load(str(a_  )  )

    # Load the reduced vocab

    # Keep order of special tokens for backward compatibility
    UpperCAmelCase    				=						{}
    UpperCAmelCase    				=						0
    for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
      if str(a_  ) not in self.fairseq_tokens_to_ids:
        UpperCAmelCase    				=						cnt
        cnt += 1
    with open(a_ ,			'r' ,			encoding='utf-8'  ) as f:
      for line in f.readlines():
        UpperCAmelCase    				=						line.strip().split()[0]
        UpperCAmelCase    				=						len(self.fairseq_tokens_to_ids  )
    if str(a_  ) not in self.fairseq_tokens_to_ids:
      UpperCAmelCase    				=						len(self.fairseq_tokens_to_ids  )

    UpperCAmelCase    				=						{v: k for k, v in self.fairseq_tokens_to_ids.items()}
  def __getstate__(  self  )				->					Any:


    """simple docstring"""
    UpperCAmelCase    				=						self.__dict__.copy()
    UpperCAmelCase    				=						None
    UpperCAmelCase    				=						self.sp_model.serialized_model_proto()
    return state
  def __setstate__(  self ,			a_  )				->					int:


    """simple docstring"""
    UpperCAmelCase    				=						d

    # for backward compatibility
    if not hasattr(self ,			'sp_model_kwargs'  ):
      UpperCAmelCase    				=						{}

    UpperCAmelCase    				=						spm.SentencePieceProcessor(**self.sp_model_kwargs  )
    self.sp_model.LoadFromSerializedProto(self.sp_model_proto  )
  def     snake_case_      (  self ,			a_ ,			a_ = None  )				->					List[int]:


    """simple docstring"""

    if token_ids_a is None:
      return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    UpperCAmelCase    				=						[self.cls_token_id]
    UpperCAmelCase    				=						[self.sep_token_id]
    return cls + token_ids_a + sep + sep + token_ids_a + sep
  def     snake_case_      (  self ,			a_ ,			a_ = None ,			a_ = False  )				->					List[int]:


    """simple docstring"""

    if already_has_special_tokens:
      return super().get_special_tokens_mask(
          token_ids_a=a_ ,			token_ids_a=a_ ,			already_has_special_tokens=a_  )

    if token_ids_a is None:
      return [1] + ([0] * len(a_  )) + [1]
    return [1] + ([0] * len(a_  )) + [1, 1] + ([0] * len(a_  )) + [1]
  def     snake_case_      (  self ,			a_ ,			a_ = None  )				->					List[int]:


    """simple docstring"""
    UpperCAmelCase    				=						[self.sep_token_id]
    UpperCAmelCase    				=						[self.cls_token_id]

    if token_ids_a is None:
      return len(cls + token_ids_a + sep  ) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]
  @property
  def     snake_case_      (  self  )				->					Union[str, Any]:


    """simple docstring"""
    return len(self.fairseq_ids_to_tokens  )
  def     snake_case_      (  self  )				->					int:


    """simple docstring"""
    UpperCAmelCase    				=						{self.convert_ids_to_tokens(a_  ): i for i in range(self.vocab_size  )}
    vocab.update(self.added_tokens_encoder  )
    return vocab
  def     snake_case_      (  self ,			a_  )				->					List[str]:


    """simple docstring"""
    return self.sp_model.encode(a_ ,			out_type=a_  )
  def     snake_case_      (  self ,			a_  )				->					Any:


    """simple docstring"""
    if token in self.fairseq_tokens_to_ids:
      return self.fairseq_tokens_to_ids[token]
    else:
      return self.unk_token_id
  def     snake_case_      (  self ,			a_  )				->					Dict:


    """simple docstring"""
    return self.fairseq_ids_to_tokens[index]
  def     snake_case_      (  self ,			a_  )				->					Union[str, Any]:


    """simple docstring"""
    UpperCAmelCase    				=						''.join(a_  ).replace(a_ ,			' '  ).strip()
    return out_string





  def     snake_case_      (  self ,			a_ ,			a_ = None  )				->					Tuple[str]:


    """simple docstring"""
    if not os.path.isdir(a_  ):
      logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
      return
    UpperCAmelCase    				=						os.path.join(
        a_ ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']  )
    UpperCAmelCase    				=						os.path.join(
        a_ ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['monolingual_vocab_file'] ,			)

    if os.path.abspath(self.vocab_file  ) != os.path.abspath(a_  ) and os.path.isfile(self.vocab_file  ):
      copyfile(self.vocab_file ,			a_  )
    elif not os.path.isfile(self.vocab_file  ):
      with open(a_ ,			'wb'  ) as fi:
        UpperCAmelCase    				=						self.sp_model.serialized_model_proto()
        fi.write(a_  )

    if os.path.abspath(self.monolingual_vocab_file  ) != os.path.abspath(
        a_  ) and os.path.isfile(self.monolingual_vocab_file  ):
      copyfile(self.monolingual_vocab_file ,			a_  )
    elif not os.path.isfile(self.monolingual_vocab_file  ):
      with open(a_ ,			'w' ,			encoding='utf-8'  ) as fp:
        for token in self.fairseq_tokens_to_ids:
          if token not in self.all_special_tokens:
            fp.write(F'''{str(a_  )} \n'''  )

    return out_vocab_file, out_monolingual_vocab_file


