


import argparse
import json
from typing import List

from ltp import LTP

from transformers import BertTokenizer






def 	_A				(lowerCAmelCase__  :Union[str, Any]		)   ->  Optional[Any]:




     '''simple docstring'''



     if (
         (cp >= 0x4_e_0_0 and cp <= 0x9_f_f_f)
         or (cp >= 0x3_4_0_0 and cp <= 0x4_d_b_f)  #
         or (cp >= 0x2_0_0_0_0 and cp <= 0x2_a_6_d_f)  #
         or (cp >= 0x2_a_7_0_0 and cp <= 0x2_b_7_3_f)  #
         or (cp >= 0x2_b_7_4_0 and cp <= 0x2_b_8_1_f)  #
         or (cp >= 0x2_b_8_2_0 and cp <= 0x2_c_e_a_f)  #
         or (cp >= 0xf_9_0_0 and cp <= 0xf_a_f_f)
         or (cp >= 0x2_f_8_0_0 and cp <= 0x2_f_a_1_f)  #
     ):  #
          return True

     return False






def 	_A				(lowerCAmelCase__  :str		)   ->  Optional[int]:




     '''simple docstring'''



     for char in word:
          _a			      =							ord(lowerCAmelCase__		)
          if not _is_chinese_char(lowerCAmelCase__		):
               return 0
     return 1






def 	_A				(lowerCAmelCase__  :List[str]		)   ->  Any:




     '''simple docstring'''



     _a			      =							set()

     for token in tokens:
          _a			      =							len(lowerCAmelCase__		) > 1 and is_chinese(lowerCAmelCase__		)
          if chinese_word:
               word_set.add(lowerCAmelCase__		)
     _a			      =							list(lowerCAmelCase__		)
     return word_list






def 	_A				(lowerCAmelCase__  :List[str]				,						lowerCAmelCase__  :set()		)   ->  int:




     '''simple docstring'''



     if not chinese_word_set:
          return bert_tokens
     _a			      =							max([len(lowerCAmelCase__		) for w in chinese_word_set]		)

     _a			      =							bert_tokens
     _a      ,   _a			      =							0, len(lowerCAmelCase__		)
     while start < end:
          _a			      =							True
          if is_chinese(bert_word[start]		):
               _a			      =							min(end - start				,						lowerCAmelCase__		)
               for i in range(lowerCAmelCase__				,						1				,						-1		):
                    _a			      =							''.join(bert_word[start : start + i]		)
                    if whole_word in chinese_word_set:
                         for j in range(start + 1				,						start + i		):
                              _a			      =							'##' + bert_word[j]
                         _a			      =							start + i
                         _a			      =							False
                         break
          if single_word:
               start += 1
     return bert_word






def 	_A				(lowerCAmelCase__  :List[str]				,						lowerCAmelCase__  :LTP				,						lowerCAmelCase__  :BertTokenizer		)   ->  List[str]:




     '''simple docstring'''



     _a			      =							[]

     for i in range(0				,						len(lowerCAmelCase__		)				,						1_00		):
          _a			      =							ltp_tokenizer.seg(lines[i : i + 1_00]		)[0]
          _a			      =							[get_chinese_word(lowerCAmelCase__		) for r in res]
          ltp_res.extend(lowerCAmelCase__		)
     assert len(lowerCAmelCase__		) == len(lowerCAmelCase__		)

     _a			      =							[]
     for i in range(0				,						len(lowerCAmelCase__		)				,						1_00		):
          _a			      =							bert_tokenizer(lines[i : i + 1_00]				,						add_special_tokens=lowerCAmelCase__				,						truncation=lowerCAmelCase__				,						max_length=5_12		)
          bert_res.extend(res['input_ids']		)
     assert len(lowerCAmelCase__		) == len(lowerCAmelCase__		)

     _a			      =							[]
     for input_ids, chinese_word in zip(lowerCAmelCase__				,						lowerCAmelCase__		):
          _a			      =							[]
          for id in input_ids:
               _a			      =							bert_tokenizer._convert_id_to_token(lowerCAmelCase__		)
               input_tokens.append(lowerCAmelCase__		)
          _a			      =							add_sub_symbol(lowerCAmelCase__				,						lowerCAmelCase__		)
          _a			      =							[]
          # We only save pos of chinese subwords start with ##, which mean is part of a whole word.
          for i, token in enumerate(lowerCAmelCase__		):
               if token[:2] == "##":
                    _a			      =							token[2:]
                    # save chinese tokens' pos
                    if len(lowerCAmelCase__		) == 1 and _is_chinese_char(ord(lowerCAmelCase__		)		):
                         ref_id.append(lowerCAmelCase__		)
          ref_ids.append(lowerCAmelCase__		)

     assert len(lowerCAmelCase__		) == len(lowerCAmelCase__		)

     return ref_ids






def 	_A				(lowerCAmelCase__  :Union[str, Any]		)   ->  Optional[Any]:




     '''simple docstring'''



     with open(args.file_name				,						'r'				,						encoding='utf-8'		) as f:
          _a			      =							f.readlines()
     _a			      =							[line.strip() for line in data if len(lowerCAmelCase__		) > 0 and not line.isspace()]  # avoid delimiter like '\u2029'
     _a			      =							LTP(args.ltp		)  # faster in GPU device
     _a			      =							BertTokenizer.from_pretrained(args.bert		)

     _a			      =							prepare_ref(lowerCAmelCase__				,						lowerCAmelCase__				,						lowerCAmelCase__		)

     with open(args.save_path				,						'w'				,						encoding='utf-8'		) as f:
          _a			      =							[json.dumps(lowerCAmelCase__		) + '\n' for ref in ref_ids]
          f.writelines(lowerCAmelCase__		)


if __name__ == "__main__":
 a_					:    Dict          =    argparse.ArgumentParser(description="prepare_chinese_ref")
 parser.add_argument(
     "--file_name",
     type=str,
     default="./resources/chinese-demo.txt",
     help="file need process, same as training data in lm",
 )
 parser.add_argument(
     "--ltp", type=str, default="./resources/ltp", help="resources for LTP tokenizer, usually a path"
 )
 parser.add_argument("--bert", type=str, default="./resources/robert", help="resources for Bert tokenizer")
 parser.add_argument("--save_path", type=str, default="./resources/ref.txt", help="path to save res")

 a_					:    str          =    parser.parse_args()
 main(args)





'''simple docstring'''






import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


a_					:    Optional[Any]          =    logging.getLogger(__name__)




@dataclass(frozen=_SCREAMING_SNAKE_CASE       )
class  a     :
 _lowerCAmelCase 					=	42
 _lowerCAmelCase 					=	42
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None





@dataclass(frozen=_SCREAMING_SNAKE_CASE       )
class  a     :
 _lowerCAmelCase 					=	42
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None


if is_torch_available():
 import torch
 from torch.utils.data import Dataset




 class  a     (     _SCREAMING_SNAKE_CASE       ):
  _lowerCAmelCase 					=	42
  def __init__( self      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__ = None      ,				__magic_name__=False      ,				__magic_name__ = False      ,				)					->							Optional[int]:
       _a			      =							hans_processors[task]()

       _a			      =							os.path.join(
           __magic_name__      ,				'cached_{}_{}_{}_{}'.format(
               'dev' if evaluate else 'train'      ,				tokenizer.__class__.__name__      ,				str(__magic_name__						)      ,				__magic_name__      ,				)      ,				)
       _a			      =							processor.get_labels()
       if tokenizer.__class__ in (
           RobertaTokenizer,
           RobertaTokenizerFast,
           XLMRobertaTokenizer,
           BartTokenizer,
           BartTokenizerFast,
       ):
            # HACK(label indices are swapped in RoBERTa pretrained model)
            _a      ,   _a			      =							label_list[2], label_list[1]
       _a			      =							label_list

       # Make sure only the first process in distributed training processes the dataset,
       # and the others will use the cache.
       _a			      =							cached_features_file + '.lock'
       with FileLock(__magic_name__						):
            if os.path.exists(__magic_name__						) and not overwrite_cache:
                 logger.info(f'Loading features from cached file {cached_features_file}'						)
                 _a			      =							torch.load(__magic_name__						)
            else:
                 logger.info(f'Creating features from dataset file at {data_dir}'						)

                 _a			      =							(
                     processor.get_dev_examples(__magic_name__						) if evaluate else processor.get_train_examples(__magic_name__						)
                 )

                 logger.info('Training examples: %s'      ,				len(__magic_name__						)						)
                 _a			      =							hans_convert_examples_to_features(__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__						)
                 logger.info('Saving features into cached file %s'      ,				__magic_name__						)
                 torch.save(self.features      ,				__magic_name__						)
  def __len__( self						)					->							List[Any]:
       return len(self.features						)
  def __getitem__( self      ,				__magic_name__						)					->							InputFeatures:
       return self.features[i]




  def 							__UpperCAmelCase		( self						)					->							Optional[Any]:
       return self.label_list


if is_tf_available():
 import tensorflow as tf




 class  a     :
  _lowerCAmelCase 					=	42
  def __init__( self      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__ = 1_28      ,				__magic_name__=False      ,				__magic_name__ = False      ,				)					->							str:
       _a			      =							hans_processors[task]()
       _a			      =							processor.get_labels()
       if tokenizer.__class__ in (
           RobertaTokenizer,
           RobertaTokenizerFast,
           XLMRobertaTokenizer,
           BartTokenizer,
           BartTokenizerFast,
       ):
            # HACK(label indices are swapped in RoBERTa pretrained model)
            _a      ,   _a			      =							label_list[2], label_list[1]
       _a			      =							label_list

       _a			      =							processor.get_dev_examples(__magic_name__						) if evaluate else processor.get_train_examples(__magic_name__						)
       _a			      =							hans_convert_examples_to_features(__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__						)

       def gen():
            for ex_index, ex in tqdm.tqdm(enumerate(self.features						)      ,				desc='convert examples to features'						):
                 if ex_index % 1_00_00 == 0:
                      logger.info('Writing example %d of %d' % (ex_index, len(__magic_name__						))						)

                 yield (
                     {
                         "example_id": 0,
                         "input_ids": ex.input_ids,
                         "attention_mask": ex.attention_mask,
                         "token_type_ids": ex.token_type_ids,
                     },
                     ex.label,
                 )

       _a			      =							tf.data.Dataset.from_generator(
           __magic_name__      ,				(
               {
                   'example_id': tf.intaa,
                   'input_ids': tf.intaa,
                   'attention_mask': tf.intaa,
                   'token_type_ids': tf.intaa,
               },
               tf.intaa,
           )      ,				(
               {
                   'example_id': tf.TensorShape([]						),
                   'input_ids': tf.TensorShape([None, None]						),
                   'attention_mask': tf.TensorShape([None, None]						),
                   'token_type_ids': tf.TensorShape([None, None]						),
               },
               tf.TensorShape([]						),
           )      ,				)
  def 							__UpperCAmelCase		( self						)					->							Tuple:
       return self.dataset
  def __len__( self						)					->							Optional[Any]:
       return len(self.features						)
  def __getitem__( self      ,				__magic_name__						)					->							InputFeatures:
       return self.features[i]




  def 							__UpperCAmelCase		( self						)					->							List[Any]:
       return self.label_list




class  a     (     _SCREAMING_SNAKE_CASE       ):
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							Tuple:
      return self._create_examples(self._read_tsv(os.path.join(__magic_name__      ,				'heuristics_train_set.txt'						)						)      ,				'train'						)
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							List[Any]:
      return self._create_examples(self._read_tsv(os.path.join(__magic_name__      ,				'heuristics_evaluation_set.txt'						)						)      ,				'dev'						)
 def 							__UpperCAmelCase		( self						)					->							Tuple:
      return ["contradiction", "entailment", "neutral"]




 def 							__UpperCAmelCase		( self      ,				__magic_name__      ,				__magic_name__						)					->							Optional[Any]:
      _a			      =							[]
      for i, line in enumerate(__magic_name__						):
           if i == 0:
                continue
           _a			      =							'%s-%s' % (set_type, line[0])
           _a			      =							line[5]
           _a			      =							line[6]
           _a			      =							line[7][2:] if line[7].startswith('ex'						) else line[7]
           _a			      =							line[0]
           examples.append(InputExample(guid=__magic_name__      ,				text_a=__magic_name__      ,				text_b=__magic_name__      ,				label=__magic_name__      ,				pairID=__magic_name__						)						)
      return examples






def 	_A				(lowerCAmelCase__  :List[InputExample]				,						lowerCAmelCase__  :List[str]				,						lowerCAmelCase__  :int				,						lowerCAmelCase__  :PreTrainedTokenizer				,						)   ->  Tuple:




     '''simple docstring'''



     _a			      =							{label: i for i, label in enumerate(lowerCAmelCase__		)}

     _a			      =							[]
     for ex_index, example in tqdm.tqdm(enumerate(lowerCAmelCase__		)				,						desc='convert examples to features'		):
          if ex_index % 1_00_00 == 0:
               logger.info('Writing example %d' % (ex_index)		)

          _a			      =							tokenizer(
              example.text_a				,						example.text_b				,						add_special_tokens=lowerCAmelCase__				,						max_length=lowerCAmelCase__				,						padding='max_length'				,						truncation=lowerCAmelCase__				,						return_overflowing_tokens=lowerCAmelCase__				,						)

          _a			      =							label_map[example.label] if example.label in label_map else 0

          _a			      =							int(example.pairID		)

          features.append(InputFeatures(**lowerCAmelCase__				,						label=lowerCAmelCase__				,						pairID=lowerCAmelCase__		)		)

     for i, example in enumerate(examples[:5]		):
          logger.info('*** Example ***'		)
          logger.info(f'guid: {example}'		)
          logger.info(f'features: {features[i]}'		)

     return features


a_					:    Optional[int]          =    {
    "hans": 3,
}






a_					:    Optional[Any]          =    {
    "hans": HansProcessor,
}

