
"""simple docstring"""






import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule





class 				a__					(						A__       ):
     UpperCAmelCase__         =      42
     UpperCAmelCase__         =      jnp.floataa
     UpperCAmelCase__         =      True



     def  lowerCamelCase_						(							self :List[Any]   ):







      '''simple docstring'''



      super().setup()
      UpperCamelCase_     :					Optional[Any]   	=nn.Dense(5      ,  dtype=self.dtype   )




     def __call__(							self :List[Any]      ,  *_lowerCamelCase :Dict      ,  **_lowerCamelCase :Optional[int]   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=super().__call__(*_lowerCamelCase      ,  **_lowerCamelCase   )
      UpperCamelCase_     :					int   	=self.cls(outputs[2]   )
      return outputs[:2] + (cls_out,)





class 				a__					(						A__       ):
     UpperCAmelCase__         =      FlaxBigBirdForNaturalQuestionsModule




def       A_  (				__lowercase						,	__lowercase						,	__lowercase						,	__lowercase						,	__lowercase						,	__lowercase	):
 def cross_entropy(__lowercase						,	__lowercase						,	__lowercase=None	):
  UpperCamelCase_     :					Dict   	=logits.shape[-1]
  UpperCamelCase_     :					Optional[Any]   	=(labels[..., None] == jnp.arange(__lowercase	)[None]).astype('f4'	)
  UpperCamelCase_     :					Union[str, Any]   	=jax.nn.log_softmax(__lowercase						,	axis=-1	)
  UpperCamelCase_     :					Tuple   	=-jnp.sum(labels * logits						,	axis=-1	)
  if reduction is not None:
   UpperCamelCase_     :					str   	=reduction(__lowercase	)
  return loss

 UpperCamelCase_     :					int   	=partial(__lowercase						,	reduction=jnp.mean	)
 UpperCamelCase_     :					List[str]   	=cross_entropy(__lowercase						,	__lowercase	)
 UpperCamelCase_     :					Dict   	=cross_entropy(__lowercase						,	__lowercase	)
 UpperCamelCase_     :					int   	=cross_entropy(__lowercase						,	__lowercase	)
 return (start_loss + end_loss + pooled_loss) / 3





@dataclass
class 				a__					:
     UpperCAmelCase__         =      "google/bigbird-roberta-base"
     UpperCAmelCase__         =      30_00
     UpperCAmelCase__         =      1_05_00

     UpperCAmelCase__         =      1_28
     UpperCAmelCase__         =      3

     UpperCAmelCase__         =      1
     UpperCAmelCase__         =      5

     # tx_args
     UpperCAmelCase__         =      3E-5
     UpperCAmelCase__         =      0.0
     UpperCAmelCase__         =      2_00_00
     UpperCAmelCase__         =      0.00_95

     UpperCAmelCase__         =      "bigbird-roberta-natural-questions"
     UpperCAmelCase__         =      "training-expt"
     UpperCAmelCase__         =      "data/nq-training.jsonl"
     UpperCAmelCase__         =      "data/nq-validation.jsonl"



     def  lowerCamelCase_						(							self :Dict   ):







      '''simple docstring'''



      os.makedirs(self.base_dir      ,  exist_ok=_lowerCamelCase   )
      UpperCamelCase_     :					int   	=os.path.join(self.base_dir      ,  self.save_dir   )
      UpperCamelCase_     :					Any   	=self.batch_size_per_device * jax.device_count()





@dataclass
class 				a__					:
     UpperCAmelCase__         =      42
     UpperCAmelCase__         =      40_96  # no dynamic padding on TPUs



     def __call__(							self :Union[str, Any]      ,  _lowerCamelCase :List[str]   ):







      '''simple docstring'''



      UpperCamelCase_     :					int   	=self.collate_fn(_lowerCamelCase   )
      UpperCamelCase_     :					List[str]   	=jax.tree_util.tree_map(_lowerCamelCase      ,  _lowerCamelCase   )
      return batch



     def  lowerCamelCase_						(							self :Union[str, Any]      ,  _lowerCamelCase :Tuple   ):







      '''simple docstring'''



      UpperCamelCase_	,     UpperCamelCase_     :					Any   	=self.fetch_inputs(features['input_ids']   )
      UpperCamelCase_     :					Union[str, Any]   	={
          'input_ids': jnp.array(_lowerCamelCase      ,  dtype=jnp.intaa   ),
          'attention_mask': jnp.array(_lowerCamelCase      ,  dtype=jnp.intaa   ),
          'start_labels': jnp.array(features['start_token']      ,  dtype=jnp.intaa   ),
          'end_labels': jnp.array(features['end_token']      ,  dtype=jnp.intaa   ),
          'pooled_labels': jnp.array(features['category']      ,  dtype=jnp.intaa   ),
      }
      return batch



     def  lowerCamelCase_						(							self :List[Any]      ,  _lowerCamelCase :list   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=[self._fetch_inputs(_lowerCamelCase   ) for ids in input_ids]
      return zip(*_lowerCamelCase   )




     def  lowerCamelCase_						(							self :Optional[Any]      ,  _lowerCamelCase :list   ):







      '''simple docstring'''



      UpperCamelCase_     :					Tuple   	=[1 for _ in range(len(_lowerCamelCase   )   )]
      while len(_lowerCamelCase   ) < self.max_length:
       input_ids.append(self.pad_id   )
       attention_mask.append(0   )
      return input_ids, attention_mask




def       A_  (				__lowercase						,	__lowercase						,	__lowercase=None	):
 if seed is not None:
  UpperCamelCase_     :					List[str]   	=dataset.shuffle(seed=__lowercase	)
 for i in range(len(__lowercase	) // batch_size	):
  UpperCamelCase_     :					str   	=dataset[i * batch_size : (i + 1) * batch_size]
  yield dict(__lowercase	)




@partial(jax.pmap						,	axis_name='batch'	)
def       A_  (				__lowercase						,	__lowercase						,	**__lowercase	):
 def loss_fn(__lowercase	):
  UpperCamelCase_     :					Dict   	=model_inputs.pop('start_labels'	)
  UpperCamelCase_     :					Dict   	=model_inputs.pop('end_labels'	)
  UpperCamelCase_     :					List[Any]   	=model_inputs.pop('pooled_labels'	)

  UpperCamelCase_     :					Union[str, Any]   	=state.apply_fn(**__lowercase						,	params=__lowercase						,	dropout_rng=__lowercase						,	train=__lowercase	)
  UpperCamelCase_	,     UpperCamelCase_	,     UpperCamelCase_     :					Tuple   	=outputs

  return state.loss_fn(
      __lowercase						,	__lowercase						,	__lowercase						,	__lowercase						,	__lowercase						,	__lowercase						,	)

 UpperCamelCase_	,     UpperCamelCase_     :					Optional[int]   	=jax.random.split(__lowercase	)
 UpperCamelCase_     :					Optional[int]   	=jax.value_and_grad(__lowercase	)
 UpperCamelCase_	,     UpperCamelCase_     :					Union[str, Any]   	=grad_fn(state.params	)
 UpperCamelCase_     :					Tuple   	=jax.lax.pmean({'loss': loss}						,	axis_name='batch'	)
 UpperCamelCase_     :					List[str]   	=jax.lax.pmean(__lowercase						,	'batch'	)

 UpperCamelCase_     :					Any   	=state.apply_gradients(grads=__lowercase	)
 return state, metrics, new_drp_rng




@partial(jax.pmap						,	axis_name='batch'	)
def       A_  (				__lowercase						,	**__lowercase	):
 UpperCamelCase_     :					Any   	=model_inputs.pop('start_labels'	)
 UpperCamelCase_     :					List[str]   	=model_inputs.pop('end_labels'	)
 UpperCamelCase_     :					Optional[int]   	=model_inputs.pop('pooled_labels'	)

 UpperCamelCase_     :					str   	=state.apply_fn(**__lowercase						,	params=state.params						,	train=__lowercase	)
 UpperCamelCase_	,     UpperCamelCase_	,     UpperCamelCase_     :					Tuple   	=outputs

 UpperCamelCase_     :					str   	=state.loss_fn(__lowercase						,	__lowercase						,	__lowercase						,	__lowercase						,	__lowercase						,	__lowercase	)
 UpperCamelCase_     :					List[Any]   	=jax.lax.pmean({'loss': loss}						,	axis_name='batch'	)
 return metrics





class 				a__					(						train_state.TrainState       ):
     UpperCAmelCase__         =      struct.field(pytree_node=A__       )





@dataclass
class 				a__					:
     UpperCAmelCase__         =      42
     UpperCAmelCase__         =      42
     UpperCAmelCase__         =      42
     UpperCAmelCase__         =      42
     UpperCAmelCase__         =      42
     UpperCAmelCase__         =      42
     UpperCAmelCase__         =      None



     def  lowerCamelCase_						(							self :Optional[int]      ,  _lowerCamelCase :Dict      ,  _lowerCamelCase :Optional[int]      ,  _lowerCamelCase :Dict      ,  _lowerCamelCase :List[Any]=None   ):







      '''simple docstring'''



      UpperCamelCase_     :					Tuple   	=model.params
      UpperCamelCase_     :					str   	=TrainState.create(
          apply_fn=model.__call__      ,  params=_lowerCamelCase      ,  tx=_lowerCamelCase      ,  loss_fn=_lowerCamelCase      ,  )
      if ckpt_dir is not None:
       UpperCamelCase_	,     UpperCamelCase_	,     UpperCamelCase_	,     UpperCamelCase_	,     UpperCamelCase_     :					List[Any]   	=restore_checkpoint(_lowerCamelCase      ,  _lowerCamelCase   )
       UpperCamelCase_     :					Optional[int]   	={
           'lr': args.lr,
           'init_lr': args.init_lr,
           'warmup_steps': args.warmup_steps,
           'num_train_steps': num_train_steps,
           'weight_decay': args.weight_decay,
       }
       UpperCamelCase_	,     UpperCamelCase_     :					Union[str, Any]   	=build_tx(**_lowerCamelCase   )
       UpperCamelCase_     :					List[Any]   	=train_state.TrainState(
           step=_lowerCamelCase      ,  apply_fn=model.__call__      ,  params=_lowerCamelCase      ,  tx=_lowerCamelCase      ,  opt_state=_lowerCamelCase      ,  )
       UpperCamelCase_     :					Any   	=args
       UpperCamelCase_     :					Dict   	=data_collator
       UpperCamelCase_     :					int   	=lr
       UpperCamelCase_     :					List[str]   	=params
      UpperCamelCase_     :					str   	=jax_utils.replicate(_lowerCamelCase   )
      return state



     def  lowerCamelCase_						(							self :Union[str, Any]      ,  _lowerCamelCase :Optional[int]      ,  _lowerCamelCase :Any      ,  _lowerCamelCase :Dict   ):







      '''simple docstring'''



      UpperCamelCase_     :					str   	=self.args
      UpperCamelCase_     :					str   	=len(_lowerCamelCase   ) // args.batch_size

      UpperCamelCase_     :					Union[str, Any]   	=jax.random.PRNGKey(0   )
      UpperCamelCase_     :					Union[str, Any]   	=jax.random.split(_lowerCamelCase      ,  jax.device_count()   )
      for epoch in range(args.max_epochs   ):
       UpperCamelCase_     :					Union[str, Any]   	=jnp.array(0      ,  dtype=jnp.floataa   )
       UpperCamelCase_     :					Optional[int]   	=get_batched_dataset(_lowerCamelCase      ,  args.batch_size      ,  seed=_lowerCamelCase   )
       UpperCamelCase_     :					Dict   	=0
       for batch in tqdm(_lowerCamelCase      ,  total=_lowerCamelCase      ,  desc=f'''Running EPOCH-{epoch}'''   ):
        UpperCamelCase_     :					Dict   	=self.data_collator(_lowerCamelCase   )
        UpperCamelCase_	,     UpperCamelCase_	,     UpperCamelCase_     :					Union[str, Any]   	=self.train_step_fn(_lowerCamelCase      ,  _lowerCamelCase      ,  **_lowerCamelCase   )
        running_loss += jax_utils.unreplicate(metrics['loss']   )
        i += 1
        if i % args.logging_steps == 0:
         UpperCamelCase_     :					str   	=jax_utils.unreplicate(state.step   )
         UpperCamelCase_     :					Tuple   	=running_loss.item() / i
         UpperCamelCase_     :					Optional[int]   	=self.scheduler_fn(state_step - 1   )

         UpperCamelCase_     :					Dict   	=self.evaluate(_lowerCamelCase      ,  _lowerCamelCase   )
         UpperCamelCase_     :					List[str]   	={
             'step': state_step.item(),
             'eval_loss': eval_loss.item(),
             'tr_loss': tr_loss,
             'lr': lr.item(),
         }
         tqdm.write(str(_lowerCamelCase   )   )
         self.logger.log(_lowerCamelCase      ,  commit=_lowerCamelCase   )

        if i % args.save_steps == 0:
         self.save_checkpoint(args.save_dir + f'''-e{epoch}-s{i}'''      ,  state=_lowerCamelCase   )



     def  lowerCamelCase_						(							self :List[str]      ,  _lowerCamelCase :str      ,  _lowerCamelCase :List[Any]   ):







      '''simple docstring'''



      UpperCamelCase_     :					List[str]   	=get_batched_dataset(_lowerCamelCase      ,  self.args.batch_size   )
      UpperCamelCase_     :					List[Any]   	=len(_lowerCamelCase   ) // self.args.batch_size
      UpperCamelCase_     :					int   	=jnp.array(0      ,  dtype=jnp.floataa   )
      UpperCamelCase_     :					Tuple   	=0
      for batch in tqdm(_lowerCamelCase      ,  total=_lowerCamelCase      ,  desc='Evaluating ... '   ):
       UpperCamelCase_     :					List[str]   	=self.data_collator(_lowerCamelCase   )
       UpperCamelCase_     :					Union[str, Any]   	=self.val_step_fn(_lowerCamelCase      ,  **_lowerCamelCase   )
       running_loss += jax_utils.unreplicate(metrics['loss']   )
       i += 1
      return running_loss / i




     def  lowerCamelCase_						(							self :List[Any]      ,  _lowerCamelCase :List[str]      ,  _lowerCamelCase :Tuple   ):







      '''simple docstring'''



      UpperCamelCase_     :					Optional[Any]   	=jax_utils.unreplicate(_lowerCamelCase   )
      print(f'''SAVING CHECKPOINT IN {save_dir}'''      ,  end=' ... '   )
      self.model_save_fn(_lowerCamelCase      ,  params=state.params   )
      with open(os.path.join(_lowerCamelCase      ,  'opt_state.msgpack'   )      ,  'wb'   ) as f:
       f.write(to_bytes(state.opt_state   )   )
      joblib.dump(self.args      ,  os.path.join(_lowerCamelCase      ,  'args.joblib'   )   )
      joblib.dump(self.data_collator      ,  os.path.join(_lowerCamelCase      ,  'data_collator.joblib'   )   )
      with open(os.path.join(_lowerCamelCase      ,  'training_state.json'   )      ,  'w'   ) as f:
       json.dump({'step': state.step.item()}      ,  _lowerCamelCase   )
      print('DONE'   )




def       A_  (				__lowercase						,	__lowercase	):
 print(F'''RESTORING CHECKPOINT FROM {save_dir}'''						,	end=' ... '	)
 with open(os.path.join(__lowercase						,	'flax_model.msgpack'	)						,	'rb'	) as f:
  UpperCamelCase_     :					Optional[Any]   	=from_bytes(state.params						,	f.read()	)

 with open(os.path.join(__lowercase						,	'opt_state.msgpack'	)						,	'rb'	) as f:
  UpperCamelCase_     :					Dict   	=from_bytes(state.opt_state						,	f.read()	)

 UpperCamelCase_     :					Tuple   	=joblib.load(os.path.join(__lowercase						,	'args.joblib'	)	)
 UpperCamelCase_     :					List[Any]   	=joblib.load(os.path.join(__lowercase						,	'data_collator.joblib'	)	)

 with open(os.path.join(__lowercase						,	'training_state.json'	)						,	'r'	) as f:
  UpperCamelCase_     :					Any   	=json.load(__lowercase	)
 UpperCamelCase_     :					Any   	=training_state['step']

 print('DONE'	)
 return params, opt_state, step, args, data_collator




def       A_  (				__lowercase						,	__lowercase						,	__lowercase						,	__lowercase	):
 UpperCamelCase_     :					Optional[Any]   	=num_train_steps - warmup_steps
 UpperCamelCase_     :					Optional[Any]   	=optax.linear_schedule(init_value=__lowercase						,	end_value=__lowercase						,	transition_steps=__lowercase	)
 UpperCamelCase_     :					List[Any]   	=optax.linear_schedule(init_value=__lowercase						,	end_value=1e-7						,	transition_steps=__lowercase	)
 UpperCamelCase_     :					Dict   	=optax.join_schedules(schedules=[warmup_fn, decay_fn]						,	boundaries=[warmup_steps]	)
 return lr





def       A_  (				__lowercase						,	__lowercase						,	__lowercase						,	__lowercase						,	__lowercase	):
 def weight_decay_mask(__lowercase	):
  UpperCamelCase_     :					str   	=traverse_util.flatten_dict(__lowercase	)
  UpperCamelCase_     :					List[Any]   	={k: (v[-1] != 'bias' and v[-2:] != ('LayerNorm', 'scale')) for k, v in params.items()}
  return traverse_util.unflatten_dict(__lowercase	)

 UpperCamelCase_     :					Union[str, Any]   	=scheduler_fn(__lowercase						,	__lowercase						,	__lowercase						,	__lowercase	)

 UpperCamelCase_     :					Optional[int]   	=optax.adamw(learning_rate=__lowercase						,	weight_decay=__lowercase						,	mask=__lowercase	)
 return tx, lr


"""simple docstring"""






import argparse
import json
from typing import List

from ltp import LTP

from transformers import BertTokenizer




def       A_  (				__lowercase	):
 # This defines a "chinese character" as anything in the CJK Unicode block:
 #   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
 #
 # Note that the CJK Unicode block is NOT all Japanese and Korean characters,
 # despite its name. The modern Korean Hangul alphabet is a different block,
 # as is Japanese Hiragana and Katakana. Those alphabets are used to write
 # space-separated words, so they are not treated specially and handled
 # like the all of the other languages.
 if (
     (cp >= 0X4_E_0_0 and cp <= 0X9_F_F_F)
     or (cp >= 0X3_4_0_0 and cp <= 0X4_D_B_F)  #
     or (cp >= 0X2_0_0_0_0 and cp <= 0X2_A_6_D_F)  #
     or (cp >= 0X2_A_7_0_0 and cp <= 0X2_B_7_3_F)  #
     or (cp >= 0X2_B_7_4_0 and cp <= 0X2_B_8_1_F)  #
     or (cp >= 0X2_B_8_2_0 and cp <= 0X2_C_E_A_F)  #
     or (cp >= 0XF_9_0_0 and cp <= 0XF_A_F_F)
     or (cp >= 0X2_F_8_0_0 and cp <= 0X2_F_A_1_F)  #
 ):  #
  return True

 return False




def       A_  (				__lowercase	):
 # word like '180' or '身高' or '神'
 for char in word:
  UpperCamelCase_     :					Union[str, Any]   	=ord(__lowercase	)
  if not _is_chinese_char(__lowercase	):
   return 0
 return 1




def       A_  (				__lowercase	):
 UpperCamelCase_     :					List[str]   	=set()

 for token in tokens:
  UpperCamelCase_     :					Optional[int]   	=len(__lowercase	) > 1 and is_chinese(__lowercase	)
  if chinese_word:
   word_set.add(__lowercase	)
 UpperCamelCase_     :					Tuple   	=list(__lowercase	)
 return word_list




def       A_  (				__lowercase						,	__lowercase	):
 if not chinese_word_set:
  return bert_tokens
 UpperCamelCase_     :					List[str]   	=max([len(__lowercase	) for w in chinese_word_set]	)

 UpperCamelCase_     :					Optional[Any]   	=bert_tokens
 UpperCamelCase_	,     UpperCamelCase_     :					Union[str, Any]   	=0, len(__lowercase	)
 while start < end:
  UpperCamelCase_     :					str   	=True
  if is_chinese(bert_word[start]	):
   UpperCamelCase_     :					Optional[int]   	=min(end - start						,	__lowercase	)
   for i in range(__lowercase						,	1						,	-1	):
    UpperCamelCase_     :					Tuple   	=''.join(bert_word[start : start + i]	)
    if whole_word in chinese_word_set:
     for j in range(start + 1						,	start + i	):
      UpperCamelCase_     :					Tuple   	='##' + bert_word[j]
     UpperCamelCase_     :					int   	=start + i
     UpperCamelCase_     :					Dict   	=False
     break
  if single_word:
   start += 1
 return bert_word




def       A_  (				__lowercase						,	__lowercase						,	__lowercase	):
 UpperCamelCase_     :					Tuple   	=[]

 for i in range(0						,	len(__lowercase	)						,	1_00	):
  UpperCamelCase_     :					Union[str, Any]   	=ltp_tokenizer.seg(lines[i : i + 1_00]	)[0]
  UpperCamelCase_     :					int   	=[get_chinese_word(__lowercase	) for r in res]
  ltp_res.extend(__lowercase	)
 assert len(__lowercase	) == len(__lowercase	)

 UpperCamelCase_     :					Dict   	=[]
 for i in range(0						,	len(__lowercase	)						,	1_00	):
  UpperCamelCase_     :					int   	=bert_tokenizer(lines[i : i + 1_00]						,	add_special_tokens=__lowercase						,	truncation=__lowercase						,	max_length=5_12	)
  bert_res.extend(res['input_ids']	)
 assert len(__lowercase	) == len(__lowercase	)

 UpperCamelCase_     :					Dict   	=[]
 for input_ids, chinese_word in zip(__lowercase						,	__lowercase	):
  UpperCamelCase_     :					List[str]   	=[]
  for id in input_ids:
   UpperCamelCase_     :					Union[str, Any]   	=bert_tokenizer._convert_id_to_token(__lowercase	)
   input_tokens.append(__lowercase	)
  UpperCamelCase_     :					Optional[int]   	=add_sub_symbol(__lowercase						,	__lowercase	)
  UpperCamelCase_     :					Dict   	=[]
  # We only save pos of chinese subwords start with ##, which mean is part of a whole word.
  for i, token in enumerate(__lowercase	):
   if token[:2] == "##":
    UpperCamelCase_     :					Optional[int]   	=token[2:]
    # save chinese tokens' pos
    if len(__lowercase	) == 1 and _is_chinese_char(ord(__lowercase	)	):
     ref_id.append(__lowercase	)
  ref_ids.append(__lowercase	)

 assert len(__lowercase	) == len(__lowercase	)

 return ref_ids




def       A_  (				__lowercase	):
 # For Chinese (Ro)Bert, the best result is from : RoBERTa-wwm-ext (https://github.com/ymcui/Chinese-BERT-wwm)
 # If we want to fine-tune these model, we have to use same tokenizer : LTP (https://github.com/HIT-SCIR/ltp)
 with open(args.file_name						,	'r'						,	encoding='utf-8'	) as f:
  UpperCamelCase_     :					Tuple   	=f.readlines()
 UpperCamelCase_     :					Optional[int]   	=[line.strip() for line in data if len(__lowercase	) > 0 and not line.isspace()]  # avoid delimiter like '\u2029'
 UpperCamelCase_     :					Optional[Any]   	=LTP(args.ltp	)  # faster in GPU device
 UpperCamelCase_     :					Dict   	=BertTokenizer.from_pretrained(args.bert	)

 UpperCamelCase_     :					int   	=prepare_ref(__lowercase						,	__lowercase						,	__lowercase	)

 with open(args.save_path						,	'w'						,	encoding='utf-8'	) as f:
  UpperCamelCase_     :					Tuple   	=[json.dumps(__lowercase	) + '\n' for ref in ref_ids]
  f.writelines(__lowercase	)


if __name__ == "__main__":
 __SCREAMING_SNAKE_CASE							=     argparse.ArgumentParser(description='prepare_chinese_ref')
 parser.add_argument(
     '--file_name',
     type=str,
     default='./resources/chinese-demo.txt',
     help='file need process, same as training data in lm',
 )
 parser.add_argument(
     '--ltp', type=str, default='./resources/ltp', help='resources for LTP tokenizer, usually a path'
 )
 parser.add_argument('--bert', type=str, default='./resources/robert', help='resources for Bert tokenizer')
 parser.add_argument('--save_path', type=str, default='./resources/ref.txt', help='path to save res')

 __SCREAMING_SNAKE_CASE							=     parser.parse_args()
 main(args)
