



"""simple docstring"""







import argparse
import json
from typing import List

from ltp import LTP

from transformers.models.bert.tokenization_bert import BertTokenizer

def     UpperCamelCase_   (   lowerCAmelCase__				:    Union[str, Any]						)						->       Optional[Any]:

  """simple docstring"""




  if (
      (cp >= 0x4_e_0_0 and cp <= 0x9_f_f_f)
      or (cp >= 0x3_4_0_0 and cp <= 0x4_d_b_f)  #
      or (cp >= 0x2_0_0_0_0 and cp <= 0x2_a_6_d_f)  #
      or (cp >= 0x2_a_7_0_0 and cp <= 0x2_b_7_3_f)  #
      or (cp >= 0x2_b_7_4_0 and cp <= 0x2_b_8_1_f)  #
      or (cp >= 0x2_b_8_2_0 and cp <= 0x2_c_e_a_f)  #
      or (cp >= 0xf_9_0_0 and cp <= 0xf_a_f_f)
      or (cp >= 0x2_f_8_0_0 and cp <= 0x2_f_a_1_f)  #
  ):  #
    return True

  return False

def     UpperCamelCase_   (   lowerCAmelCase__				:    str						)						->       Tuple:

  """simple docstring"""




  for char in word:
    lowerCAmelCase_       :    List[Any]								=			ord(lowerCAmelCase__						)
    if not _is_chinese_char(lowerCAmelCase__						):
      return 0
  return 1

def     UpperCamelCase_   (   lowerCAmelCase__				:    List[str]						)						->       Any:

  """simple docstring"""




  lowerCAmelCase_       :    Tuple								=			set()

  for token in tokens:
    lowerCAmelCase_       :    Dict								=			len(lowerCAmelCase__						) > 1 and is_chinese(lowerCAmelCase__						)
    if chinese_word:
      word_set.add(lowerCAmelCase__						)
  lowerCAmelCase_       :    Tuple								=			list(lowerCAmelCase__						)
  return word_list

def     UpperCamelCase_   (   lowerCAmelCase__				:    List[str]   ,       lowerCAmelCase__				:    set()						)						->       int:

  """simple docstring"""




  if not chinese_word_set:
    return bert_tokens
  lowerCAmelCase_       :    List[str]								=			max([len(lowerCAmelCase__						) for w in chinese_word_set]						)

  lowerCAmelCase_       :    Dict								=			bert_tokens
  lowerCAmelCase_						,lowerCAmelCase_       :    Dict								=			0, len(lowerCAmelCase__						)
  while start < end:
    lowerCAmelCase_       :    str								=			True
    if is_chinese(bert_word[start]						):
      lowerCAmelCase_       :    Union[str, Any]								=			min(end - start   ,       lowerCAmelCase__						)
      for i in range(lowerCAmelCase__   ,       1   ,       -1						):
        lowerCAmelCase_       :    str								=			''.join(bert_word[start : start + i]						)
        if whole_word in chinese_word_set:
          for j in range(start + 1   ,       start + i						):
            lowerCAmelCase_       :    Dict								=			'##' + bert_word[j]
          lowerCAmelCase_       :    int								=			start + i
          lowerCAmelCase_       :    str								=			False
          break
    if single_word:
      start += 1
  return bert_word

def     UpperCamelCase_   (   lowerCAmelCase__				:    List[str]   ,       lowerCAmelCase__				:    LTP   ,       lowerCAmelCase__				:    BertTokenizer						)						->       Optional[Any]:

  """simple docstring"""




  lowerCAmelCase_       :    Tuple								=			[]

  for i in range(0   ,       len(lowerCAmelCase__						)   ,       100						):
    lowerCAmelCase_       :    Any								=			ltp_tokenizer.pipeline(lines[i : i + 100]   ,       tasks=['cws']						).cws
    lowerCAmelCase_       :    Optional[int]								=			[get_chinese_word(lowerCAmelCase__						) for r in res]
    ltp_res.extend(lowerCAmelCase__						)
  assert len(lowerCAmelCase__						) == len(lowerCAmelCase__						)

  lowerCAmelCase_       :    Optional[Any]								=			[]
  for i in range(0   ,       len(lowerCAmelCase__						)   ,       100						):
    lowerCAmelCase_       :    str								=			bert_tokenizer(lines[i : i + 100]   ,       add_special_tokens=lowerCAmelCase__   ,       truncation=lowerCAmelCase__   ,       max_length=512						)
    bert_res.extend(res['input_ids']						)
  assert len(lowerCAmelCase__						) == len(lowerCAmelCase__						)

  lowerCAmelCase_       :    int								=			[]
  for input_ids, chinese_word in zip(lowerCAmelCase__   ,       lowerCAmelCase__						):
    lowerCAmelCase_       :    Optional[int]								=			[]
    for id in input_ids:
      lowerCAmelCase_       :    Optional[Any]								=			bert_tokenizer._convert_id_to_token(lowerCAmelCase__						)
      input_tokens.append(lowerCAmelCase__						)
    lowerCAmelCase_       :    Optional[Any]								=			add_sub_symbol(lowerCAmelCase__   ,       lowerCAmelCase__						)
    lowerCAmelCase_       :    int								=			[]
    # We only save pos of chinese subwords start with ##, which mean is part of a whole word.
    for i, token in enumerate(lowerCAmelCase__						):
      if token[:2] == "##":
        lowerCAmelCase_       :    int								=			token[2:]
        # save chinese tokens' pos
        if len(lowerCAmelCase__						) == 1 and _is_chinese_char(ord(lowerCAmelCase__						)						):
          ref_id.append(lowerCAmelCase__						)
    ref_ids.append(lowerCAmelCase__						)

  assert len(lowerCAmelCase__						) == len(lowerCAmelCase__						)

  return ref_ids

def     UpperCamelCase_   (   lowerCAmelCase__				:    List[Any]						)						->       Optional[int]:

  """simple docstring"""




  with open(args.file_name   ,       'r'   ,       encoding='utf-8'						) as f:
    lowerCAmelCase_       :    List[Any]								=			f.readlines()
  lowerCAmelCase_       :    Union[str, Any]								=			[line.strip() for line in data if len(lowerCAmelCase__						) > 0 and not line.isspace()]  # avoid delimiter like '\u2029'
  lowerCAmelCase_       :    Any								=			LTP(args.ltp						)  # faster in GPU device
  lowerCAmelCase_       :    int								=			BertTokenizer.from_pretrained(args.bert						)

  lowerCAmelCase_       :    Tuple								=			prepare_ref(lowerCAmelCase__   ,       lowerCAmelCase__   ,       lowerCAmelCase__						)

  with open(args.save_path   ,       'w'   ,       encoding='utf-8'						) as f:
    lowerCAmelCase_       :    str								=			[json.dumps(lowerCAmelCase__						) + '\n' for ref in ref_ids]
    f.writelines(lowerCAmelCase__						)


if __name__ == "__main__":
     lowercase__		:    str				   =     argparse.ArgumentParser(description="""prepare_chinese_ref""")
     parser.add_argument(
         """--file_name""",
         required=False,
         type=str,
         default="""./resources/chinese-demo.txt""",
         help="""file need process, same as training data in lm""",
     )
     parser.add_argument(
         """--ltp""",
         required=False,
         type=str,
         default="""./resources/ltp""",
         help="""resources for LTP tokenizer, usually a path""",
     )
     parser.add_argument(
         """--bert""",
         required=False,
         type=str,
         default="""./resources/robert""",
         help="""resources for Bert tokenizer""",
     )
     parser.add_argument(
         """--save_path""",
         required=False,
         type=str,
         default="""./resources/ref.txt""",
         help="""path to save res""",
     )

     lowercase__		:    Dict				   =     parser.parse_args()
     main(args)










"""simple docstring"""







from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__		:    List[Any]				   =     logging.get_logger(__name__)

lowercase__		:    List[str]				   =     {
    """s-JoL/Open-Llama-V1""": """https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json""",
}







class 				UpperCamelCase__       (     lowercase_  ):




      """simple docstring"""




      _SCREAMING_SNAKE_CASE      =					"""open-llama"""



      def __init__(				self    :				Optional[int]   ,      SCREAMING_SNAKE_CASE_    :				Tuple=1_0_0_0_0_0   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]=4_0_9_6   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=1_1_0_0_8   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]=3_2   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=3_2   ,      SCREAMING_SNAKE_CASE_    :				str="silu"   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=2_0_4_8   ,      SCREAMING_SNAKE_CASE_    :				Union[str, Any]=0.02   ,      SCREAMING_SNAKE_CASE_    :				int=1E-6   ,      SCREAMING_SNAKE_CASE_    :				List[str]=True   ,      SCREAMING_SNAKE_CASE_    :				Dict=0   ,      SCREAMING_SNAKE_CASE_    :				Dict=1   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=2   ,      SCREAMING_SNAKE_CASE_    :				str=False   ,      SCREAMING_SNAKE_CASE_    :				Optional[int]=True   ,      SCREAMING_SNAKE_CASE_    :				List[Any]=0.1   ,      SCREAMING_SNAKE_CASE_    :				Optional[Any]=0.1   ,      SCREAMING_SNAKE_CASE_    :				Dict=True   ,      SCREAMING_SNAKE_CASE_    :				Any=True   ,      SCREAMING_SNAKE_CASE_    :				Tuple=None   ,      **SCREAMING_SNAKE_CASE_    :				Optional[int]   ,      ):
        lowerCAmelCase_       :    Union[str, Any]								=			vocab_size
        lowerCAmelCase_       :    int								=			max_position_embeddings
        lowerCAmelCase_       :    Union[str, Any]								=			hidden_size
        lowerCAmelCase_       :    Optional[Any]								=			intermediate_size
        lowerCAmelCase_       :    List[Any]								=			num_hidden_layers
        lowerCAmelCase_       :    Optional[int]								=			num_attention_heads
        lowerCAmelCase_       :    List[Any]								=			hidden_act
        lowerCAmelCase_       :    List[Any]								=			initializer_range
        lowerCAmelCase_       :    List[str]								=			rms_norm_eps
        lowerCAmelCase_       :    List[Any]								=			use_cache
        lowerCAmelCase_       :    Optional[int]								=			kwargs.pop(
            'use_memorry_efficient_attention'   ,      SCREAMING_SNAKE_CASE_	)
        lowerCAmelCase_       :    List[Any]								=			hidden_dropout_prob
        lowerCAmelCase_       :    List[str]								=			attention_dropout_prob
        lowerCAmelCase_       :    Tuple								=			use_stable_embedding
        lowerCAmelCase_       :    Optional[Any]								=			shared_input_output_embedding
        lowerCAmelCase_       :    Optional[Any]								=			rope_scaling
        self._rope_scaling_validation()

        super().__init__(
            pad_token_id=SCREAMING_SNAKE_CASE_   ,      bos_token_id=SCREAMING_SNAKE_CASE_   ,      eos_token_id=SCREAMING_SNAKE_CASE_   ,      tie_word_embeddings=SCREAMING_SNAKE_CASE_   ,      **SCREAMING_SNAKE_CASE_   ,      )





      def 					SCREAMING_SNAKE_CASE__			(				self    :				List[str]	):
        if self.rope_scaling is None:
          return

        if not isinstance(self.rope_scaling   ,      SCREAMING_SNAKE_CASE_	) or len(self.rope_scaling	) != 2:
          raise ValueError(
              '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
              F"got {self.rope_scaling}"	)
        lowerCAmelCase_       :    int								=			self.rope_scaling.get('type'   ,      SCREAMING_SNAKE_CASE_	)
        lowerCAmelCase_       :    Union[str, Any]								=			self.rope_scaling.get('factor'   ,      SCREAMING_SNAKE_CASE_	)
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
          raise ValueError(
              F"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"	)
        if rope_scaling_factor is None or not isinstance(SCREAMING_SNAKE_CASE_   ,      SCREAMING_SNAKE_CASE_	) or rope_scaling_factor <= 1.0:
          raise ValueError(F"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"	)





