


import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__a	 =      logging.get_logger(__name__)

__a	 =      """▁"""

__a	 =      {"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

__a	 =      {
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

__a	 =      {
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

__a	 =      {
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

__a	 =      {
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}






class 	__lowercase					(	__snake_case	):
      UpperCamelCase										=							["input_ids"]

      UpperCamelCase										=							VOCAB_FILES_NAMES
      UpperCamelCase										=							PRETRAINED_INIT_CONFIGURATION
      UpperCamelCase										=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCamelCase										=							PRETRAINED_VOCAB_FILES_MAP
      UpperCamelCase										=							RESOURCE_FILES_NAMES





      def __init__(		self    :       Optional[Any]       ,							__lowerCamelCase    :       Tuple       ,							__lowerCamelCase    :       Any=None       ,							__lowerCamelCase    :       int=False       ,							__lowerCamelCase    :       str="utf8"       ,							__lowerCamelCase    :       List[Any]="[UNK]"       ,							__lowerCamelCase    :       Union[str, Any]="[SEP]"       ,							__lowerCamelCase    :       Optional[Any]="[PAD]"       ,							__lowerCamelCase    :       List[Any]="[CLS]"       ,							__lowerCamelCase    :       Union[str, Any]="[MASK]"       ,							__lowerCamelCase    :       Optional[Dict[str, Any]] = None       ,							**__lowerCamelCase    :       List[str]       ,							)     -> None:






         """simple docstring"""







         UpperCAmelCase      					=  {} if sp_model_kwargs is None else sp_model_kwargs
         super().__init__(
             do_lower_case=__lowerCamelCase       ,							unk_token=__lowerCamelCase       ,							sep_token=__lowerCamelCase       ,							pad_token=__lowerCamelCase       ,							cls_token=__lowerCamelCase       ,							mask_token=__lowerCamelCase       ,							vocab_file=__lowerCamelCase       ,							encoding=__lowerCamelCase       ,							sp_model_kwargs=self.sp_model_kwargs       ,							**__lowerCamelCase       ,							)
         UpperCAmelCase      					=  do_lower_case
         UpperCAmelCase      					=  sentencepiece_model_ckpt
         UpperCAmelCase      					=  spm.SentencePieceProcessor(**self.sp_model_kwargs							)
         self.sp_model.Load(__lowerCamelCase							)

         # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
         if vocab_file is not None:
            UpperCAmelCase      					=  self.load_vocab(filepath=__lowerCamelCase							)
         else:
            UpperCAmelCase      					=  {self.sp_model.id_to_piece(__lowerCamelCase							): id for id in range(self.sp_model.get_piece_size()							)}
         UpperCAmelCase      					=  {v: k for k, v in self.vocab.items()}





      def 							_lowercase					(		self    :       str       ,							__lowerCamelCase    :       Optional[int]							)     -> Tuple:






         """simple docstring"""







         if text is None:
            return None

         UpperCAmelCase      					=  self.tokenize(__lowerCamelCase							)
         UpperCAmelCase					,	UpperCAmelCase      					=  """""", []

         for i, ch in enumerate(__lowerCamelCase							):
            if ch in self.SP_CHAR_MAPPING:
               UpperCAmelCase      					=  self.SP_CHAR_MAPPING.get(__lowerCamelCase							)
            else:
               UpperCAmelCase      					=  unicodedata.normalize("""NFKC"""       ,							__lowerCamelCase							)
            if self.is_whitespace(__lowerCamelCase							):
               continue
            normalized_text += ch
            char_mapping.extend([i] * len(__lowerCamelCase							)							)

         UpperCAmelCase					,	UpperCAmelCase					,	UpperCAmelCase      					=  normalized_text, [], 0

         if self.do_lower_case:
            UpperCAmelCase      					=  text.lower()

         for token in split_tokens:
            if token[:1] == "▁":
               UpperCAmelCase      					=  token[1:]
            UpperCAmelCase      					=  text[offset:].index(__lowerCamelCase							) + offset
            UpperCAmelCase      					=  start + len(__lowerCamelCase							)

            token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)							)
            UpperCAmelCase      					=  end
         return token_mapping





      @property
      def 							_lowercase					(		self    :       Optional[Any]							)     -> Dict:






         """simple docstring"""







         return len(self.vocab							)





      def 							_lowercase					(		self    :       str							)     -> List[str]:






         """simple docstring"""







         return dict(self.vocab       ,							**self.added_tokens_encoder							)





      def __getstate__(		self    :       Optional[Any]							)     -> Any:






         """simple docstring"""







         UpperCAmelCase      					=  self.__dict__.copy()
         UpperCAmelCase      					=  None
         return state





      def __setstate__(		self    :       int       ,							__lowerCamelCase    :       List[Any]							)     -> Optional[int]:






         """simple docstring"""







         UpperCAmelCase      					=  d

         # for backward compatibility
         if not hasattr(self       ,							"""sp_model_kwargs"""							):
            UpperCAmelCase      					=  {}

         UpperCAmelCase      					=  spm.SentencePieceProcessor(**self.sp_model_kwargs							)
         self.sp_model.Load(self.sentencepiece_model_ckpt							)





      def 							_lowercase					(		self    :       Optional[int]       ,							__lowerCamelCase    :       str							)     -> Optional[Any]:






         """simple docstring"""







         return "".join((self.SP_CHAR_MAPPING.get(__lowerCamelCase       ,							__lowerCamelCase							) for c in text)							)





      def 							_lowercase					(		self    :       List[Any]       ,							__lowerCamelCase    :       List[Any]       ,							__lowerCamelCase    :       Dict=False       ,							__lowerCamelCase    :       int=6_4       ,							__lowerCamelCase    :       Dict=0.1							)     -> Any:






         """simple docstring"""







         if self.sp_model_kwargs.get("""enable_sampling"""							) is True:
            UpperCAmelCase      					=  True
         if self.sp_model_kwargs.get("""alpha"""							) is not None:
            UpperCAmelCase      					=  self.sp_model_kwargs.get("""alpha"""							)
         if self.sp_model_kwargs.get("""nbest_size"""							) is not None:
            UpperCAmelCase      					=  self.sp_model_kwargs.get("""nbest_size"""							)

         if not enable_sampling:
            UpperCAmelCase      					=  self.sp_model.EncodeAsPieces(__lowerCamelCase							)
         else:
            UpperCAmelCase      					=  self.sp_model.SampleEncodeAsPieces(__lowerCamelCase       ,							__lowerCamelCase       ,							__lowerCamelCase							)
         UpperCAmelCase      					=  []
         for pi, piece in enumerate(__lowerCamelCase							):
            if piece == SPIECE_UNDERLINE:
               if not pieces[pi + 1].startswith(__lowerCamelCase							) and pi != 0:
                  new_pieces.append(__lowerCamelCase							)
                  continue
               else:
                  continue
            UpperCAmelCase      					=  0
            for i, chunk in enumerate(__lowerCamelCase							):
               if chunk == SPIECE_UNDERLINE:
                  continue
               if self.is_ch_char(__lowerCamelCase							) or self.is_punct(__lowerCamelCase							):
                  if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                     new_pieces.append(piece[lst_i:i]							)
                  new_pieces.append(__lowerCamelCase							)
                  UpperCAmelCase      					=  i + 1
               elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                  if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                     new_pieces.append(piece[lst_i:i]							)
                  UpperCAmelCase      					=  i
               elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                  if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                     new_pieces.append(piece[lst_i:i]							)
                  UpperCAmelCase      					=  i
            if len(__lowerCamelCase							) > lst_i:
               new_pieces.append(piece[lst_i:]							)
         return new_pieces





      def 							_lowercase					(		self    :       Union[str, Any]       ,							__lowerCamelCase    :       List[str]							)     -> str:






         """simple docstring"""







         UpperCAmelCase      					=  """""".join(__lowerCamelCase							).replace(__lowerCamelCase       ,							""" """							).strip()
         return out_string





      def 							_lowercase					(		self    :       Any       ,							__lowerCamelCase    :       Optional[int]							)     -> Union[str, Any]:






         """simple docstring"""







         UpperCAmelCase      					=  self.convert_ids_to_tokens(__lowerCamelCase							)
         UpperCAmelCase      					=  """""".join(__lowerCamelCase							).replace(__lowerCamelCase       ,							""" """							).strip()
         return out_string





      def 							_lowercase					(		self    :       Optional[int]       ,							__lowerCamelCase    :       List[Any]							)     -> int:






         """simple docstring"""







         return self.vocab.get(__lowerCamelCase       ,							self.vocab.get(self.unk_token							)							)





      def 							_lowercase					(		self    :       Optional[int]       ,							__lowerCamelCase    :       Optional[Any]							)     -> Optional[Any]:






         """simple docstring"""







         return self.reverse_vocab.get(__lowerCamelCase       ,							self.unk_token							)





      def 							_lowercase					(		self    :       int       ,							__lowerCamelCase    :       Tuple       ,							__lowerCamelCase    :       Optional[Any]=None							)     -> List[str]:






         """simple docstring"""







         if token_ids_a is None:
            return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         UpperCAmelCase      					=  [self.cls_token_id]
         UpperCAmelCase      					=  [self.sep_token_id]
         return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep





      def 							_lowercase					(		self    :       Optional[int]       ,							__lowerCamelCase    :       Optional[int]       ,							__lowerCamelCase    :       List[str]=None							)     -> Tuple:






         """simple docstring"""







         if offset_mapping_a is None:
            return [(0, 0)] + offset_mapping_a + [(0, 0)]

         return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]





      def 							_lowercase					(		self    :       int       ,							__lowerCamelCase    :       Dict       ,							__lowerCamelCase    :       Optional[Any]=None       ,							__lowerCamelCase    :       Dict=False							)     -> Optional[int]:






         """simple docstring"""







         if already_has_special_tokens:
            if token_ids_a is not None:
               raise ValueError(
                   """You should not supply a second sequence if the provided sequence of """
                   """ids is already formatted with special tokens for the model."""							)
            return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

         if token_ids_a is not None:
            return [1] + ([0] * len(__lowerCamelCase							)) + [1, 1] + ([0] * len(__lowerCamelCase							)) + [1]
         return [1] + ([0] * len(__lowerCamelCase							)) + [1]





      def 							_lowercase					(		self    :       List[str]       ,							__lowerCamelCase    :       List[int]       ,							__lowerCamelCase    :       Optional[List[int]] = None							)     -> List[int]:






         """simple docstring"""







         if token_ids_a is None:
            # [CLS] X [SEP]
            return (len(__lowerCamelCase							) + 2) * [0]

         # [CLS] A [SEP] [SEP] B [SEP]
         return [0] * (len(__lowerCamelCase							) + 1) + [1] * (len(__lowerCamelCase							) + 3)





      def 							_lowercase					(		self    :       Union[str, Any]       ,							__lowerCamelCase    :       str							)     -> List[str]:






         """simple docstring"""







         if "\u4e00" <= char <= "\u9fff":
            return True
         return False





      def 							_lowercase					(		self    :       Optional[Any]       ,							__lowerCamelCase    :       Dict							)     -> Union[str, Any]:






         """simple docstring"""







         if ("a" <= char <= "z") or ("A" <= char <= "Z"):
            return True
         return False





      def 							_lowercase					(		self    :       Optional[int]       ,							__lowerCamelCase    :       int							)     -> int:






         """simple docstring"""







         if char in ",;:.?!~，；：。？！《》【】":
            return True
         return False





      def 							_lowercase					(		self    :       Optional[Any]       ,							__lowerCamelCase    :       Optional[Any]							)     -> List[str]:






         """simple docstring"""







         if char == " " or char == "\t" or char == "\n" or char == "\r":
            return True
         if len(__lowerCamelCase							) == 1:
            UpperCAmelCase      					=  unicodedata.category(__lowerCamelCase							)
            if cat == "Zs":
               return True
         return False





      def 							_lowercase					(		self    :       Any       ,							__lowerCamelCase    :       List[Any]							)     -> Union[str, Any]:






         """simple docstring"""







         UpperCAmelCase      					=  {}
         with io.open(__lowerCamelCase       ,							"""r"""       ,							encoding="""utf-8"""							) as f:
            for index, line in enumerate(__lowerCamelCase							):
               UpperCAmelCase      					=  line.rstrip("""\n"""							)
               UpperCAmelCase      					=  int(__lowerCamelCase							)

         return token_to_idx





      def 							_lowercase					(		self    :       Tuple       ,							__lowerCamelCase    :       str       ,							__lowerCamelCase    :       Optional[str] = None							)     -> Tuple[str]:






         """simple docstring"""







         UpperCAmelCase      					=  0
         if os.path.isdir(__lowerCamelCase							):
            UpperCAmelCase      					=  os.path.join(
                __lowerCamelCase       ,							(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)
         else:
            UpperCAmelCase      					=  (filename_prefix + """-""" if filename_prefix else """""") + save_directory
         with open(__lowerCamelCase       ,							"""w"""       ,							encoding="""utf-8"""							) as writer:
            for token, token_index in sorted(self.vocab.items()       ,							key=lambda __lowerCamelCase							: kv[1]							):
               if index != token_index:
                  logger.warning(
                      F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                      """ Please check that the vocabulary is not corrupted!"""							)
                  UpperCAmelCase      					=  token_index
               writer.write(token + """\n"""							)
               index += 1

         UpperCAmelCase      					=  os.path.join(__lowerCamelCase       ,							"""sentencepiece.bpe.model"""							)
         with open(__lowerCamelCase       ,							"""wb"""							) as fi:
            UpperCAmelCase      					=  self.sp_model.serialized_model_proto()
            fi.write(__lowerCamelCase							)

         return (vocab_file,)







def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->Any:
   UpperCAmelCase      					=  0
   UpperCAmelCase      					=  len(lowerCAmelCase_ )

   for i in range(n - 1 ):
      for j in range(i + 1	,    lowerCAmelCase_ ):
         if arr[i] > arr[j]:
            num_inversions += 1

   return num_inversions
def 	_UpperCamelCase					(    lowerCAmelCase_ )    ->Any:
   if len(lowerCAmelCase_ ) <= 1:
      return arr, 0
   UpperCAmelCase      					=  len(lowerCAmelCase_ ) // 2
   UpperCAmelCase      					=  arr[0:mid]
   UpperCAmelCase      					=  arr[mid:]

   UpperCAmelCase					,	UpperCAmelCase      					=  count_inversions_recursive(lowerCAmelCase_ )
   UpperCAmelCase					,	UpperCAmelCase      					=  count_inversions_recursive(lowerCAmelCase_ )
   UpperCAmelCase					,	UpperCAmelCase      					=  _count_cross_inversions(lowerCAmelCase_	,    lowerCAmelCase_ )

   UpperCAmelCase      					=  inversion_p + inversions_q + cross_inversions
   return c, num_inversions
def 	_UpperCamelCase					(    lowerCAmelCase_	,    lowerCAmelCase_ )    ->int:
   UpperCAmelCase      					=  []
   UpperCAmelCase      					=  UpperCAmelCase      					=  UpperCAmelCase      					=  0
   while i < len(lowerCAmelCase_ ) and j < len(lowerCAmelCase_ ):
      if p[i] > q[j]:
         # if P[1] > Q[j], then P[k] > Q[k] for all  i < k <= len(P)
         # These are all inversions. The claim emerges from the
         # property that P is sorted.
         num_inversion += len(lowerCAmelCase_ ) - i
         r.append(q[j] )
         j += 1
      else:
         r.append(p[i] )
         i += 1

   if i < len(lowerCAmelCase_ ):
      r.extend(p[i:] )
   else:
      r.extend(q[j:] )

   return r, num_inversion
def 	_UpperCamelCase					(    )    ->int:
   UpperCAmelCase      					=  [1_0, 2, 1, 5, 5, 2, 1_1]

   # this arr has 8 inversions:
   # (10, 2), (10, 1), (10, 5), (10, 5), (10, 2), (2, 1), (5, 2), (5, 2)

   UpperCAmelCase      					=  count_inversions_bf(lowerCAmelCase_ )
   UpperCAmelCase					,	UpperCAmelCase      					=  count_inversions_recursive(lowerCAmelCase_ )

   assert num_inversions_bf == num_inversions_recursive == 8

   print("""number of inversions = """	,    lowerCAmelCase_ )

   # testing an array with zero inversion (a sorted arr_1)

   arr_a.sort()
   UpperCAmelCase      					=  count_inversions_bf(lowerCAmelCase_ )
   UpperCAmelCase					,	UpperCAmelCase      					=  count_inversions_recursive(lowerCAmelCase_ )

   assert num_inversions_bf == num_inversions_recursive == 0
   print("""number of inversions = """	,    lowerCAmelCase_ )

   # an empty list should also have zero inversions
   UpperCAmelCase      					=  []
   UpperCAmelCase      					=  count_inversions_bf(lowerCAmelCase_ )
   UpperCAmelCase					,	UpperCAmelCase      					=  count_inversions_recursive(lowerCAmelCase_ )

   assert num_inversions_bf == num_inversions_recursive == 0
   print("""number of inversions = """	,    lowerCAmelCase_ )


if __name__ == "__main__":
 main()



