



import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a_								=       logging.get_logger(__name__)

a_								=       """▁"""

a_								=       {"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

a_								=       {
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

a_								=       {
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

a_								=       {
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

a_								=       {
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}



class 			__lowerCAmelCase  (						lowerCAmelCase__ ):
 lowerCAmelCase__							       =	["input_ids"]

 lowerCAmelCase__							       =	VOCAB_FILES_NAMES
 lowerCAmelCase__							       =	PRETRAINED_INIT_CONFIGURATION
 lowerCAmelCase__							       =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowerCAmelCase__							       =	PRETRAINED_VOCAB_FILES_MAP
 lowerCAmelCase__							       =	RESOURCE_FILES_NAMES




 def __init__(    self						,					__UpperCAmelCase						,					__UpperCAmelCase=None						,					__UpperCAmelCase=False						,					__UpperCAmelCase="utf8"						,					__UpperCAmelCase="[UNK]"						,					__UpperCAmelCase="[SEP]"						,					__UpperCAmelCase="[PAD]"						,					__UpperCAmelCase="[CLS]"						,					__UpperCAmelCase="[MASK]"						,					__UpperCAmelCase = None						,					**__UpperCAmelCase						,					):



    '''simple docstring'''
    # Mask token behave like a normal word, i.e. include the space before it and
    # is included in the raw text, there should be a match in a non-normalized sentence.

    __lowerCamelCase    				=     {} if sp_model_kwargs is None else sp_model_kwargs
    super().__init__(
        do_lower_case=UpperCAmelCase_						,					unk_token=UpperCAmelCase_						,					sep_token=UpperCAmelCase_						,					pad_token=UpperCAmelCase_						,					cls_token=UpperCAmelCase_						,					mask_token=UpperCAmelCase_						,					vocab_file=UpperCAmelCase_						,					encoding=UpperCAmelCase_						,					sp_model_kwargs=self.sp_model_kwargs						,					**UpperCAmelCase_						,					)
    __lowerCamelCase    				=     do_lower_case
    __lowerCamelCase    				=     sentencepiece_model_ckpt
    __lowerCamelCase    				=     spm.SentencePieceProcessor(**self.sp_model_kwargs					)
    self.sp_model.Load(UpperCAmelCase_					)

    # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
    if vocab_file is not None:
       __lowerCamelCase    				=     self.load_vocab(filepath=UpperCAmelCase_					)
    else:
       __lowerCamelCase    				=     {self.sp_model.id_to_piece(UpperCAmelCase_					): id for id in range(self.sp_model.get_piece_size()					)}
    __lowerCamelCase    				=     {v: k for k, v in self.vocab.items()}




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    if text is None:
       return None

    __lowerCamelCase    				=     self.tokenize(UpperCAmelCase_					)
    __lowerCamelCase				,__lowerCamelCase    				=     '''''', []

    for i, ch in enumerate(UpperCAmelCase_					):
       if ch in self.SP_CHAR_MAPPING:
          __lowerCamelCase    				=     self.SP_CHAR_MAPPING.get(UpperCAmelCase_					)
       else:
          __lowerCamelCase    				=     unicodedata.normalize('''NFKC'''						,					UpperCAmelCase_					)
       if self.is_whitespace(UpperCAmelCase_					):
          continue
       normalized_text += ch
       char_mapping.extend([i] * len(UpperCAmelCase_					)					)

    __lowerCamelCase				,__lowerCamelCase				,__lowerCamelCase    				=     normalized_text, [], 0

    if self.do_lower_case:
       __lowerCamelCase    				=     text.lower()

    for token in split_tokens:
       if token[:1] == "▁":
          __lowerCamelCase    				=     token[1:]
       __lowerCamelCase    				=     text[offset:].index(UpperCAmelCase_					) + offset
       __lowerCamelCase    				=     start + len(UpperCAmelCase_					)

       token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)					)
       __lowerCamelCase    				=     end
    return token_mapping




 @property
 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    return len(self.vocab					)




 def 							lowerCamelCase   (    self					):



    '''simple docstring'''
    return dict(self.vocab						,					**self.added_tokens_encoder					)




 def __getstate__(    self					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.__dict__.copy()
    __lowerCamelCase    				=     None
    return state




 def __setstate__(    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    __lowerCamelCase    				=     d

    # for backward compatibility
    if not hasattr(self						,					'''sp_model_kwargs'''					):
       __lowerCamelCase    				=     {}

    __lowerCamelCase    				=     spm.SentencePieceProcessor(**self.sp_model_kwargs					)
    self.sp_model.Load(self.sentencepiece_model_ckpt					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    return "".join((self.SP_CHAR_MAPPING.get(UpperCAmelCase_						,					UpperCAmelCase_					) for c in text)					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase=False						,					__UpperCAmelCase=64						,					__UpperCAmelCase=0.1					):



    '''simple docstring'''

    if self.sp_model_kwargs.get('''enable_sampling'''					) is True:
       __lowerCamelCase    				=     True
    if self.sp_model_kwargs.get('''alpha'''					) is not None:
       __lowerCamelCase    				=     self.sp_model_kwargs.get('''alpha'''					)
    if self.sp_model_kwargs.get('''nbest_size'''					) is not None:
       __lowerCamelCase    				=     self.sp_model_kwargs.get('''nbest_size'''					)

    if not enable_sampling:
       __lowerCamelCase    				=     self.sp_model.EncodeAsPieces(UpperCAmelCase_					)
    else:
       __lowerCamelCase    				=     self.sp_model.SampleEncodeAsPieces(UpperCAmelCase_						,					UpperCAmelCase_						,					UpperCAmelCase_					)
    __lowerCamelCase    				=     []
    for pi, piece in enumerate(UpperCAmelCase_					):
       if piece == SPIECE_UNDERLINE:
          if not pieces[pi + 1].startswith(UpperCAmelCase_					) and pi != 0:
             new_pieces.append(UpperCAmelCase_					)
             continue
          else:
             continue
       __lowerCamelCase    				=     0
       for i, chunk in enumerate(UpperCAmelCase_					):
          if chunk == SPIECE_UNDERLINE:
             continue
          if self.is_ch_char(UpperCAmelCase_					) or self.is_punct(UpperCAmelCase_					):
             if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                new_pieces.append(piece[lst_i:i]					)
             new_pieces.append(UpperCAmelCase_					)
             __lowerCamelCase    				=     i + 1
          elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
             if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                new_pieces.append(piece[lst_i:i]					)
             __lowerCamelCase    				=     i
          elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
             if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                new_pieces.append(piece[lst_i:i]					)
             __lowerCamelCase    				=     i
       if len(UpperCAmelCase_					) > lst_i:
          new_pieces.append(piece[lst_i:]					)
    return new_pieces




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    __lowerCamelCase    				=     ''''''.join(UpperCAmelCase_					).replace(UpperCAmelCase_						,					''' '''					).strip()
    return out_string




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    __lowerCamelCase    				=     self.convert_ids_to_tokens(UpperCAmelCase_					)
    __lowerCamelCase    				=     ''''''.join(UpperCAmelCase_					).replace(UpperCAmelCase_						,					''' '''					).strip()
    return out_string




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    return self.vocab.get(UpperCAmelCase_						,					self.vocab.get(self.unk_token					)					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    return self.reverse_vocab.get(UpperCAmelCase_						,					self.unk_token					)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase=None					):



    '''simple docstring'''
    if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    __lowerCamelCase    				=     [self.cls_token_id]
    __lowerCamelCase    				=     [self.sep_token_id]
    return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase=None					):



    '''simple docstring'''
    if offset_mapping_a is None:
       return [(0, 0)] + offset_mapping_a + [(0, 0)]

    return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase=None						,					__UpperCAmelCase=False					):



    '''simple docstring'''

    if already_has_special_tokens:
       if token_ids_a is not None:
          raise ValueError(
              '''You should not supply a second sequence if the provided sequence of '''
              '''ids is already formatted with special tokens for the model.'''					)
       return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

    if token_ids_a is not None:
       return [1] + ([0] * len(UpperCAmelCase_					)) + [1, 1] + ([0] * len(UpperCAmelCase_					)) + [1]
    return [1] + ([0] * len(UpperCAmelCase_					)) + [1]




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase = None					):



    '''simple docstring'''
    # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
    if token_ids_a is None:
       # [CLS] X [SEP]
       return (len(UpperCAmelCase_					) + 2) * [0]

    # [CLS] A [SEP] [SEP] B [SEP]
    return [0] * (len(UpperCAmelCase_					) + 1) + [1] * (len(UpperCAmelCase_					) + 3)




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    if "\u4e00" <= char <= "\u9fff":
       return True
    return False




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    if ("a" <= char <= "z") or ("A" <= char <= "Z"):
       return True
    return False




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    if char in ",;:.?!~，；：。？！《》【】":
       return True
    return False




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    if char == " " or char == "\t" or char == "\n" or char == "\r":
       return True
    if len(UpperCAmelCase_					) == 1:
       __lowerCamelCase    				=     unicodedata.category(UpperCAmelCase_					)
       if cat == "Zs":
          return True
    return False




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase					):



    '''simple docstring'''
    __lowerCamelCase    				=     {}
    with io.open(UpperCAmelCase_						,					'''r'''						,					encoding='''utf-8'''					) as f:
       for index, line in enumerate(UpperCAmelCase_					):
          __lowerCamelCase    				=     line.rstrip('''\n'''					)
          __lowerCamelCase    				=     int(UpperCAmelCase_					)

    return token_to_idx




 def 							lowerCamelCase   (    self						,					__UpperCAmelCase						,					__UpperCAmelCase = None					):



    '''simple docstring'''
    __lowerCamelCase    				=     0
    if os.path.isdir(UpperCAmelCase_					):
       __lowerCamelCase    				=     os.path.join(
           UpperCAmelCase_						,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)
    else:
       __lowerCamelCase    				=     (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
    with open(UpperCAmelCase_						,					'''w'''						,					encoding='''utf-8'''					) as writer:
       for token, token_index in sorted(self.vocab.items()						,					key=lambda __UpperCAmelCase					: kv[1]					):
          if index != token_index:
             logger.warning(
                 F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                 ''' Please check that the vocabulary is not corrupted!'''					)
             __lowerCamelCase    				=     token_index
          writer.write(token + '''\n'''					)
          index += 1

    __lowerCamelCase    				=     os.path.join(UpperCAmelCase_						,					'''sentencepiece.bpe.model'''					)
    with open(UpperCAmelCase_						,					'''wb'''					) as fi:
       __lowerCamelCase    				=     self.sp_model.serialized_model_proto()
       fi.write(UpperCAmelCase_					)

    return (vocab_file,)











'''simple docstring'''





def 							lowerCamelCase	(					_snake_case	:    Optional[Any]  ,_snake_case	:    Union[str, Any]  ,_snake_case	:    Dict  ,_snake_case	:    List[str]  ,_snake_case	:    List[str]  ,_snake_case	:    List[str]							):





    '''simple docstring'''


    if index == r:
        for j in range(_snake_case							):
            print(data[j]  ,end=" "							)
        print(" "							)
        return
    #  When no more elements are there to put in data[]
    if i >= n:
        return
    # current is included, put next at next location
    lowercase__									=    arr[i]
    combination_util(_snake_case  ,_snake_case  ,_snake_case  ,index + 1  ,_snake_case  ,i + 1							)
    # current is excluded, replace it with
    # next (Note that i+1 is passed, but
    # index is not changed)
    combination_util(_snake_case  ,_snake_case  ,_snake_case  ,_snake_case  ,_snake_case  ,i + 1							)
    # The main function that prints all combinations
    # of size r in arr[] of size n. This function
    # mainly uses combinationUtil()




def 							lowerCamelCase	(					_snake_case	:    Optional[Any]  ,_snake_case	:    Tuple  ,_snake_case	:    Union[str, Any]							):





    '''simple docstring'''


    lowercase__									=    [0] * r
    # Print all combination using temporary array 'data[]'
    combination_util(_snake_case  ,_snake_case  ,_snake_case  ,0  ,_snake_case  ,0							)


if __name__ == "__main__":
 # Driver code to check the function above
 SCREAMING_SNAKE_CASE__          =  [10, 20, 30, 40, 50]
 print_combination(arr, len(arr), 3)
 # This code is contributed by Ambuj sahu




