
"""simple docstring"""



import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
 import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a       :						str       		=		logging.get_logger(__name__)

a       :						Dict       		=		{'''vocab_file''': '''vocab.txt'''}

a       :						Dict       		=		{
    '''vocab_file''': {
        '''openbmb/cpm-ant-10b''': '''https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt''',
    },
}

a       :						Tuple       		=		{
    '''openbmb/cpm-ant-10b''': 1_0_2_4,
}


def 			_UpperCamelCase						(    _A   )     ->	Dict:





    """simple docstring"""

    _UpperCAmelCase        = collections.OrderedDict()
    with open(_A       ,   """r"""       ,   encoding="""utf-8"""   ) as reader:
        _UpperCAmelCase        = reader.readlines()
    for index, token in enumerate(_A   ):
        _UpperCAmelCase        = token.rstrip("""\n"""   )
        _UpperCAmelCase        = index
    return vocab




class 		a_    (			_UpperCAmelCase						):







    def __init__(      self      :							Optional[Any]  ,       __UpperCamelCase      :							int  ,       __UpperCamelCase      :							List[str]="<unk>"  ,       __UpperCamelCase      :							Dict=2_00  )		->Optional[int]:






        '''simple docstring'''






        _UpperCAmelCase        = vocab
        _UpperCAmelCase        = unk_token
        _UpperCAmelCase        = max_input_chars_per_word







    def   _snake_case       (      self      :							Union[str, Any]  ,       __UpperCamelCase      :							int  )		->Tuple:






        '''simple docstring'''






        _UpperCAmelCase        = list(__UpperCamelCase  )
        if len(__UpperCamelCase  ) > self.max_input_chars_per_word:
            return [self.unk_token]

        _UpperCAmelCase        = 0
        _UpperCAmelCase        = []
        while start < len(__UpperCamelCase  ):
            _UpperCAmelCase        = len(__UpperCamelCase  )
            _UpperCAmelCase        = None
            while start < end:
                _UpperCAmelCase        = """""".join(chars[start:end]  )
                if substr in self.vocab:
                    _UpperCAmelCase        = substr
                    break
                end -= 1
            if cur_substr is None:
                sub_tokens.append(self.unk_token  )
                start += 1
            else:
                sub_tokens.append(__UpperCamelCase  )
                _UpperCAmelCase        = end

        return sub_tokens







class 		a_    (			_UpperCAmelCase						):
    a   :					List[Any]			     =					VOCAB_FILES_NAMES
    a   :					Optional[int]			     =					PRETRAINED_VOCAB_FILES_MAP
    a   :					List[str]			     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a   :					Union[str, Any]			     =					['input_ids', 'attention_mask']
    a   :					int			     =					False







    def __init__(      self      :							Tuple  ,       __UpperCamelCase      :							Dict  ,       __UpperCamelCase      :							Union[str, Any]="<d>"  ,       __UpperCamelCase      :							Tuple="</d>"  ,       __UpperCamelCase      :							List[Any]="<s>"  ,       __UpperCamelCase      :							Any="</s>"  ,       __UpperCamelCase      :							Any="<pad>"  ,       __UpperCamelCase      :							str="<unk>"  ,       __UpperCamelCase      :							Optional[int]="</n>"  ,       __UpperCamelCase      :							Dict="</_>"  ,       __UpperCamelCase      :							str="left"  ,       **__UpperCamelCase      :							Optional[int]  ,       )		->Optional[int]:






        '''simple docstring'''






        requires_backends(self  ,       ["""jieba"""]  )
        super().__init__(
            bod_token=__UpperCamelCase  ,       eod_token=__UpperCamelCase  ,       bos_token=__UpperCamelCase  ,       eos_token=__UpperCamelCase  ,       pad_token=__UpperCamelCase  ,       unk_token=__UpperCamelCase  ,       line_token=__UpperCamelCase  ,       space_token=__UpperCamelCase  ,       padding_side=__UpperCamelCase  ,       **__UpperCamelCase  ,       )
        _UpperCAmelCase        = bod_token
        _UpperCAmelCase        = eod_token
        _UpperCAmelCase        = load_vocab(__UpperCamelCase  )
        _UpperCAmelCase        = self.encoder[space_token]
        _UpperCAmelCase        = self.encoder[line_token]

        del self.encoder[space_token]
        del self.encoder[line_token]

        _UpperCAmelCase        = collections.OrderedDict(sorted(self.encoder.items()  ,       key=lambda __UpperCamelCase  : x[1]  )  )
        _UpperCAmelCase        = {v: k for k, v in self.encoder.items()}

        _UpperCAmelCase        = WordpieceTokenizer(vocab=self.encoder  ,       unk_token=self.unk_token  )







    @property
    def   _snake_case       (      self      :							Optional[Any]  )		->Optional[Any]:






        '''simple docstring'''






        return self.encoder[self.bod_token]







    @property
    def   _snake_case       (      self      :							Union[str, Any]  )		->Optional[Any]:






        '''simple docstring'''






        return self.encoder[self.eod_token]







    @property
    def   _snake_case       (      self      :							str  )		->Union[str, Any]:






        '''simple docstring'''






        return self.encoder["\n"]







    @property
    def   _snake_case       (      self      :							int  )		->int:






        '''simple docstring'''






        return len(self.encoder  )







    def   _snake_case       (      self      :							int  )		->Optional[Any]:






        '''simple docstring'''






        return dict(self.encoder  ,       **self.added_tokens_encoder  )







    def   _snake_case       (      self      :							Any  ,       __UpperCamelCase      :							int  )		->Optional[int]:






        '''simple docstring'''






        _UpperCAmelCase        = []
        for x in jieba.cut(__UpperCamelCase  ,       cut_all=__UpperCamelCase  ):
            output_tokens.extend(self.wordpiece_tokenizer.tokenize(__UpperCamelCase  )  )
        return output_tokens







    def   _snake_case       (      self      :							Tuple  ,       __UpperCamelCase      :							Dict  ,       **__UpperCamelCase      :							str  )		->Dict:






        '''simple docstring'''






        _UpperCAmelCase        = [i for i in token_ids if i >= 0]
        _UpperCAmelCase        = [
            x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
        ]
        return super()._decode(__UpperCamelCase  ,       **__UpperCamelCase  )







    def   _snake_case       (      self      :							int  ,       __UpperCamelCase      :							int  )		->Union[str, Any]:






        '''simple docstring'''






        return token in self.encoder







    def   _snake_case       (      self      :							Optional[Any]  ,       __UpperCamelCase      :							List[str]  )		->str:






        '''simple docstring'''






        return "".join(__UpperCamelCase  )







    def   _snake_case       (      self      :							Any  ,       __UpperCamelCase      :							Optional[int]  )		->Tuple:






        '''simple docstring'''






        return self.encoder.get(__UpperCamelCase  ,       self.encoder.get(self.unk_token  )  )







    def   _snake_case       (      self      :							Optional[int]  ,       __UpperCamelCase      :							List[Any]  )		->Union[str, Any]:






        '''simple docstring'''






        return self.decoder.get(__UpperCamelCase  ,       self.unk_token  )







    def   _snake_case       (      self      :							List[Any]  ,       __UpperCamelCase      :							str  ,       __UpperCamelCase      :							Optional[str] = None  )		->Tuple[str]:






        '''simple docstring'''






        if os.path.isdir(__UpperCamelCase  ):
            _UpperCAmelCase        = os.path.join(
                __UpperCamelCase  ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )
        else:
            _UpperCAmelCase        = (filename_prefix + """-""" if filename_prefix else """""") + save_directory
        _UpperCAmelCase        = 0
        if " " in self.encoder:
            _UpperCAmelCase        = self.encoder[""" """]
            del self.encoder[" "]
        if "\n" in self.encoder:
            _UpperCAmelCase        = self.encoder["""\n"""]
            del self.encoder["\n"]
        _UpperCAmelCase        = collections.OrderedDict(sorted(self.encoder.items()  ,       key=lambda __UpperCamelCase  : x[1]  )  )
        with open(__UpperCamelCase  ,       """w"""  ,       encoding="""utf-8"""  ) as writer:
            for token, token_index in self.encoder.items():
                if index != token_index:
                    logger.warning(
                        f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                        """ Please check that the vocabulary is not corrupted!"""  )
                    _UpperCAmelCase        = token_index
                writer.write(token + """\n"""  )
                index += 1
        return (vocab_file,)







    def   _snake_case       (      self      :							List[Any]  ,       __UpperCamelCase      :							List[int]  ,       __UpperCamelCase      :							List[int] = None  )		->List[int]:






        '''simple docstring'''






        if token_ids_a is None:
            return [self.bos_token_id] + token_ids_a
        return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a







    def   _snake_case       (      self      :							Any  ,       __UpperCamelCase      :							List[int]  ,       __UpperCamelCase      :							Optional[List[int]] = None  ,       __UpperCamelCase      :							bool = False  )		->List[int]:






        '''simple docstring'''






        if already_has_special_tokens:
            return super().get_special_tokens_mask(
                token_ids_a=__UpperCamelCase  ,       token_ids_a=__UpperCamelCase  ,       already_has_special_tokens=__UpperCamelCase  )

        if token_ids_a is not None:
            return [1] + ([0] * len(__UpperCamelCase  )) + [1] + ([0] * len(__UpperCamelCase  ))
        return [1] + ([0] * len(__UpperCamelCase  ))

"""simple docstring"""



import warnings
from functools import wraps
from typing import Callable


def 			_UpperCamelCase						(    _A   )     ->	Callable:





    """simple docstring"""

    @wraps(_A   )
    def _inner_fn(*_A       ,   **_A   ):
        warnings.warn(
            (F"""'{fn.__name__}' is experimental and might be subject to breaking changes in the future.""")       ,   _A       ,   )
        return fn(*_A       ,   **_A   )

    return _inner_fn