


"""simple docstring"""



import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
  import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case       =       logging.get_logger(__name__)

__snake_case       =       {"""vocab_file""": """vocab.txt"""}

__snake_case       =       {
    """vocab_file""": {
        """openbmb/cpm-ant-10b""": """https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt""",
    },
}

__snake_case       =       {
    """openbmb/cpm-ant-10b""": 1024,
}







def     __lowerCAmelCase		(		lowercase    :    int					)      ->		Optional[int]:


    """simple docstring"""





    snake_case       : Optional[Any]    =       collections.OrderedDict()
    with open(lowercase		,				"r"		,				encoding="utf-8"					) as reader:
        snake_case       : List[Any]    =       reader.readlines()
    for index, token in enumerate(lowercase					):
        snake_case       : Dict    =       token.rstrip("\n"					)
        snake_case       : str    =       index
    return vocab







class _lowerCAmelCase							( snake_case_      ):







   def __init__(      self		,      UpperCamelCase__		,      UpperCamelCase__="<unk>"		,      UpperCamelCase__=200		)      ->						Union[str, Any]:

       '''simple docstring'''




       snake_case       : Dict    =       vocab
       snake_case       : Union[str, Any]    =       unk_token
       snake_case       : int    =       max_input_chars_per_word







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						Optional[Any]:

       '''simple docstring'''




       snake_case       : List[str]    =       list(UpperCamelCase__		)
       if len(UpperCamelCase__		) > self.max_input_chars_per_word:
           return [self.unk_token]

       snake_case       : Optional[Any]    =       0
       snake_case       : Tuple    =       []
       while start < len(UpperCamelCase__		):
           snake_case       : Union[str, Any]    =       len(UpperCamelCase__		)
           snake_case       : Tuple    =       None
           while start < end:
               snake_case       : Optional[int]    =       "".join(chars[start:end]		)
               if substr in self.vocab:
                   snake_case       : List[str]    =       substr
                   break
               end -= 1
           if cur_substr is None:
               sub_tokens.append(self.unk_token		)
               start += 1
           else:
               sub_tokens.append(UpperCamelCase__		)
               snake_case       : str    =       end

       return sub_tokens







class _lowerCAmelCase							( snake_case_      ):
   __UpperCAmelCase					:			Any				    =    VOCAB_FILES_NAMES
   __UpperCAmelCase					:			Optional[int]				    =    PRETRAINED_VOCAB_FILES_MAP
   __UpperCAmelCase					:			List[Any]				    =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCAmelCase					:			Optional[int]				    =    ['''input_ids''', '''attention_mask''']
   __UpperCAmelCase					:			List[Any]				    =    False







   def __init__(      self		,      UpperCamelCase__		,      UpperCamelCase__="<d>"		,      UpperCamelCase__="</d>"		,      UpperCamelCase__="<s>"		,      UpperCamelCase__="</s>"		,      UpperCamelCase__="<pad>"		,      UpperCamelCase__="<unk>"		,      UpperCamelCase__="</n>"		,      UpperCamelCase__="</_>"		,      UpperCamelCase__="left"		,      **UpperCamelCase__		,      )      ->						Any:

       '''simple docstring'''




       requires_backends(self		,      ["jieba"]		)
       super().__init__(
           bod_token=UpperCamelCase__		,      eod_token=UpperCamelCase__		,      bos_token=UpperCamelCase__		,      eos_token=UpperCamelCase__		,      pad_token=UpperCamelCase__		,      unk_token=UpperCamelCase__		,      line_token=UpperCamelCase__		,      space_token=UpperCamelCase__		,      padding_side=UpperCamelCase__		,      **UpperCamelCase__		,      )
       snake_case       : str    =       bod_token
       snake_case       : Optional[Any]    =       eod_token
       snake_case       : Union[str, Any]    =       load_vocab(UpperCamelCase__		)
       snake_case       : Optional[Any]    =       self.encoder[space_token]
       snake_case       : Optional[int]    =       self.encoder[line_token]

       del self.encoder[space_token]
       del self.encoder[line_token]

       snake_case       : List[Any]    =       collections.OrderedDict(sorted(self.encoder.items()		,      key=lambda UpperCamelCase__		: x[1]		)		)
       snake_case       : Dict    =       {v: k for k, v in self.encoder.items()}

       snake_case       : List[Any]    =       WordpieceTokenizer(vocab=self.encoder		,      unk_token=self.unk_token		)







   @property
   def 					lowerCamelCase						(      self		)      ->						Tuple:

       '''simple docstring'''




       return self.encoder[self.bod_token]







   @property
   def 					lowerCamelCase						(      self		)      ->						Optional[int]:

       '''simple docstring'''




       return self.encoder[self.eod_token]







   @property
   def 					lowerCamelCase						(      self		)      ->						Any:

       '''simple docstring'''




       return self.encoder["\n"]







   @property
   def 					lowerCamelCase						(      self		)      ->						int:

       '''simple docstring'''




       return len(self.encoder		)







   def 					lowerCamelCase						(      self		)      ->						Tuple:

       '''simple docstring'''




       return dict(self.encoder		,      **self.added_tokens_encoder		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						int:

       '''simple docstring'''




       snake_case       : Any    =       []
       for x in jieba.cut(UpperCamelCase__		,      cut_all=UpperCamelCase__		):
           output_tokens.extend(self.wordpiece_tokenizer.tokenize(UpperCamelCase__		)		)
       return output_tokens







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      **UpperCamelCase__		)      ->						Optional[Any]:

       '''simple docstring'''




       snake_case       : Dict    =       [i for i in token_ids if i >= 0]
       snake_case       : Dict    =       [
           x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
       ]
       return super()._decode(UpperCamelCase__		,      **UpperCamelCase__		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						Optional[Any]:

       '''simple docstring'''




       return token in self.encoder







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						str:

       '''simple docstring'''




       return "".join(UpperCamelCase__		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						Optional[Any]:

       '''simple docstring'''




       return self.encoder.get(UpperCamelCase__		,      self.encoder.get(self.unk_token		)		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		)      ->						Tuple:

       '''simple docstring'''




       return self.decoder.get(UpperCamelCase__		,      self.unk_token		)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__ = None		)      ->						Tuple[str]:

       '''simple docstring'''




       if os.path.isdir(UpperCamelCase__		):
           snake_case       : List[Any]    =       os.path.join(
               UpperCamelCase__		,      (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
       else:
           snake_case       : List[Any]    =       (filename_prefix + "-" if filename_prefix else "") + save_directory
       snake_case       : List[str]    =       0
       if " " in self.encoder:
           snake_case       : Any    =       self.encoder[" "]
           del self.encoder[" "]
       if "\n" in self.encoder:
           snake_case       : List[str]    =       self.encoder["\n"]
           del self.encoder["\n"]
       snake_case       : Optional[Any]    =       collections.OrderedDict(sorted(self.encoder.items()		,      key=lambda UpperCamelCase__		: x[1]		)		)
       with open(UpperCamelCase__		,      "w"		,      encoding="utf-8"		) as writer:
           for token, token_index in self.encoder.items():
               if index != token_index:
                   logger.warning(
                       F'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
                       " Please check that the vocabulary is not corrupted!"		)
                   snake_case       : Any    =       token_index
               writer.write(token + "\n"		)
               index += 1
       return (vocab_file,)







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__ = None		)      ->						List[int]:

       '''simple docstring'''




       if token_ids_a is None:
           return [self.bos_token_id] + token_ids_a
       return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a







   def 					lowerCamelCase						(      self		,      UpperCamelCase__		,      UpperCamelCase__ = None		,      UpperCamelCase__ = False		)      ->						List[int]:

       '''simple docstring'''




       if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=UpperCamelCase__		,      token_ids_a=UpperCamelCase__		,      already_has_special_tokens=UpperCamelCase__		)

       if token_ids_a is not None:
           return [1] + ([0] * len(UpperCamelCase__		)) + [1] + ([0] * len(UpperCamelCase__		))
       return [1] + ([0] * len(UpperCamelCase__		))









"""simple docstring"""



from .imports import is_tqdm_available


if is_tqdm_available():
  from tqdm.auto import tqdm as _tqdm

from ..state import PartialState







def     __lowerCAmelCase		(		lowercase    :    bool = True		,				*lowercase    :    str		,				**lowercase    :    List[Any]					)      ->		List[Any]:


    """simple docstring"""





    if not is_tqdm_available():
        raise ImportError("Accelerate's `tqdm` module requires `tqdm` to be installed. Please run `pip install tqdm`."					)
    snake_case       : Dict    =       False
    if main_process_only:
        snake_case       : int    =       PartialState().local_process_index == 0
    return _tqdm(*lowercase		,				**lowercase		,				disable=lowercase					)





