





"""simple docstring"""




import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
      import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowercase_          =		logging.get_logger(__name__)

lowercase_          =		{"""vocab_file""": """vocab.txt"""}

lowercase_          =		{
    """vocab_file""": {
        """openbmb/cpm-ant-10b""": """https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt""",
    },
}

lowercase_          =		{
    """openbmb/cpm-ant-10b""": 1_024,
}




def       __UpperCamelCase  (_SCREAMING_SNAKE_CASE		)    ->       Optional[Any]:
  lowercase__           =  collections.OrderedDict()
  with open(_A		,						'r'		,						encoding='utf-8'		) as reader:
    lowercase__           =  reader.readlines()
  for index, token in enumerate(_A		):
    lowercase__           =  token.rstrip('\n'		)
    lowercase__           =  index
  return vocab
class     SCREAMING_SNAKE_CASE       (SCREAMING_SNAKE_CASE_				):


   def __init__(   self      :    Optional[Any]      ,					a      :    Tuple      ,					a      :    Tuple="<unk>"      ,					a      :    Dict=200	)->       Any:





     """simple docstring"""







     lowercase__           =  vocab
     lowercase__           =  unk_token
     lowercase__           =  max_input_chars_per_word


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[Any]      ,					a      :    Optional[int]	)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  list(snake_case__	)
     if len(snake_case__	) > self.max_input_chars_per_word:
       return [self.unk_token]

     lowercase__           =  0
     lowercase__           =  []
     while start < len(snake_case__	):
       lowercase__           =  len(snake_case__	)
       lowercase__           =  None
       while start < end:
         lowercase__           =  ''.join(chars[start:end]	)
         if substr in self.vocab:
           lowercase__           =  substr
           break
         end -= 1
       if cur_substr is None:
         sub_tokens.append(self.unk_token	)
         start += 1
       else:
         sub_tokens.append(snake_case__	)
         lowercase__           =  end

     return sub_tokens






class     SCREAMING_SNAKE_CASE       (SCREAMING_SNAKE_CASE_				):
   _UpperCamelCase   :		Tuple			 =	VOCAB_FILES_NAMES
   _UpperCamelCase   :		int			 =	PRETRAINED_VOCAB_FILES_MAP
   _UpperCamelCase   :		Union[str, Any]			 =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCamelCase   :		Any			 =	['input_ids', 'attention_mask']
   _UpperCamelCase   :		Dict			 =	False


   def __init__(   self      :    str      ,					a      :    List[Any]      ,					a      :    Tuple="<d>"      ,					a      :    Optional[Any]="</d>"      ,					a      :    Any="<s>"      ,					a      :    str="</s>"      ,					a      :    int="<pad>"      ,					a      :    int="<unk>"      ,					a      :    Dict="</n>"      ,					a      :    Tuple="</_>"      ,					a      :    int="left"      ,					**a      :    Dict      ,					)->       str:





     """simple docstring"""







     requires_backends(self      ,					['jieba']	)
     super().__init__(
         bod_token=snake_case__      ,					eod_token=snake_case__      ,					bos_token=snake_case__      ,					eos_token=snake_case__      ,					pad_token=snake_case__      ,					unk_token=snake_case__      ,					line_token=snake_case__      ,					space_token=snake_case__      ,					padding_side=snake_case__      ,					**snake_case__      ,					)
     lowercase__           =  bod_token
     lowercase__           =  eod_token
     lowercase__           =  load_vocab(snake_case__	)
     lowercase__           =  self.encoder[space_token]
     lowercase__           =  self.encoder[line_token]

     del self.encoder[space_token]
     del self.encoder[line_token]

     lowercase__           =  collections.OrderedDict(sorted(self.encoder.items()      ,					key=lambda a	: x[1]	)	)
     lowercase__           =  {v: k for k, v in self.encoder.items()}

     lowercase__           =  WordpieceTokenizer(vocab=self.encoder      ,					unk_token=self.unk_token	)


   @property
   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]	)->       Optional[Any]:





     """simple docstring"""







     return self.encoder[self.bod_token]


   @property
   def 	SCREAMING_SNAKE_CASE_							(   self      :    Union[str, Any]	)->       Optional[int]:





     """simple docstring"""







     return self.encoder[self.eod_token]


   @property
   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple	)->       Optional[Any]:





     """simple docstring"""







     return self.encoder["\n"]


   @property
   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple	)->       Dict:





     """simple docstring"""







     return len(self.encoder	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict	)->       int:





     """simple docstring"""







     return dict(self.encoder      ,					**self.added_tokens_encoder	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[Any]      ,					a      :    Union[str, Any]	)->       List[str]:





     """simple docstring"""







     lowercase__           =  []
     for x in jieba.cut(snake_case__      ,					cut_all=snake_case__	):
       output_tokens.extend(self.wordpiece_tokenizer.tokenize(snake_case__	)	)
     return output_tokens


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]      ,					a      :    Dict      ,					**a      :    Union[str, Any]	)->       Tuple:





     """simple docstring"""







     lowercase__           =  [i for i in token_ids if i >= 0]
     lowercase__           =  [
         x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
     ]
     return super()._decode(snake_case__      ,					**snake_case__	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]      ,					a      :    int	)->       int:





     """simple docstring"""







     return token in self.encoder


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[Any]      ,					a      :    int	)->       Any:





     """simple docstring"""







     return "".join(snake_case__	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    List[str]	)->       Optional[Any]:





     """simple docstring"""







     return self.encoder.get(snake_case__      ,					self.encoder.get(self.unk_token	)	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int      ,					a      :    Any	)->       Any:





     """simple docstring"""







     return self.decoder.get(snake_case__      ,					self.unk_token	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[str]      ,					a      :    List[str]      ,					a      :    Union[str, Any] = None	)->       Dict:





     """simple docstring"""







     if os.path.isdir(snake_case__	):
       lowercase__           =  os.path.join(
           snake_case__      ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
     else:
       lowercase__           =  (filename_prefix + '-' if filename_prefix else '') + save_directory
     lowercase__           =  0
     if " " in self.encoder:
       lowercase__           =  self.encoder[' ']
       del self.encoder[" "]
     if "\n" in self.encoder:
       lowercase__           =  self.encoder['\n']
       del self.encoder["\n"]
     lowercase__           =  collections.OrderedDict(sorted(self.encoder.items()      ,					key=lambda a	: x[1]	)	)
     with open(snake_case__      ,					'w'      ,					encoding='utf-8'	) as writer:
       for token, token_index in self.encoder.items():
         if index != token_index:
           logger.warning(
               f"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
               ' Please check that the vocabulary is not corrupted!'	)
           lowercase__           =  token_index
         writer.write(token + '\n'	)
         index += 1
     return (vocab_file,)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    str      ,					a      :    Union[str, Any] = None	)->       int:





     """simple docstring"""







     if token_ids_a is None:
       return [self.bos_token_id] + token_ids_a
     return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int      ,					a      :    str      ,					a      :    Dict = None      ,					a      :    Tuple = False	)->       Any:





     """simple docstring"""







     if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=snake_case__      ,					token_ids_a=snake_case__      ,					already_has_special_tokens=snake_case__	)

     if token_ids_a is not None:
       return [1] + ([0] * len(snake_case__	)) + [1] + ([0] * len(snake_case__	))
     return [1] + ([0] * len(snake_case__	))









from collections.abc import Iterator, MutableMapping
from dataclasses import dataclass
from typing import Generic, TypeVar

lowercase_          =		TypeVar("""KEY""")
lowercase_          =		TypeVar("""VAL""")
@dataclass(frozen=UpperCAmelCase      ,     slots=UpperCAmelCase				)
class     SCREAMING_SNAKE_CASE       (Generic[KEY, VAL]				):
   _UpperCamelCase   :		KEY
   _UpperCamelCase   :		VAL






class     SCREAMING_SNAKE_CASE       (_Item				):


   def __init__(   self      :    Optional[int]	)->       None:





     """simple docstring"""







     super().__init__(a      ,					a	)


   def __bool__(   self      :    str	)->       bool:





     """simple docstring"""







     return False






lowercase_          =		_DeletedItem()
class     SCREAMING_SNAKE_CASE       (MutableMapping[KEY, VAL]				):


   def __init__(   self      :    Tuple      ,					a      :    int = 8      ,					a      :    float = 0.75	)->       None:





     """simple docstring"""







     lowercase__           =  initial_block_size
     lowercase__           =  [None] * initial_block_size
     assert 0.0 < capacity_factor < 1.0
     lowercase__           =  capacity_factor
     lowercase__           =  0


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    KEY	)->       int:





     """simple docstring"""







     return hash(a	) % len(self._buckets	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[Any]      ,					a      :    int	)->       int:





     """simple docstring"""







     return (ind + 1) % len(self._buckets	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    int      ,					a      :    KEY      ,					a      :    VAL	)->       bool:





     """simple docstring"""







     lowercase__           =  self._buckets[ind]
     if not stored:
       lowercase__           =  _Item(a      ,					a	)
       self._len += 1
       return True
     elif stored.key == key:
       lowercase__           =  _Item(a      ,					a	)
       return True
     else:
       return False


   def 	SCREAMING_SNAKE_CASE_							(   self      :    str	)->       bool:





     """simple docstring"""







     lowercase__           =  len(self._buckets	) * self._capacity_factor
     return len(self	) >= int(a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple	)->       bool:





     """simple docstring"""







     if len(self._buckets	) <= self._initial_block_size:
       return False
     lowercase__           =  len(self._buckets	) * self._capacity_factor / 2
     return len(self	) < limit


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[str]      ,					a      :    int	)->       None:





     """simple docstring"""







     lowercase__           =  self._buckets
     lowercase__           =  [None] * new_size
     lowercase__           =  0
     for item in old_buckets:
       if item:
         self._add_item(item.key      ,					item.val	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]	)->       None:





     """simple docstring"""







     self._resize(len(self._buckets	) * 2	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]	)->       None:





     """simple docstring"""







     self._resize(len(self._buckets	) // 2	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    KEY	)->       Iterator[int]:





     """simple docstring"""







     lowercase__           =  self._get_bucket_index(a	)
     for _ in range(len(self._buckets	)	):
       yield ind
       lowercase__           =  self._get_next_ind(a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple      ,					a      :    KEY      ,					a      :    VAL	)->       None:





     """simple docstring"""







     for ind in self._iterate_buckets(a	):
       if self._try_set(a      ,					a      ,					a	):
         break


   def __setitem__(   self      :    List[Any]      ,					a      :    KEY      ,					a      :    VAL	)->       None:





     """simple docstring"""







     if self._is_full():
       self._size_up()

     self._add_item(a      ,					a	)


   def __delitem__(   self      :    str      ,					a      :    KEY	)->       None:





     """simple docstring"""







     for ind in self._iterate_buckets(a	):
       lowercase__           =  self._buckets[ind]
       if item is None:
         raise KeyError(a	)
       if item is _deleted:
         continue
       if item.key == key:
         lowercase__           =  _deleted
         self._len -= 1
         break
     if self._is_sparse():
       self._size_down()


   def __getitem__(   self      :    List[str]      ,					a      :    KEY	)->       VAL:





     """simple docstring"""







     for ind in self._iterate_buckets(a	):
       lowercase__           =  self._buckets[ind]
       if item is None:
         break
       if item is _deleted:
         continue
       if item.key == key:
         return item.val
     raise KeyError(a	)


   def __len__(   self      :    Tuple	)->       int:





     """simple docstring"""







     return self._len


   def __iter__(   self      :    int	)->       Iterator[KEY]:





     """simple docstring"""







     yield from (item.key for item in self._buckets if item)


   def __repr__(   self      :    Union[str, Any]	)->       str:





     """simple docstring"""







     lowercase__           =  ' ,'.join(
         f"""{item.key}: {item.val}""" for item in self._buckets if item	)
     return f"""HashMap({val_string})"""


