






'''simple docstring'''







import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


__lowerCAmelCase        =							logging.get_logger(__name__)

__lowerCAmelCase        =							{"vocab_file": "spiece.model"}

__lowerCAmelCase        =							{
    "vocab_file": {
        "TsinghuaAI/CPM-Generate": "https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model",
    }
}




class   __SCREAMING_SNAKE_CASE       (__A       ):


  """simple docstring"""


  def __init__(			self   ,	UpperCamelCase__   ,	UpperCamelCase__=False   ,	UpperCamelCase__=True   ,	UpperCamelCase__=False   ,	UpperCamelCase__="<s>"   ,	UpperCamelCase__="</s>"   ,	UpperCamelCase__="<unk>"   ,	UpperCamelCase__="<sep>"   ,	UpperCamelCase__="<pad>"   ,	UpperCamelCase__="<cls>"   ,	UpperCamelCase__="<mask>"   ,	UpperCamelCase__=["<eop>", "<eod>"]   ,	UpperCamelCase__ = None   ,	**UpperCamelCase__   ,	):






         """simple docstring"""


         a_						     =			AddedToken(lowercase_   ,	lstrip=lowercase_   ,	rstrip=lowercase_	) if isinstance(lowercase_   ,	lowercase_	) else mask_token

         a_						     =			{} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             do_lower_case=lowercase_   ,	remove_space=lowercase_   ,	keep_accents=lowercase_   ,	bos_token=lowercase_   ,	eos_token=lowercase_   ,	unk_token=lowercase_   ,	sep_token=lowercase_   ,	pad_token=lowercase_   ,	cls_token=lowercase_   ,	mask_token=lowercase_   ,	additional_special_tokens=lowercase_   ,	sp_model_kwargs=self.sp_model_kwargs   ,	**lowercase_   ,	)

         a_						     =			3

         a_						     =			do_lower_case
         a_						     =			remove_space
         a_						     =			keep_accents
         a_						     =			vocab_file

         a_						     =			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
         self.sp_model.Load(lowercase_	)

         try:
                import jieba
         except ModuleNotFoundError as error:
                raise error.__class__(
                    'You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '
                    'See https://pypi.org/project/jieba/ for installation.'	)
         a_						     =			jieba
         a_						     =			str.maketrans(' \n'   ,	'\u2582\u2583'	)







  @property
  # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
  def    _a       (			self	):






         """simple docstring"""


         return len(self.sp_model	)







  def    _a       (			self	):






         """simple docstring"""


         a_						     =			{self.convert_ids_to_tokens(lowercase_	): i for i in range(self.vocab_size	)}
         vocab.update(self.added_tokens_encoder	)
         return vocab







  def __getstate__(			self	):






         """simple docstring"""


         a_						     =			self.__dict__.copy()
         a_						     =			None
         return state







  def __setstate__(			self   ,	UpperCamelCase__	):






         """simple docstring"""


         a_						     =			d

         # for backward compatibility
         if not hasattr(self   ,	'sp_model_kwargs'	):
                a_						     =			{}

         a_						     =			spm.SentencePieceProcessor(**self.sp_model_kwargs	)
         self.sp_model.Load(self.vocab_file	)







  def    _a       (			self   ,	UpperCamelCase__	):






         """simple docstring"""


         if self.remove_space:
                a_						     =			' '.join(inputs.strip().split()	)
         else:
                a_						     =			inputs
         a_						     =			outputs.replace('``'   ,	'\"'	).replace('\'\''   ,	'\"'	)

         if not self.keep_accents:
                a_						     =			unicodedata.normalize('NFKD'   ,	lowercase_	)
                a_						     =			''.join([c for c in outputs if not unicodedata.combining(lowercase_	)]	)
         if self.do_lower_case:
                a_						     =			outputs.lower()

         return outputs







  def    _a       (			self   ,	UpperCamelCase__	):






         """simple docstring"""


         a_						     =			self.preprocess_text(lowercase_	)
         a_						     =			self.sp_model.encode(lowercase_   ,	out_type=lowercase_	)
         a_						     =			[]
         for piece in pieces:
                if len(lowercase_	) > 1 and piece[-1] == str(','	) and piece[-2].isdigit():
                       a_						     =			self.sp_model.EncodeAsPieces(piece[:-1].replace(lowercase_   ,	''	)	)
                       if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                              if len(cur_pieces[0]	) == 1:
                                     a_						     =			cur_pieces[1:]
                              else:
                                     a_						     =			cur_pieces[0][1:]
                       cur_pieces.append(piece[-1]	)
                       new_pieces.extend(lowercase_	)
                else:
                       new_pieces.append(lowercase_	)

         return new_pieces







  def    _a       (			self   ,	UpperCamelCase__	):






         """simple docstring"""


         return self.sp_model.PieceToId(lowercase_	)







  def    _a       (			self   ,	UpperCamelCase__	):






         """simple docstring"""


         return self.sp_model.IdToPiece(lowercase_	)







  def    _a       (			self   ,	UpperCamelCase__	):






         """simple docstring"""


         a_						     =			''.join(lowercase_	).replace(lowercase_   ,	' '	).strip()
         return out_string







  def    _a       (			self   ,	UpperCamelCase__   ,	UpperCamelCase__ = None	):






         """simple docstring"""


         a_						     =			[self.sep_token_id]
         a_						     =			[self.cls_token_id]
         if token_ids_a is None:
                return token_ids_a + sep + cls
         return token_ids_a + sep + token_ids_a + sep + cls







  def    _a       (			self   ,	UpperCamelCase__   ,	UpperCamelCase__ = None   ,	UpperCamelCase__ = False	):






         """simple docstring"""


         if already_has_special_tokens:
                return super().get_special_tokens_mask(
                    token_ids_a=lowercase_   ,	token_ids_a=lowercase_   ,	already_has_special_tokens=lowercase_	)

         if token_ids_a is not None:
                return ([0] * len(lowercase_	)) + [1] + ([0] * len(lowercase_	)) + [1, 1]
         return ([0] * len(lowercase_	)) + [1, 1]







  def    _a       (			self   ,	UpperCamelCase__   ,	UpperCamelCase__ = None	):






         """simple docstring"""


         a_						     =			[self.sep_token_id]
         a_						     =			[2]

         if token_ids_a is None:
                return len(token_ids_a + sep	) * [0] + cls_segment_id
         return len(token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1] + cls_segment_id







  def    _a       (			self   ,	UpperCamelCase__   ,	UpperCamelCase__ = None	):






         """simple docstring"""


         if not os.path.isdir(lowercase_	):
                logger.error(f'Vocabulary path ({save_directory}) should be a directory'	)
                return
         a_						     =			os.path.join(
             lowercase_   ,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)

         if os.path.abspath(self.vocab_file	) != os.path.abspath(lowercase_	) and os.path.isfile(self.vocab_file	):
                copyfile(self.vocab_file   ,	lowercase_	)
         elif not os.path.isfile(self.vocab_file	):
                with open(lowercase_   ,	'wb'	) as fi:
                       a_						     =			self.sp_model.serialized_model_proto()
                       fi.write(lowercase_	)

         return (out_vocab_file,)







  def    _a       (			self   ,	*UpperCamelCase__   ,	**UpperCamelCase__	):






         """simple docstring"""


         a_						     =			super()._decode(*lowercase_   ,	**lowercase_	)
         a_						     =			text.replace(' '   ,	''	).replace('\u2582'   ,	' '	).replace('\u2583'   ,	'\n'	)
         return text





import enum
import os
from hashlib import shaaaa
from typing import Optional

from .. import config
from .logging import get_logger


lowerCamelCase__  =					get_logger(__name__)




class 					_UpperCAmelCase			(     enum.Enum  ):



 '''simple docstring'''




 __A			       =					'''all_checks'''
 __A			       =					'''basic_checks'''
 __A			       =					'''no_checks'''




class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''








class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''








class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''








class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''






def 				lowerCAmelCase__       (			a__    ,     a__    ,     a__=None )      ->int:




  '''simple docstring'''

  if expected_checksums is None:
    logger.info("Unable to verify checksums." )
    return
  if len(set(a__ ) - set(a__ ) ) > 0:
    raise ExpectedMoreDownloadedFiles(str(set(a__ ) - set(a__ ) ) )
  if len(set(a__ ) - set(a__ ) ) > 0:
    raise UnexpectedDownloadedFile(str(set(a__ ) - set(a__ ) ) )
  _UpperCamelCase           =    [url for url in expected_checksums if expected_checksums[url] != recorded_checksums[url]]
  _UpperCamelCase           =    " for " + verification_name if verification_name is not None else ""
  if len(a__ ) > 0:
    raise NonMatchingChecksumError(
        f'Checksums didn\'t match{for_verification_name}:\n'
        f'{bad_urls}\n'
        "Set `verification_mode='no_checks'` to skip checksums verification and ignore this error" )
  logger.info("All the checksums matched successfully" + for_verification_name )




class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''








class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''








class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''








class 					_UpperCAmelCase			(     lowerCAmelCase  ):



 '''simple docstring'''






def 				lowerCAmelCase__       (			a__    ,     a__ )      ->Dict:




  '''simple docstring'''

  if expected_splits is None:
    logger.info("Unable to verify splits sizes." )
    return
  if len(set(a__ ) - set(a__ ) ) > 0:
    raise ExpectedMoreSplits(str(set(a__ ) - set(a__ ) ) )
  if len(set(a__ ) - set(a__ ) ) > 0:
    raise UnexpectedSplits(str(set(a__ ) - set(a__ ) ) )
  _UpperCamelCase           =    [
      {"expected": expected_splits[name], "recorded": recorded_splits[name]}
      for name in expected_splits
      if expected_splits[name].num_examples != recorded_splits[name].num_examples
  ]
  if len(a__ ) > 0:
    raise NonMatchingSplitsSizesError(str(a__ ) )
  logger.info("All the splits matched successfully." )


def 				lowerCAmelCase__       (			a__    ,     a__ = True )      ->dict:




  '''simple docstring'''

  if record_checksum:
    _UpperCamelCase           =    shaaaa()
    with open(a__    ,     "rb" ) as f:
      for chunk in iter(lambda: f.read(1 << 20 )    ,     B"" ):
        m.update(a__ )
      _UpperCamelCase           =    m.hexdigest()
  else:
    _UpperCamelCase           =    None
  return {"num_bytes": os.path.getsize(a__ ), "checksum": checksum}







def 				lowerCAmelCase__       (			a__ )      ->Any:




  '''simple docstring'''

  if dataset_size and config.IN_MEMORY_MAX_SIZE:
    return dataset_size < config.IN_MEMORY_MAX_SIZE
  else:
    return False




