


'''simple docstring'''

from unittest.mock import Mock, patch

from file_transfer.send_file import send_file

@patch('''socket.socket'''			)
@patch('''builtins.open'''			)
def _SCREAMING_SNAKE_CASE (A    ,					A			)			->			int:






       """simple docstring"""







       lowercase__      =					Mock()
       lowercase__      =					conn, Mock()
       lowercase__      =					iter([1, None]			)
       lowercase__      =					lambda A			: next(A			)

       # ===== invoke =====
       send_file(filename='''mytext.txt'''    ,					testing=A			)

       # ===== ensurance =====
       sock.assert_called_once()
       sock.return_value.bind.assert_called_once()
       sock.return_value.listen.assert_called_once()
       sock.return_value.accept.assert_called_once()
       conn.recv.assert_called_once()

       file.return_value.__enter__.assert_called_once()
       file.return_value.__enter__.return_value.read.assert_called()

       conn.send.assert_called_once()
       conn.close.assert_called_once()
       sock.return_value.shutdown.assert_called_once()
       sock.return_value.close.assert_called_once()








'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCamelCase				:  str    		=      logging.get_logger(__name__)

lowerCamelCase				:  str    		=      {'vocab_file': 'spiece.model'}

lowerCamelCase				:  int    		=      {
    'vocab_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model',
    }
}
class     __lowerCAmelCase		(lowercase_	):





      '''simple docstring'''






      def __init__(self       :       List[Any]  ,		UpperCamelCase       :       Optional[Any]  ,		UpperCamelCase       :       Any=False  ,		UpperCamelCase       :       Dict=True  ,		UpperCamelCase       :       int=False  ,		UpperCamelCase       :       Optional[Any]="<s>"  ,		UpperCamelCase       :       Union[str, Any]="</s>"  ,		UpperCamelCase       :       Optional[int]="<unk>"  ,		UpperCamelCase       :       List[Any]="<sep>"  ,		UpperCamelCase       :       Dict="<pad>"  ,		UpperCamelCase       :       int="<cls>"  ,		UpperCamelCase       :       int="<mask>"  ,		UpperCamelCase       :       str=["<eop>", "<eod>"]  ,		UpperCamelCase       :       Optional[Dict[str, Any]] = None  ,		**UpperCamelCase       :       Dict  ,		):

             '''simple docstring'''







             lowercase__      =					AddedToken(UpperCamelCase  ,		lstrip=UpperCamelCase  ,		rstrip=UpperCamelCase					) if isinstance(UpperCamelCase  ,		UpperCamelCase					) else mask_token

             lowercase__      =					{} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 do_lower_case=UpperCamelCase  ,		remove_space=UpperCamelCase  ,		keep_accents=UpperCamelCase  ,		bos_token=UpperCamelCase  ,		eos_token=UpperCamelCase  ,		unk_token=UpperCamelCase  ,		sep_token=UpperCamelCase  ,		pad_token=UpperCamelCase  ,		cls_token=UpperCamelCase  ,		mask_token=UpperCamelCase  ,		additional_special_tokens=UpperCamelCase  ,		sp_model_kwargs=self.sp_model_kwargs  ,		**UpperCamelCase  ,		)

             lowercase__      =					3

             lowercase__      =					do_lower_case
             lowercase__      =					remove_space
             lowercase__      =					keep_accents
             lowercase__      =					vocab_file

             lowercase__      =					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
             self.sp_model.Load(UpperCamelCase					)

             try:
                    import jieba
             except ModuleNotFoundError as error:
                    raise error.__class__(
                        '''You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '''
                        '''See https://pypi.org/project/jieba/ for installation.'''					)
             lowercase__      =					jieba
             lowercase__      =					str.maketrans(''' \n'''  ,		'''\u2582\u2583'''					)



      @property
      # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
      def  UpperCamelCase__						(self       :       List[str]					):

             '''simple docstring'''







             return len(self.sp_model					)



      def  UpperCamelCase__						(self       :       List[Any]					):

             '''simple docstring'''







             lowercase__      =					{self.convert_ids_to_tokens(UpperCamelCase					): i for i in range(self.vocab_size					)}
             vocab.update(self.added_tokens_encoder					)
             return vocab



      def __getstate__(self       :       Union[str, Any]					):

             '''simple docstring'''







             lowercase__      =					self.__dict__.copy()
             lowercase__      =					None
             return state



      def __setstate__(self       :       Dict  ,		UpperCamelCase       :       int					):

             '''simple docstring'''







             lowercase__      =					d

             # for backward compatibility
             if not hasattr(self  ,		'''sp_model_kwargs'''					):
                    lowercase__      =					{}

             lowercase__      =					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
             self.sp_model.Load(self.vocab_file					)



      def  UpperCamelCase__						(self       :       Optional[int]  ,		UpperCamelCase       :       Dict					):

             '''simple docstring'''







             if self.remove_space:
                    lowercase__      =					''' '''.join(inputs.strip().split()					)
             else:
                    lowercase__      =					inputs
             lowercase__      =					outputs.replace('''``'''  ,		'''"'''					).replace('''\'\''''  ,		'''"'''					)

             if not self.keep_accents:
                    lowercase__      =					unicodedata.normalize('''NFKD'''  ,		UpperCamelCase					)
                    lowercase__      =					''''''.join([c for c in outputs if not unicodedata.combining(UpperCamelCase					)]					)
             if self.do_lower_case:
                    lowercase__      =					outputs.lower()

             return outputs



      def  UpperCamelCase__						(self       :       List[Any]  ,		UpperCamelCase       :       str					):

             '''simple docstring'''







             lowercase__      =					self.preprocess_text(UpperCamelCase					)
             lowercase__      =					self.sp_model.encode(UpperCamelCase  ,		out_type=UpperCamelCase					)
             lowercase__      =					[]
             for piece in pieces:
                    if len(UpperCamelCase					) > 1 and piece[-1] == str(''','''					) and piece[-2].isdigit():
                           lowercase__      =					self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase  ,		''''''					)					)
                           if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                  if len(cur_pieces[0]					) == 1:
                                         lowercase__      =					cur_pieces[1:]
                                  else:
                                         lowercase__      =					cur_pieces[0][1:]
                           cur_pieces.append(piece[-1]					)
                           new_pieces.extend(UpperCamelCase					)
                    else:
                           new_pieces.append(UpperCamelCase					)

             return new_pieces



      def  UpperCamelCase__						(self       :       int  ,		UpperCamelCase       :       str					):

             '''simple docstring'''







             return self.sp_model.PieceToId(UpperCamelCase					)



      def  UpperCamelCase__						(self       :       str  ,		UpperCamelCase       :       Optional[Any]					):

             '''simple docstring'''







             return self.sp_model.IdToPiece(UpperCamelCase					)



      def  UpperCamelCase__						(self       :       Union[str, Any]  ,		UpperCamelCase       :       str					):

             '''simple docstring'''







             lowercase__      =					''''''.join(UpperCamelCase					).replace(UpperCamelCase  ,		''' '''					).strip()
             return out_string



      def  UpperCamelCase__						(self       :       Any  ,		UpperCamelCase       :       List[int]  ,		UpperCamelCase       :       Optional[List[int]] = None					):

             '''simple docstring'''







             lowercase__      =					[self.sep_token_id]
             lowercase__      =					[self.cls_token_id]
             if token_ids_a is None:
                    return token_ids_a + sep + cls
             return token_ids_a + sep + token_ids_a + sep + cls



      def  UpperCamelCase__						(self       :       Tuple  ,		UpperCamelCase       :       List[int]  ,		UpperCamelCase       :       Optional[List[int]] = None  ,		UpperCamelCase       :       bool = False					):

             '''simple docstring'''







             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=UpperCamelCase  ,		token_ids_a=UpperCamelCase  ,		already_has_special_tokens=UpperCamelCase					)

             if token_ids_a is not None:
                    return ([0] * len(UpperCamelCase					)) + [1] + ([0] * len(UpperCamelCase					)) + [1, 1]
             return ([0] * len(UpperCamelCase					)) + [1, 1]



      def  UpperCamelCase__						(self       :       str  ,		UpperCamelCase       :       List[int]  ,		UpperCamelCase       :       Optional[List[int]] = None					):

             '''simple docstring'''







             lowercase__      =					[self.sep_token_id]
             lowercase__      =					[2]

             if token_ids_a is None:
                    return len(token_ids_a + sep					) * [0] + cls_segment_id
             return len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1] + cls_segment_id



      def  UpperCamelCase__						(self       :       Tuple  ,		UpperCamelCase       :       str  ,		UpperCamelCase       :       Optional[str] = None					):

             '''simple docstring'''







             if not os.path.isdir(UpperCamelCase					):
                    logger.error(f"Vocabulary path ({save_directory}) should be a directory"					)
                    return
             lowercase__      =					os.path.join(
                 UpperCamelCase  ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

             if os.path.abspath(self.vocab_file					) != os.path.abspath(UpperCamelCase					) and os.path.isfile(self.vocab_file					):
                    copyfile(self.vocab_file  ,		UpperCamelCase					)
             elif not os.path.isfile(self.vocab_file					):
                    with open(UpperCamelCase  ,		'''wb'''					) as fi:
                           lowercase__      =					self.sp_model.serialized_model_proto()
                           fi.write(UpperCamelCase					)

             return (out_vocab_file,)






      def  UpperCamelCase__						(self       :       List[str]  ,		*UpperCamelCase       :       List[Any]  ,		**UpperCamelCase       :       Union[str, Any]					):

             '''simple docstring'''







             lowercase__      =					super()._decode(*UpperCamelCase  ,		**UpperCamelCase					)
             lowercase__      =					text.replace(''' '''  ,		''''''					).replace('''\u2582'''  ,		''' '''					).replace('''\u2583'''  ,		'''\n'''					)
             return text




