





"""simple docstring"""




import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 			__UpperCamelCase							(		lowercase_						,			unittest.TestCase		):
							lowerCamelCase  :   Optional[int]    =CodeGenTokenizer
							lowerCamelCase  :   Dict    =CodeGenTokenizerFast
							lowerCamelCase  :   Union[str, Any]    =True
							lowerCamelCase  :   List[Any]    ={"""add_prefix_space""": True}
							lowerCamelCase  :   List[str]    =False






							def        __a  (  self )      ->					str:
								super().setUp()

								# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
								a :						List[Any]      =							[
								    "l",
								    "o",
								    "w",
								    "e",
								    "r",
								    "s",
								    "t",
								    "i",
								    "d",
								    "n",
								    "\u0120",
								    "\u0120l",
								    "\u0120n",
								    "\u0120lo",
								    "\u0120low",
								    "er",
								    "\u0120lowest",
								    "\u0120newer",
								    "\u0120wider",
								    "<unk>",
								    "<|endoftext|>",
								]
								a :						Optional[int]      =							dict(zip(lowerCAmelCase__ ,   range(len(lowerCAmelCase__ ) ) ) )
								a :						Any      =							["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
								a :						Optional[Any]      =							{"unk_token": "<unk>"}

								a :						Optional[Any]      =							os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES["vocab_file"] )
								a :						int      =							os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES["merges_file"] )
								with open(self.vocab_file ,   "w" ,   encoding="utf-8" ) as fp:
									fp.write(json.dumps(lowerCAmelCase__ ) + "\n" )
								with open(self.merges_file ,   "w" ,   encoding="utf-8" ) as fp:
									fp.write("\n".join(lowerCAmelCase__ ) )






							def        __a  (  self ,   **lowerCAmelCase__ )      ->					List[str]:
								kwargs.update(self.special_tokens_map )
								return CodeGenTokenizer.from_pretrained(self.tmpdirname ,   **lowerCAmelCase__ )






							def        __a  (  self ,   **lowerCAmelCase__ )      ->					Union[str, Any]:
								kwargs.update(self.special_tokens_map )
								return CodeGenTokenizerFast.from_pretrained(self.tmpdirname ,   **lowerCAmelCase__ )






							def        __a  (  self ,   lowerCAmelCase__ )      ->					Union[str, Any]:
								a :						Optional[int]      =							"lower newer"
								a :						Union[str, Any]      =							"lower newer"
								return input_text, output_text






							def        __a  (  self )      ->					str:
								a :						str      =							CodeGenTokenizer(self.vocab_file ,   self.merges_file ,   **self.special_tokens_map )
								a :						Dict      =							"lower newer"
								a :						Optional[int]      =							["\u0120low", "er", "\u0120", "n", "e", "w", "er"]
								a :						Optional[int]      =							tokenizer.tokenize(lowerCAmelCase__ ,   add_prefix_space=lowerCAmelCase__ )
								self.assertListEqual(lowerCAmelCase__ ,   lowerCAmelCase__ )

								a :						List[Any]      =							tokens + [tokenizer.unk_token]
								a :						List[str]      =							[14, 15, 10, 9, 3, 2, 15, 19]
								self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__ ) ,   lowerCAmelCase__ )






							def        __a  (  self )      ->					Optional[Any]:







								if not self.test_rust_tokenizer:
									return

								a :						int      =							self.get_tokenizer()
								a :						Union[str, Any]      =							self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase__ )

								a :						List[str]      =							"lower newer"

								# Testing tokenization
								a :						Tuple      =							tokenizer.tokenize(lowerCAmelCase__ ,   add_prefix_space=lowerCAmelCase__ )
								a :						Dict      =							rust_tokenizer.tokenize(lowerCAmelCase__ )
								self.assertListEqual(lowerCAmelCase__ ,   lowerCAmelCase__ )

								# Testing conversion to ids without special tokens
								a :						str      =							tokenizer.encode(lowerCAmelCase__ ,   add_special_tokens=lowerCAmelCase__ ,   add_prefix_space=lowerCAmelCase__ )
								a :						str      =							rust_tokenizer.encode(lowerCAmelCase__ ,   add_special_tokens=lowerCAmelCase__ )
								self.assertListEqual(lowerCAmelCase__ ,   lowerCAmelCase__ )

								# Testing conversion to ids with special tokens
								a :						List[str]      =							self.get_rust_tokenizer(add_prefix_space=lowerCAmelCase__ )
								a :						int      =							tokenizer.encode(lowerCAmelCase__ ,   add_prefix_space=lowerCAmelCase__ )
								a :						Any      =							rust_tokenizer.encode(lowerCAmelCase__ )
								self.assertListEqual(lowerCAmelCase__ ,   lowerCAmelCase__ )

								# Testing the unknown token
								a :						Union[str, Any]      =							tokens + [rust_tokenizer.unk_token]
								a :						Optional[Any]      =							[14, 15, 10, 9, 3, 2, 15, 19]
								self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(lowerCAmelCase__ ) ,   lowerCAmelCase__ )






							def        __a  (  self ,   *lowerCAmelCase__ ,   **lowerCAmelCase__ )      ->					int:
								pass






							def        __a  (  self ,   lowerCAmelCase__=15 )      ->					Optional[int]:







								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
									with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
										a :						int      =							self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__ ,   **lowerCAmelCase__ )

										# Simple input
										a :						List[Any]      =							"This is a simple input"
										a :						Union[str, Any]      =							["This is a simple input 1", "This is a simple input 2"]
										a :						Any      =							("This is a simple input", "This is a pair")
										a :						List[Any]      =							[
										    ("This is a simple input 1", "This is a simple input 2"),
										    ("This is a simple pair 1", "This is a simple pair 2"),
										]

										# Simple input tests
										self.assertRaises(lowerCAmelCase__ ,   tokenizer_r.encode ,   lowerCAmelCase__ ,   max_length=lowerCAmelCase__ ,   padding="max_length" )

										# Simple input
										self.assertRaises(lowerCAmelCase__ ,   tokenizer_r.encode_plus ,   lowerCAmelCase__ ,   max_length=lowerCAmelCase__ ,   padding="max_length" )

										# Simple input
										self.assertRaises(
										    lowerCAmelCase__ ,   tokenizer_r.batch_encode_plus ,   lowerCAmelCase__ ,   max_length=lowerCAmelCase__ ,   padding="max_length" ,   )

										# Pair input
										self.assertRaises(lowerCAmelCase__ ,   tokenizer_r.encode ,   lowerCAmelCase__ ,   max_length=lowerCAmelCase__ ,   padding="max_length" )

										# Pair input
										self.assertRaises(lowerCAmelCase__ ,   tokenizer_r.encode_plus ,   lowerCAmelCase__ ,   max_length=lowerCAmelCase__ ,   padding="max_length" )

										# Pair input
										self.assertRaises(
										    lowerCAmelCase__ ,   tokenizer_r.batch_encode_plus ,   lowerCAmelCase__ ,   max_length=lowerCAmelCase__ ,   padding="max_length" ,   )






							def        __a  (  self )      ->					List[str]:
								a :						List[Any]      =							CodeGenTokenizer.from_pretrained(self.tmpdirname ,   pad_token="<pad>" )

								# Simple input
								a :						Union[str, Any]      =							"This is a simple input"
								a :						str      =							["This is a simple input looooooooong", "This is a simple input"]
								a :						Any      =							("This is a simple input", "This is a pair")
								a :						Tuple      =							[
								    ("This is a simple input loooooong", "This is a simple input"),
								    ("This is a simple pair loooooong", "This is a simple pair"),
								]

								a :						Tuple      =							tokenizer.pad_token_id

								a :						int      =							tokenizer(lowerCAmelCase__ ,   padding="max_length" ,   max_length=30 ,   return_tensors="np" )
								a :						Optional[int]      =							tokenizer(lowerCAmelCase__ ,   padding=lowerCAmelCase__ ,   truncate=lowerCAmelCase__ ,   return_tensors="np" )
								a :						List[str]      =							tokenizer(*lowerCAmelCase__ ,   padding="max_length" ,   max_length=60 ,   return_tensors="np" )
								a :						str      =							tokenizer(lowerCAmelCase__ ,   padding=lowerCAmelCase__ ,   truncate=lowerCAmelCase__ ,   return_tensors="np" )

								# s
								# test single string max_length padding
								self.assertEqual(out_s["input_ids"].shape[-1] ,   30 )
								self.assertTrue(pad_token_id in out_s["input_ids"] )
								self.assertTrue(0 in out_s["attention_mask"] )

								# s2
								# test automatic padding
								self.assertEqual(out_sa["input_ids"].shape[-1] ,   33 )
								# long slice doesn't have padding
								self.assertFalse(pad_token_id in out_sa["input_ids"][0] )
								self.assertFalse(0 in out_sa["attention_mask"][0] )
								# short slice does have padding
								self.assertTrue(pad_token_id in out_sa["input_ids"][1] )
								self.assertTrue(0 in out_sa["attention_mask"][1] )

								# p
								# test single pair max_length padding
								self.assertEqual(out_p["input_ids"].shape[-1] ,   60 )
								self.assertTrue(pad_token_id in out_p["input_ids"] )
								self.assertTrue(0 in out_p["attention_mask"] )

								# p2
								# test automatic padding pair
								self.assertEqual(out_pa["input_ids"].shape[-1] ,   52 )
								# long slice pair doesn't have padding
								self.assertFalse(pad_token_id in out_pa["input_ids"][0] )
								self.assertFalse(0 in out_pa["attention_mask"][0] )
								# short slice pair does have padding
								self.assertTrue(pad_token_id in out_pa["input_ids"][1] )
								self.assertTrue(0 in out_pa["attention_mask"][1] )






							def        __a  (  self )      ->					str:
								a :						Tuple      =							"$$$"
								a :						List[str]      =							CodeGenTokenizer.from_pretrained(self.tmpdirname ,   bos_token=lowerCAmelCase__ ,   add_bos_token=lowerCAmelCase__ )

								a :						Any      =							"This is a simple input"
								a :						Union[str, Any]      =							["This is a simple input 1", "This is a simple input 2"]

								a :						Optional[int]      =							tokenizer.bos_token_id

								a :						Tuple      =							tokenizer(lowerCAmelCase__ )
								a :						List[Any]      =							tokenizer(lowerCAmelCase__ )

								self.assertEqual(out_s.input_ids[0] ,   lowerCAmelCase__ )
								self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids ) )

								a :						Tuple      =							tokenizer.decode(out_s.input_ids )
								a :						Union[str, Any]      =							tokenizer.batch_decode(out_sa.input_ids )

								self.assertEqual(decode_s.split()[0] ,   lowerCAmelCase__ )
								self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa ) )






							@slow
							def        __a  (  self )      ->					Any:
								a :						List[str]      =							CodeGenTokenizer.from_pretrained("Salesforce/codegen-350M-mono" )

								a :						Any      =							"\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#"
								a :						int      =							"\nif len_a > len_b:      result = a\nelse:      result = b"

								a :						Optional[int]      =							tokenizer.encode(lowerCAmelCase__ )
								a :						int      =							["^#", re.escape("<|endoftext|>" ), "^\'\'\'", "^\"\"\"", "\n\n\n"]
								a :						List[Any]      =							tokenizer.decode(lowerCAmelCase__ ,   truncate_before_pattern=lowerCAmelCase__ )
								self.assertEqual(lowerCAmelCase__ ,   lowerCAmelCase__ )






							def        __a  (  self )      ->					List[str]:
								pass






'''simple docstring'''

import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


lowerCamelCase				:  str    		=      logging.get_logger(__name__)

lowerCamelCase				:  str    		=      {'vocab_file': 'spiece.model'}

lowerCamelCase				:  int    		=      {
    'vocab_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model',
    }
}
class     __lowerCAmelCase		(lowercase_	):





      '''simple docstring'''






      def __init__(self       :       List[Any]  ,		UpperCamelCase       :       Optional[Any]  ,		UpperCamelCase       :       Any=False  ,		UpperCamelCase       :       Dict=True  ,		UpperCamelCase       :       int=False  ,		UpperCamelCase       :       Optional[Any]="<s>"  ,		UpperCamelCase       :       Union[str, Any]="</s>"  ,		UpperCamelCase       :       Optional[int]="<unk>"  ,		UpperCamelCase       :       List[Any]="<sep>"  ,		UpperCamelCase       :       Dict="<pad>"  ,		UpperCamelCase       :       int="<cls>"  ,		UpperCamelCase       :       int="<mask>"  ,		UpperCamelCase       :       str=["<eop>", "<eod>"]  ,		UpperCamelCase       :       Optional[Dict[str, Any]] = None  ,		**UpperCamelCase       :       Dict  ,		):

             '''simple docstring'''







             lowercase__      =					AddedToken(UpperCamelCase  ,		lstrip=UpperCamelCase  ,		rstrip=UpperCamelCase					) if isinstance(UpperCamelCase  ,		UpperCamelCase					) else mask_token

             lowercase__      =					{} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 do_lower_case=UpperCamelCase  ,		remove_space=UpperCamelCase  ,		keep_accents=UpperCamelCase  ,		bos_token=UpperCamelCase  ,		eos_token=UpperCamelCase  ,		unk_token=UpperCamelCase  ,		sep_token=UpperCamelCase  ,		pad_token=UpperCamelCase  ,		cls_token=UpperCamelCase  ,		mask_token=UpperCamelCase  ,		additional_special_tokens=UpperCamelCase  ,		sp_model_kwargs=self.sp_model_kwargs  ,		**UpperCamelCase  ,		)

             lowercase__      =					3

             lowercase__      =					do_lower_case
             lowercase__      =					remove_space
             lowercase__      =					keep_accents
             lowercase__      =					vocab_file

             lowercase__      =					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
             self.sp_model.Load(UpperCamelCase					)

             try:
                    import jieba
             except ModuleNotFoundError as error:
                    raise error.__class__(
                        '''You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '''
                        '''See https://pypi.org/project/jieba/ for installation.'''					)
             lowercase__      =					jieba
             lowercase__      =					str.maketrans(''' \n'''  ,		'''\u2582\u2583'''					)



      @property
      # Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
      def  UpperCamelCase__						(self       :       List[str]					):

             '''simple docstring'''







             return len(self.sp_model					)



      def  UpperCamelCase__						(self       :       List[Any]					):

             '''simple docstring'''







             lowercase__      =					{self.convert_ids_to_tokens(UpperCamelCase					): i for i in range(self.vocab_size					)}
             vocab.update(self.added_tokens_encoder					)
             return vocab



      def __getstate__(self       :       Union[str, Any]					):

             '''simple docstring'''







             lowercase__      =					self.__dict__.copy()
             lowercase__      =					None
             return state



      def __setstate__(self       :       Dict  ,		UpperCamelCase       :       int					):

             '''simple docstring'''







             lowercase__      =					d

             # for backward compatibility
             if not hasattr(self  ,		'''sp_model_kwargs'''					):
                    lowercase__      =					{}

             lowercase__      =					spm.SentencePieceProcessor(**self.sp_model_kwargs					)
             self.sp_model.Load(self.vocab_file					)



      def  UpperCamelCase__						(self       :       Optional[int]  ,		UpperCamelCase       :       Dict					):

             '''simple docstring'''







             if self.remove_space:
                    lowercase__      =					''' '''.join(inputs.strip().split()					)
             else:
                    lowercase__      =					inputs
             lowercase__      =					outputs.replace('''``'''  ,		'''"'''					).replace('''\'\''''  ,		'''"'''					)

             if not self.keep_accents:
                    lowercase__      =					unicodedata.normalize('''NFKD'''  ,		UpperCamelCase					)
                    lowercase__      =					''''''.join([c for c in outputs if not unicodedata.combining(UpperCamelCase					)]					)
             if self.do_lower_case:
                    lowercase__      =					outputs.lower()

             return outputs



      def  UpperCamelCase__						(self       :       List[Any]  ,		UpperCamelCase       :       str					):

             '''simple docstring'''







             lowercase__      =					self.preprocess_text(UpperCamelCase					)
             lowercase__      =					self.sp_model.encode(UpperCamelCase  ,		out_type=UpperCamelCase					)
             lowercase__      =					[]
             for piece in pieces:
                    if len(UpperCamelCase					) > 1 and piece[-1] == str(''','''					) and piece[-2].isdigit():
                           lowercase__      =					self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCamelCase  ,		''''''					)					)
                           if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                  if len(cur_pieces[0]					) == 1:
                                         lowercase__      =					cur_pieces[1:]
                                  else:
                                         lowercase__      =					cur_pieces[0][1:]
                           cur_pieces.append(piece[-1]					)
                           new_pieces.extend(UpperCamelCase					)
                    else:
                           new_pieces.append(UpperCamelCase					)

             return new_pieces



      def  UpperCamelCase__						(self       :       int  ,		UpperCamelCase       :       str					):

             '''simple docstring'''







             return self.sp_model.PieceToId(UpperCamelCase					)



      def  UpperCamelCase__						(self       :       str  ,		UpperCamelCase       :       Optional[Any]					):

             '''simple docstring'''







             return self.sp_model.IdToPiece(UpperCamelCase					)



      def  UpperCamelCase__						(self       :       Union[str, Any]  ,		UpperCamelCase       :       str					):

             '''simple docstring'''







             lowercase__      =					''''''.join(UpperCamelCase					).replace(UpperCamelCase  ,		''' '''					).strip()
             return out_string



      def  UpperCamelCase__						(self       :       Any  ,		UpperCamelCase       :       List[int]  ,		UpperCamelCase       :       Optional[List[int]] = None					):

             '''simple docstring'''







             lowercase__      =					[self.sep_token_id]
             lowercase__      =					[self.cls_token_id]
             if token_ids_a is None:
                    return token_ids_a + sep + cls
             return token_ids_a + sep + token_ids_a + sep + cls



      def  UpperCamelCase__						(self       :       Tuple  ,		UpperCamelCase       :       List[int]  ,		UpperCamelCase       :       Optional[List[int]] = None  ,		UpperCamelCase       :       bool = False					):

             '''simple docstring'''







             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=UpperCamelCase  ,		token_ids_a=UpperCamelCase  ,		already_has_special_tokens=UpperCamelCase					)

             if token_ids_a is not None:
                    return ([0] * len(UpperCamelCase					)) + [1] + ([0] * len(UpperCamelCase					)) + [1, 1]
             return ([0] * len(UpperCamelCase					)) + [1, 1]



      def  UpperCamelCase__						(self       :       str  ,		UpperCamelCase       :       List[int]  ,		UpperCamelCase       :       Optional[List[int]] = None					):

             '''simple docstring'''







             lowercase__      =					[self.sep_token_id]
             lowercase__      =					[2]

             if token_ids_a is None:
                    return len(token_ids_a + sep					) * [0] + cls_segment_id
             return len(token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1] + cls_segment_id



      def  UpperCamelCase__						(self       :       Tuple  ,		UpperCamelCase       :       str  ,		UpperCamelCase       :       Optional[str] = None					):

             '''simple docstring'''







             if not os.path.isdir(UpperCamelCase					):
                    logger.error(f"Vocabulary path ({save_directory}) should be a directory"					)
                    return
             lowercase__      =					os.path.join(
                 UpperCamelCase  ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

             if os.path.abspath(self.vocab_file					) != os.path.abspath(UpperCamelCase					) and os.path.isfile(self.vocab_file					):
                    copyfile(self.vocab_file  ,		UpperCamelCase					)
             elif not os.path.isfile(self.vocab_file					):
                    with open(UpperCamelCase  ,		'''wb'''					) as fi:
                           lowercase__      =					self.sp_model.serialized_model_proto()
                           fi.write(UpperCamelCase					)

             return (out_vocab_file,)






      def  UpperCamelCase__						(self       :       List[str]  ,		*UpperCamelCase       :       List[Any]  ,		**UpperCamelCase       :       Union[str, Any]					):

             '''simple docstring'''







             lowercase__      =					super()._decode(*UpperCamelCase  ,		**UpperCamelCase					)
             lowercase__      =					text.replace(''' '''  ,		''''''					).replace('''\u2582'''  ,		''' '''					).replace('''\u2583'''  ,		'''\n'''					)
             return text




