




import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


__a							     =					['\nclass', '\ndef', '\n#', '\n@', '\nprint', '\nif']

class     __a(	_a  ):






    """simple docstring"""





    def __init__( self  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE=None  ,_SCREAMING_SNAKE_CASE=1    )					->						Dict:
         UpperCAmelCase_     :				List[Any]  =     tokenizer
         UpperCAmelCase_     :				int  =     dataset
         UpperCAmelCase_     :				Dict  =     len(_SCREAMING_SNAKE_CASE    ) if n_tasks is None else n_tasks
         UpperCAmelCase_     :				Optional[int]  =     n_copies





    def __iter__( self    )					->						Any:
         UpperCAmelCase_     :				List[Any]  =     []
         for task in range(self.n_tasks    ):
              # without strip, the model generate commented codes ...
              prompts.append(self.tokenizer.eos_token + self.dataset[task]['''prompt'''].strip()    )
         UpperCAmelCase_     :				Union[str, Any]  =     self.tokenizer(_SCREAMING_SNAKE_CASE  ,padding=_SCREAMING_SNAKE_CASE  ,return_tensors='''pt'''    )
         for task in range(self.n_tasks    ):
              for _ in range(self.n_copies    ):
                   yield {
                       "ids": outputs.input_ids[task],
                       "task_id": task,
                       "input_len": outputs.attention_mask[task].sum(),
                   }


class     __a(	_a  ):






    """simple docstring"""





    def __init__( self  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )					->						List[Any]:
         UpperCAmelCase_     :				str  =     start_length
         UpperCAmelCase_     :				Optional[int]  =     eof_strings
         UpperCAmelCase_     :				str  =     tokenizer





    def __call__( self  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE  ,**_SCREAMING_SNAKE_CASE    )					->						List[Any]:
         UpperCAmelCase_     :				Optional[Any]  =     self.tokenizer.batch_decode(input_ids[:, self.start_length :]    )
         UpperCAmelCase_     :				Optional[int]  =     []
         for decoded_generation in decoded_generations:
              done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings    )    )
         return all(_SCREAMING_SNAKE_CASE    )







def lowerCamelCase__  (     _lowercase ):

     '''simple docstring'''


     UpperCAmelCase_     :				Tuple  =     re.split('''(%s)''' % '''|'''.join(_lowercase )					,		_lowercase )
     # last string should be ""
     return "".join(string_list[:-2] )







def lowerCamelCase__  (     _lowercase					,		_lowercase					,		_lowercase					,		_lowercase					,		_lowercase					,		_lowercase=20					,		**_lowercase ):

     '''simple docstring'''


     UpperCAmelCase_     :				List[Any]  =     defaultdict(_lowercase )  # dict of list of generated tokens
     for step, batch in tqdm(enumerate(_lowercase ) ):
          with torch.no_grad():
               UpperCAmelCase_     :				Dict  =     batch['''ids'''].shape[-1]
               UpperCAmelCase_     :				Optional[Any]  =     accelerator.unwrap_model(_lowercase ).generate(
                   input_ids=batch['''ids'''][:, : batch['''input_len''']]					,		num_return_sequences=_lowercase					,		**_lowercase )
               # each task is generated batch_size times
               UpperCAmelCase_     :				Union[str, Any]  =     batch['''task_id'''].repeat(_lowercase )
               UpperCAmelCase_     :				Dict  =     accelerator.pad_across_processes(
                   _lowercase					,		dim=1					,		pad_index=tokenizer.pad_token_id )

               UpperCAmelCase_,       UpperCAmelCase_     :				List[str]  =     accelerator.gather((generated_tokens, generated_tasks) )
               UpperCAmelCase_     :				Union[str, Any]  =     generated_tokens.cpu().numpy()
               UpperCAmelCase_     :				Union[str, Any]  =     generated_tasks.cpu().numpy()

               for task, generated_tokens in zip(_lowercase					,		_lowercase ):
                    gen_token_dict[task].append(_lowercase )

     UpperCAmelCase_     :				Union[str, Any]  =     [[] for _ in range(_lowercase )]
     for task, generated_tokens in gen_token_dict.items():
          for s in generated_tokens:
               UpperCAmelCase_     :				int  =     tokenizer.decode(_lowercase					,		skip_special_tokens=_lowercase					,		clean_up_tokenization_spaces=_lowercase )
               code_gens[task].append(remove_last_block(_lowercase ) )
     return code_gens







def lowerCamelCase__  (     ):

     '''simple docstring'''


     UpperCAmelCase_     :				Optional[Any]  =     HfArgumentParser(_lowercase )
     UpperCAmelCase_     :				int  =     parser.parse_args()

     transformers.logging.set_verbosity_error()
     # enables code execution in code_eval metric
     UpperCAmelCase_     :				Optional[Any]  =     args.HF_ALLOW_CODE_EVAL
     # make sure tokenizer plays nice with multiprocessing
     UpperCAmelCase_     :				List[Any]  =     '''false'''

     if args.num_workers is None:
          UpperCAmelCase_     :				Optional[Any]  =     multiprocessing.cpu_count()

     # Use dataset load to feed to accelerate
     UpperCAmelCase_     :				int  =     Accelerator()
     set_seed(args.seed					,		device_specific=_lowercase )

     # Load model and tokenizer
     UpperCAmelCase_     :				Tuple  =     AutoTokenizer.from_pretrained(args.model_ckpt )
     UpperCAmelCase_     :				Any  =     tokenizer.eos_token
     UpperCAmelCase_     :				List[str]  =     AutoModelForCausalLM.from_pretrained(args.model_ckpt )

     # Generation settings
     UpperCAmelCase_     :				str  =     {
         '''do_sample''': args.do_sample,
         '''temperature''': args.temperature,
         '''max_new_tokens''': args.max_new_tokens,
         '''top_p''': args.top_p,
         '''top_k''': args.top_k,
         '''stopping_criteria''': StoppingCriteriaList([EndOfFunctionCriteria(0					,		_lowercase					,		_lowercase )] ),
     }

     # Load evaluation dataset and metric
     UpperCAmelCase_     :				Tuple  =     load_dataset('''openai_humaneval''' )
     UpperCAmelCase_     :				Dict  =     load_metric('''code_eval''' )

     UpperCAmelCase_     :				Optional[int]  =     args.num_tasks if args.num_tasks is not None else len(human_eval['''test'''] )
     UpperCAmelCase_     :				str  =     args.n_samples // args.batch_size

     UpperCAmelCase_     :				str  =     TokenizedDataset(_lowercase					,		human_eval['''test''']					,		n_copies=_lowercase					,		n_tasks=_lowercase )
     # do not confuse args.batch_size, which is actually the num_return_sequences
     UpperCAmelCase_     :				Optional[Any]  =     DataLoader(_lowercase					,		batch_size=1 )

     # Run a quick test to see if code evaluation is enabled
     try:
          UpperCAmelCase_     :				Any  =     code_eval_metric.compute(references=['''''']					,		predictions=[['''''']] )
     except ValueError as exception:
          print(
              '''Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'''
              ''' flag to enable code evaluation.''' )
          raise exception

     UpperCAmelCase_,       UpperCAmelCase_     :				int  =     accelerator.prepare(_lowercase					,		_lowercase )

     UpperCAmelCase_     :				int  =     complete_code(
         _lowercase					,		_lowercase					,		_lowercase					,		_lowercase					,		n_tasks=_lowercase					,		batch_size=args.batch_size					,		**_lowercase					,		)

     if accelerator.is_main_process:
          UpperCAmelCase_     :				Any  =     []

          for task in tqdm(range(_lowercase ) ):
               UpperCAmelCase_     :				int  =     human_eval['''test'''][task]['''test''']
               UpperCAmelCase_     :				str  =     f'''check({human_eval["test"][task]["entry_point"]})'''
               references.append('''\n''' + test_func + '''\n''' + entry_point )

          # Evaluate completions with "code_eval" metric
          UpperCAmelCase_,       UpperCAmelCase_     :				Any  =     code_eval_metric.compute(
              references=_lowercase					,		predictions=_lowercase					,		num_workers=args.num_workers )
          print(f'''Results: {pass_at_k}''' )

          # Save results to json file
          with open(args.output_file					,		'''w''' ) as fp:
               json.dump(_lowercase					,		_lowercase )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
     main()




'''simple docstring'''





import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english



@require_tokenizers
class  snake_case__						(    SCREAMING_SNAKE_CASE_  ,     unittest.TestCase				):
 A__				= BertTokenizer
 A__				= BertTokenizerFast
 A__				= True
 A__				= True
 A__				= filter_non_english
 def A_   (   self						:							Optional[Any]  )      ->   Dict:







       '''simple docstring'''
       super().setUp()

       __snake_case :   List[Any]													=    [
           '[UNK]',
           '[CLS]',
           '[SEP]',
           '[PAD]',
           '[MASK]',
           'want',
           '##want',
           '##ed',
           'wa',
           'un',
           'runn',
           '##ing',
           ',',
           'low',
           'lowest',
       ]
       __snake_case :   Optional[int]													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['vocab_file']  )
       with open(self.vocab_file      ,     'w'      ,     encoding='utf-8'  ) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]  )  )
 def A_   (   self						:							Dict      ,     __a						:							Optional[int]  )      ->   int:







       '''simple docstring'''
       __snake_case :   List[Any]													=    'UNwant\u00E9d,running'
       __snake_case :   Tuple													=    'unwanted, running'
       return input_text, output_text
 def A_   (   self						:							Tuple  )      ->   Dict:







       '''simple docstring'''
       __snake_case :   Any													=    self.tokenizer_class(self.vocab_file  )

       __snake_case :   Tuple													=    tokenizer.tokenize('UNwant\u00E9d,running'  )
       self.assertListEqual(__a      ,     ['un', '##want', '##ed', ',', 'runn', '##ing']  )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__a  )      ,     [9, 6, 7, 12, 10, 11]  )
 def A_   (   self						:							Optional[Any]  )      ->   Tuple:







       '''simple docstring'''
       if not self.test_rust_tokenizer:
             return

       __snake_case :   int													=    self.get_tokenizer()
       __snake_case :   Optional[Any]													=    self.get_rust_tokenizer()

       __snake_case :   int													=    'UNwant\u00E9d,running'

       __snake_case :   Any													=    tokenizer.tokenize(__a  )
       __snake_case :   Dict													=    rust_tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   Optional[int]													=    tokenizer.encode(__a      ,     add_special_tokens=__a  )
       __snake_case :   Optional[int]													=    rust_tokenizer.encode(__a      ,     add_special_tokens=__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   str													=    self.get_rust_tokenizer()
       __snake_case :   Tuple													=    tokenizer.encode(__a  )
       __snake_case :   Tuple													=    rust_tokenizer.encode(__a  )
       self.assertListEqual(__a      ,     __a  )

       # With lower casing
       __snake_case :   Union[str, Any]													=    self.get_tokenizer(do_lower_case=__a  )
       __snake_case :   Any													=    self.get_rust_tokenizer(do_lower_case=__a  )

       __snake_case :   Optional[int]													=    'UNwant\u00E9d,running'

       __snake_case :   List[Any]													=    tokenizer.tokenize(__a  )
       __snake_case :   List[str]													=    rust_tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   List[str]													=    tokenizer.encode(__a      ,     add_special_tokens=__a  )
       __snake_case :   Union[str, Any]													=    rust_tokenizer.encode(__a      ,     add_special_tokens=__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   Union[str, Any]													=    self.get_rust_tokenizer()
       __snake_case :   Dict													=    tokenizer.encode(__a  )
       __snake_case :   str													=    rust_tokenizer.encode(__a  )
       self.assertListEqual(__a      ,     __a  )
 def A_   (   self						:							Tuple  )      ->   Optional[int]:







       '''simple docstring'''
       __snake_case :   Optional[Any]													=    BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'  )      ,     ['ah', '\u535A', '\u63A8', 'zz']  )
 def A_   (   self						:							List[str]  )      ->   Any:







       '''simple docstring'''
       __snake_case :   Optional[Any]													=    BasicTokenizer(do_lower_case=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )      ,     ['hello', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['hello']  )
 def A_   (   self						:							str  )      ->   int:







       '''simple docstring'''
       __snake_case :   Dict													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['hällo', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['h\u00E9llo']  )
 def A_   (   self						:							List[str]  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   str													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['hallo', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['hello']  )
 def A_   (   self						:							str  )      ->   str:







       '''simple docstring'''
       __snake_case :   Tuple													=    BasicTokenizer(do_lower_case=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['hallo', '!', 'how', 'are', 'you', '?']  )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'  )      ,     ['hello']  )
 def A_   (   self						:							str  )      ->   Optional[Any]:







       '''simple docstring'''
       __snake_case :   int													=    BasicTokenizer(do_lower_case=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '  )      ,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?']  )
 def A_   (   self						:							Dict  )      ->   List[str]:







       '''simple docstring'''
       __snake_case :   Any													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['HäLLo', '!', 'how', 'Are', 'yoU', '?']  )
 def A_   (   self						:							Dict  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   List[Any]													=    BasicTokenizer(do_lower_case=__a      ,     strip_accents=__a  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '  )      ,     ['HaLLo', '!', 'how', 'Are', 'yoU', '?']  )
 def A_   (   self						:							Optional[Any]  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   List[str]													=    BasicTokenizer(do_lower_case=__a      ,     never_split=['[UNK]']  )

       self.assertListEqual(
           tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'  )      ,     ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']  )
 def A_   (   self						:							Optional[Any]  )      ->   Optional[int]:







       '''simple docstring'''
       __snake_case :   int													=    BasicTokenizer()
       __snake_case :   List[str]													=    'a\n\'ll !!to?\'d of, can\'t.'
       __snake_case :   str													=    ['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
       self.assertListEqual(tokenizer.tokenize(__a  )      ,     __a  )
 def A_   (   self						:							List[Any]  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   str													=    ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

       __snake_case :   Dict													=    {}
       for i, token in enumerate(__a  ):
             __snake_case :   str													=    i
       __snake_case :   List[str]													=    WordpieceTokenizer(vocab=__a      ,     unk_token='[UNK]'  )

       self.assertListEqual(tokenizer.tokenize(''  )      ,     []  )

       self.assertListEqual(tokenizer.tokenize('unwanted running'  )      ,     ['un', '##want', '##ed', 'runn', '##ing']  )

       self.assertListEqual(tokenizer.tokenize('unwantedX running'  )      ,     ['[UNK]', 'runn', '##ing']  )
 def A_   (   self						:							Dict  )      ->   List[str]:







       '''simple docstring'''
       self.assertTrue(_is_whitespace(' '  )  )
       self.assertTrue(_is_whitespace('\t'  )  )
       self.assertTrue(_is_whitespace('\r'  )  )
       self.assertTrue(_is_whitespace('\n'  )  )
       self.assertTrue(_is_whitespace('\u00A0'  )  )

       self.assertFalse(_is_whitespace('A'  )  )
       self.assertFalse(_is_whitespace('-'  )  )
 def A_   (   self						:							List[str]  )      ->   List[Any]:







       '''simple docstring'''
       self.assertTrue(_is_control('\u0005'  )  )

       self.assertFalse(_is_control('A'  )  )
       self.assertFalse(_is_control(' '  )  )
       self.assertFalse(_is_control('\t'  )  )
       self.assertFalse(_is_control('\r'  )  )
 def A_   (   self						:							Any  )      ->   Union[str, Any]:







       '''simple docstring'''
       self.assertTrue(_is_punctuation('-'  )  )
       self.assertTrue(_is_punctuation('$'  )  )
       self.assertTrue(_is_punctuation('`'  )  )
       self.assertTrue(_is_punctuation('.'  )  )

       self.assertFalse(_is_punctuation('A'  )  )
       self.assertFalse(_is_punctuation(' '  )  )
 def A_   (   self						:							List[Any]  )      ->   Tuple:







       '''simple docstring'''
       __snake_case :   str													=    self.get_tokenizer()
       __snake_case :   int													=    self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(__a  ) for t in ['Test', '\xad', 'test']]      ,     [['[UNK]'], [], ['[UNK]']]  )

       self.assertListEqual(
           [rust_tokenizer.tokenize(__a  ) for t in ['Test', '\xad', 'test']]      ,     [['[UNK]'], [], ['[UNK]']]  )
 @slow
 def A_   (   self						:							List[str]  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   Dict													=    self.tokenizer_class.from_pretrained('bert-base-uncased'  )

       __snake_case :   Dict													=    tokenizer.encode('sequence builders'      ,     add_special_tokens=__a  )
       __snake_case :   int													=    tokenizer.encode('multi-sequence build'      ,     add_special_tokens=__a  )

       __snake_case :   int													=    tokenizer.build_inputs_with_special_tokens(__a  )
       __snake_case :   Union[str, Any]													=    tokenizer.build_inputs_with_special_tokens(__a      ,     __a  )

       assert encoded_sentence == [101] + text + [102]
       assert encoded_pair == [101] + text + [102] + text_a + [102]
 def A_   (   self						:							Optional[int]  )      ->   str:







       '''simple docstring'''
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                   __snake_case :   Optional[int]													=    self.rust_tokenizer_class.from_pretrained(__a      ,     **__a  )

                   __snake_case :   Optional[Any]													=    f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                   __snake_case :   Tuple													=    tokenizer_r.encode_plus(
                       __a      ,     return_attention_mask=__a      ,     return_token_type_ids=__a      ,     return_offsets_mapping=__a      ,     add_special_tokens=__a      ,     )

                   __snake_case :   Optional[Any]													=    tokenizer_r.do_lower_case if hasattr(__a      ,     'do_lower_case'  ) else False
                   __snake_case :   Any													=    (
                       [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'A'),
                           ((1, 2), ','),
                           ((3, 5), 'na'),
                           ((5, 6), '##ï'),
                           ((6, 8), '##ve'),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), 'Allen'),
                           ((21, 23), '##NL'),
                           ((23, 24), '##P'),
                           ((25, 33), 'sentence'),
                           ((33, 34), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                       if not do_lower_case
                       else [
                           ((0, 0), tokenizer_r.cls_token),
                           ((0, 1), 'a'),
                           ((1, 2), ','),
                           ((3, 8), 'naive'),
                           ((9, 15), tokenizer_r.mask_token),
                           ((16, 21), 'allen'),
                           ((21, 23), '##nl'),
                           ((23, 24), '##p'),
                           ((25, 33), 'sentence'),
                           ((33, 34), '.'),
                           ((0, 0), tokenizer_r.sep_token),
                       ]
                   )

                   self.assertEqual(
                       [e[1] for e in expected_results]      ,     tokenizer_r.convert_ids_to_tokens(tokens['input_ids']  )  )
                   self.assertEqual([e[0] for e in expected_results]      ,     tokens['offset_mapping']  )





 def A_   (   self						:							Any  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   Any													=    ['的', '人', '有']
       __snake_case :   List[Any]													=    ''.join(__a  )
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                   __snake_case :   List[Any]													=    True
                   __snake_case :   int													=    self.tokenizer_class.from_pretrained(__a      ,     **__a  )
                   __snake_case :   Tuple													=    self.rust_tokenizer_class.from_pretrained(__a      ,     **__a  )

                   __snake_case :   Union[str, Any]													=    tokenizer_p.encode(__a      ,     add_special_tokens=__a  )
                   __snake_case :   Optional[Any]													=    tokenizer_r.encode(__a      ,     add_special_tokens=__a  )

                   __snake_case :   Dict													=    tokenizer_r.convert_ids_to_tokens(__a  )
                   __snake_case :   str													=    tokenizer_p.convert_ids_to_tokens(__a  )

                   # it is expected that each Chinese character is not preceded by "##"
                   self.assertListEqual(__a      ,     __a  )
                   self.assertListEqual(__a      ,     __a  )

                   __snake_case :   Any													=    False
                   __snake_case :   Tuple													=    self.rust_tokenizer_class.from_pretrained(__a      ,     **__a  )
                   __snake_case :   int													=    self.tokenizer_class.from_pretrained(__a      ,     **__a  )

                   __snake_case :   int													=    tokenizer_r.encode(__a      ,     add_special_tokens=__a  )
                   __snake_case :   str													=    tokenizer_p.encode(__a      ,     add_special_tokens=__a  )

                   __snake_case :   Optional[Any]													=    tokenizer_r.convert_ids_to_tokens(__a  )
                   __snake_case :   Optional[Any]													=    tokenizer_p.convert_ids_to_tokens(__a  )

                   # it is expected that only the first Chinese character is not preceded by "##".
                   __snake_case :   Dict													=    [
                       f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__a  )
                   ]
                   self.assertListEqual(__a      ,     __a  )
                   self.assertListEqual(__a      ,     __a  )



