




import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam





class 			lowerCamelCase_				( datasets.BeamBasedBuilder   ):


 '''simple docstring'''







 def  lowerCAmelCase_    (				self    :						Any		):
       return datasets.DatasetInfo(
           features=datasets.Features({'content': datasets.Value('string'		)}		)							,							supervised_keys=_lowerCAmelCase							,							)



 def  lowerCAmelCase_    (				self    :						Any							,							_lowerCAmelCase    :						List[Any]							,							_lowerCAmelCase    :						Optional[Any]		):
       return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,							gen_kwargs={'examples': get_test_dummy_examples()}		)]




 def  lowerCAmelCase_    (				self    :						Any							,							_lowerCAmelCase    :						List[str]							,							_lowerCAmelCase    :						List[Any]		):
       import apache_beam as beam

       return pipeline | "Load Examples" >> beam.Create(_lowerCAmelCase		)





class 			lowerCamelCase_				( datasets.BeamBasedBuilder   ):


 '''simple docstring'''







 def  lowerCAmelCase_    (				self    :						str		):
       return datasets.DatasetInfo(
           features=datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'		)}		)}		)							,							supervised_keys=_lowerCAmelCase							,							)



 def  lowerCAmelCase_    (				self    :						Tuple							,							_lowerCAmelCase    :						Union[str, Any]							,							_lowerCAmelCase    :						Tuple		):
       return [
           datasets.SplitGenerator(name=datasets.Split.TRAIN							,							gen_kwargs={'examples': get_test_nested_examples()}		)
       ]




 def  lowerCAmelCase_    (				self    :						List[str]							,							_lowerCAmelCase    :						Any							,							_lowerCAmelCase    :						Optional[Any]		):
       import apache_beam as beam

       return pipeline | "Load Examples" >> beam.Create(_lowerCAmelCase		)


def    UpperCAmelCase_				( )   ->					Optional[Any]:
      return [(i, {"content": content}) for i, content in enumerate(['foo', 'bar', 'foobar']				)]

def    UpperCAmelCase_				( )   ->					Union[str, Any]:
      return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['foo', 'bar', 'foobar']				)]





class 			lowerCamelCase_				( _SCREAMING_SNAKE_CASE   ):


 '''simple docstring'''







 @require_beam
 def  lowerCAmelCase_    (				self    :						List[Any]		):
       SCREAMING_SNAKE_CASE_      		=     len(get_test_dummy_examples()		)
       with tempfile.TemporaryDirectory() as tmp_cache_dir:
             SCREAMING_SNAKE_CASE_      		=     DummyBeamDataset(cache_dir=_lowerCAmelCase							,							beam_runner='DirectRunner'		)
             builder.download_and_prepare()
             self.assertTrue(
                 os.path.exists(
                     os.path.join(_lowerCAmelCase							,							builder.name							,							'default'							,							'0.0.0'							,							F"{builder.name}-train.arrow"		)		)		)
             self.assertDictEqual(builder.info.features							,							datasets.Features({'content': datasets.Value('string'		)}		)		)
             SCREAMING_SNAKE_CASE_      		=     builder.as_dataset()
             self.assertEqual(dset['train'].num_rows							,							_lowerCAmelCase		)
             self.assertEqual(dset['train'].info.splits['train'].num_examples							,							_lowerCAmelCase		)
             self.assertDictEqual(dset['train'][0]							,							get_test_dummy_examples()[0][1]		)
             self.assertDictEqual(
                 dset['train'][expected_num_examples - 1]							,							get_test_dummy_examples()[expected_num_examples - 1][1]		)
             self.assertTrue(
                 os.path.exists(os.path.join(_lowerCAmelCase							,							builder.name							,							'default'							,							'0.0.0'							,							'dataset_info.json'		)		)		)
             del dset



 @require_beam
 def  lowerCAmelCase_    (				self    :						List[str]		):
       import apache_beam as beam

       SCREAMING_SNAKE_CASE_      		=     beam.io.parquetio.WriteToParquet

       SCREAMING_SNAKE_CASE_      		=     len(get_test_dummy_examples()		)
       with tempfile.TemporaryDirectory() as tmp_cache_dir:
             SCREAMING_SNAKE_CASE_      		=     DummyBeamDataset(cache_dir=_lowerCAmelCase							,							beam_runner='DirectRunner'		)
             with patch('apache_beam.io.parquetio.WriteToParquet'		) as write_parquet_mock:
                   SCREAMING_SNAKE_CASE_      		=     partial(_lowerCAmelCase							,							num_shards=2		)
                   builder.download_and_prepare()
             self.assertTrue(
                 os.path.exists(
                     os.path.join(
                         _lowerCAmelCase							,							builder.name							,							'default'							,							'0.0.0'							,							F"{builder.name}-train-00000-of-00002.arrow"		)		)		)
             self.assertTrue(
                 os.path.exists(
                     os.path.join(
                         _lowerCAmelCase							,							builder.name							,							'default'							,							'0.0.0'							,							F"{builder.name}-train-00000-of-00002.arrow"		)		)		)
             self.assertDictEqual(builder.info.features							,							datasets.Features({'content': datasets.Value('string'		)}		)		)
             SCREAMING_SNAKE_CASE_      		=     builder.as_dataset()
             self.assertEqual(dset['train'].num_rows							,							_lowerCAmelCase		)
             self.assertEqual(dset['train'].info.splits['train'].num_examples							,							_lowerCAmelCase		)
             # Order is not preserved when sharding, so we just check that all the elements are there
             self.assertListEqual(sorted(dset['train']['content']		)							,							sorted(['foo', 'bar', 'foobar']		)		)
             self.assertTrue(
                 os.path.exists(os.path.join(_lowerCAmelCase							,							builder.name							,							'default'							,							'0.0.0'							,							'dataset_info.json'		)		)		)
             del dset



 @require_beam
 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       with tempfile.TemporaryDirectory() as tmp_cache_dir:
             SCREAMING_SNAKE_CASE_      		=     DummyBeamDataset(cache_dir=_lowerCAmelCase		)
             self.assertRaises(datasets.builder.MissingBeamOptions							,							builder.download_and_prepare		)




 @require_beam
 def  lowerCAmelCase_    (				self    :						str		):
       SCREAMING_SNAKE_CASE_      		=     len(get_test_nested_examples()		)
       with tempfile.TemporaryDirectory() as tmp_cache_dir:
             SCREAMING_SNAKE_CASE_      		=     NestedBeamDataset(cache_dir=_lowerCAmelCase							,							beam_runner='DirectRunner'		)
             builder.download_and_prepare()
             self.assertTrue(
                 os.path.exists(
                     os.path.join(_lowerCAmelCase							,							builder.name							,							'default'							,							'0.0.0'							,							F"{builder.name}-train.arrow"		)		)		)
             self.assertDictEqual(
                 builder.info.features							,							datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'		)}		)}		)		)
             SCREAMING_SNAKE_CASE_      		=     builder.as_dataset()
             self.assertEqual(dset['train'].num_rows							,							_lowerCAmelCase		)
             self.assertEqual(dset['train'].info.splits['train'].num_examples							,							_lowerCAmelCase		)
             self.assertDictEqual(dset['train'][0]							,							get_test_nested_examples()[0][1]		)
             self.assertDictEqual(
                 dset['train'][expected_num_examples - 1]							,							get_test_nested_examples()[expected_num_examples - 1][1]		)
             self.assertTrue(
                 os.path.exists(os.path.join(_lowerCAmelCase							,							builder.name							,							'default'							,							'0.0.0'							,							'dataset_info.json'		)		)		)
             del dset



'''simple docstring'''






import json
import os
import re
import unittest

from transformers import CodeGenTokenizer, CodeGenTokenizerFast
from transformers.models.codegen.tokenization_codegen import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class    __magic_name__				(						lowerCAmelCase   ,unittest.TestCase							):
       UpperCAmelCase		 =CodeGenTokenizer
       UpperCAmelCase		 =CodeGenTokenizerFast
       UpperCAmelCase		 =True
       UpperCAmelCase		 ={"add_prefix_space": True}
       UpperCAmelCase		 =False






       def      lowerCAmelCase						(     self)		->    Optional[int]:




          '''simple docstring'''
          super().setUp()

          # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
          _UpperCAmelCase  :  Optional[Any]				       =[
              'l',
              'o',
              'w',
              'e',
              'r',
              's',
              't',
              'i',
              'd',
              'n',
              '\u0120',
              '\u0120l',
              '\u0120n',
              '\u0120lo',
              '\u0120low',
              'er',
              '\u0120lowest',
              '\u0120newer',
              '\u0120wider',
              '<unk>',
              '<|endoftext|>',
          ]
          _UpperCAmelCase  :  Dict				       =dict(zip(snake_case    ,				range(len(snake_case))))
          _UpperCAmelCase  :  List[Any]				       =['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
          _UpperCAmelCase  :  List[Any]				       ={'unk_token': '<unk>'}

          _UpperCAmelCase  :  List[Any]				       =os.path.join(self.tmpdirname    ,				VOCAB_FILES_NAMES['vocab_file'])
          _UpperCAmelCase  :  List[str]				       =os.path.join(self.tmpdirname    ,				VOCAB_FILES_NAMES['merges_file'])
          with open(self.vocab_file    ,				'w'    ,				encoding='utf-8') as fp:
             fp.write(json.dumps(snake_case) + '\n')
          with open(self.merges_file    ,				'w'    ,				encoding='utf-8') as fp:
             fp.write('\n'.join(snake_case))






       def      lowerCAmelCase						(     self    ,				**snake_case)		->    str:




          '''simple docstring'''
          kwargs.update(self.special_tokens_map)
          return CodeGenTokenizer.from_pretrained(self.tmpdirname    ,				**snake_case)






       def      lowerCAmelCase						(     self    ,				**snake_case)		->    Optional[Any]:




          '''simple docstring'''
          kwargs.update(self.special_tokens_map)
          return CodeGenTokenizerFast.from_pretrained(self.tmpdirname    ,				**snake_case)






       def      lowerCAmelCase						(     self    ,				snake_case)		->    Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Union[str, Any]				       ='lower newer'
          _UpperCAmelCase  :  int				       ='lower newer'
          return input_text, output_text






       def      lowerCAmelCase						(     self)		->    int:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[int]				       =CodeGenTokenizer(self.vocab_file    ,				self.merges_file    ,				**self.special_tokens_map)
          _UpperCAmelCase  :  str				       ='lower newer'
          _UpperCAmelCase  :  Union[str, Any]				       =['\u0120low', 'er', '\u0120', 'n', 'e', 'w', 'er']
          _UpperCAmelCase  :  Any				       =tokenizer.tokenize(snake_case    ,				add_prefix_space=snake_case)
          self.assertListEqual(snake_case    ,				snake_case)

          _UpperCAmelCase  :  Any				       =tokens + [tokenizer.unk_token]
          _UpperCAmelCase  :  List[str]				       =[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case)    ,				snake_case)






       def      lowerCAmelCase						(     self)		->    Union[str, Any]:




          '''simple docstring'''
          if not self.test_rust_tokenizer:
             return

          _UpperCAmelCase  :  Tuple				       =self.get_tokenizer()
          _UpperCAmelCase  :  Dict				       =self.get_rust_tokenizer(add_prefix_space=snake_case)

          _UpperCAmelCase  :  Union[str, Any]				       ='lower newer'

          # Testing tokenization
          _UpperCAmelCase  :  List[str]				       =tokenizer.tokenize(snake_case    ,				add_prefix_space=snake_case)
          _UpperCAmelCase  :  Optional[int]				       =rust_tokenizer.tokenize(snake_case)
          self.assertListEqual(snake_case    ,				snake_case)

          # Testing conversion to ids without special tokens
          _UpperCAmelCase  :  str				       =tokenizer.encode(snake_case    ,				add_special_tokens=snake_case    ,				add_prefix_space=snake_case)
          _UpperCAmelCase  :  Optional[int]				       =rust_tokenizer.encode(snake_case    ,				add_special_tokens=snake_case)
          self.assertListEqual(snake_case    ,				snake_case)

          # Testing conversion to ids with special tokens
          _UpperCAmelCase  :  Dict				       =self.get_rust_tokenizer(add_prefix_space=snake_case)
          _UpperCAmelCase  :  Tuple				       =tokenizer.encode(snake_case    ,				add_prefix_space=snake_case)
          _UpperCAmelCase  :  int				       =rust_tokenizer.encode(snake_case)
          self.assertListEqual(snake_case    ,				snake_case)

          # Testing the unknown token
          _UpperCAmelCase  :  List[str]				       =tokens + [rust_tokenizer.unk_token]
          _UpperCAmelCase  :  Union[str, Any]				       =[1_4, 1_5, 1_0, 9, 3, 2, 1_5, 1_9]
          self.assertListEqual(rust_tokenizer.convert_tokens_to_ids(snake_case)    ,				snake_case)






       def      lowerCAmelCase						(     self    ,				*snake_case    ,				**snake_case)		->    Any:




          '''simple docstring'''
          # It's very difficult to mix/test pretokenization with byte-level
          # And get both CodeGen and Roberta to work at the same time (mostly an issue of adding a space before the string)
          pass






       def      lowerCAmelCase						(     self    ,				snake_case=1_5)		->    Optional[int]:




          '''simple docstring'''
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"):
                _UpperCAmelCase  :  Optional[Any]				       =self.rust_tokenizer_class.from_pretrained(snake_case    ,				**snake_case)

                # Simple input
                _UpperCAmelCase  :  List[str]				       ='This is a simple input'
                _UpperCAmelCase  :  Optional[Any]				       =['This is a simple input 1', 'This is a simple input 2']
                _UpperCAmelCase  :  int				       =('This is a simple input', 'This is a pair')
                _UpperCAmelCase  :  Union[str, Any]				       =[
                    ('This is a simple input 1', 'This is a simple input 2'),
                    ('This is a simple pair 1', 'This is a simple pair 2'),
                ]

                # Simple input tests
                self.assertRaises(snake_case    ,				tokenizer_r.encode    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length')

                # Simple input
                self.assertRaises(snake_case    ,				tokenizer_r.encode_plus    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length')

                # Simple input
                self.assertRaises(
                    snake_case    ,				tokenizer_r.batch_encode_plus    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length'    ,				)

                # Pair input
                self.assertRaises(snake_case    ,				tokenizer_r.encode    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length')

                # Pair input
                self.assertRaises(snake_case    ,				tokenizer_r.encode_plus    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length')

                # Pair input
                self.assertRaises(
                    snake_case    ,				tokenizer_r.batch_encode_plus    ,				snake_case    ,				max_length=snake_case    ,				padding='max_length'    ,				)






       def      lowerCAmelCase						(     self)		->    Optional[Any]:




          '''simple docstring'''
          _UpperCAmelCase  :  Any				       =CodeGenTokenizer.from_pretrained(self.tmpdirname    ,				pad_token='<pad>')

          # Simple input
          _UpperCAmelCase  :  List[Any]				       ='This is a simple input'
          _UpperCAmelCase  :  Any				       =['This is a simple input looooooooong', 'This is a simple input']
          _UpperCAmelCase  :  List[str]				       =('This is a simple input', 'This is a pair')
          _UpperCAmelCase  :  Optional[int]				       =[
              ('This is a simple input loooooong', 'This is a simple input'),
              ('This is a simple pair loooooong', 'This is a simple pair'),
          ]

          _UpperCAmelCase  :  List[Any]				       =tokenizer.pad_token_id

          _UpperCAmelCase  :  Union[str, Any]				       =tokenizer(snake_case    ,				padding='max_length'    ,				max_length=3_0    ,				return_tensors='np')
          _UpperCAmelCase  :  List[Any]				       =tokenizer(snake_case    ,				padding=snake_case    ,				truncate=snake_case    ,				return_tensors='np')
          _UpperCAmelCase  :  Optional[Any]				       =tokenizer(*snake_case    ,				padding='max_length'    ,				max_length=6_0    ,				return_tensors='np')
          _UpperCAmelCase  :  List[Any]				       =tokenizer(snake_case    ,				padding=snake_case    ,				truncate=snake_case    ,				return_tensors='np')

          # s
          # test single string max_length padding
          self.assertEqual(out_s['input_ids'].shape[-1]    ,				3_0)
          self.assertTrue(pad_token_id in out_s['input_ids'])
          self.assertTrue(0 in out_s['attention_mask'])

          # s2
          # test automatic padding
          self.assertEqual(out_sa['input_ids'].shape[-1]    ,				3_3)
          # long slice doesn't have padding
          self.assertFalse(pad_token_id in out_sa['input_ids'][0])
          self.assertFalse(0 in out_sa['attention_mask'][0])
          # short slice does have padding
          self.assertTrue(pad_token_id in out_sa['input_ids'][1])
          self.assertTrue(0 in out_sa['attention_mask'][1])

          # p
          # test single pair max_length padding
          self.assertEqual(out_p['input_ids'].shape[-1]    ,				6_0)
          self.assertTrue(pad_token_id in out_p['input_ids'])
          self.assertTrue(0 in out_p['attention_mask'])

          # p2
          # test automatic padding pair
          self.assertEqual(out_pa['input_ids'].shape[-1]    ,				5_2)
          # long slice pair doesn't have padding
          self.assertFalse(pad_token_id in out_pa['input_ids'][0])
          self.assertFalse(0 in out_pa['attention_mask'][0])
          # short slice pair does have padding
          self.assertTrue(pad_token_id in out_pa['input_ids'][1])
          self.assertTrue(0 in out_pa['attention_mask'][1])






       def      lowerCAmelCase						(     self)		->    Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase  :  Any				       ='$$$'
          _UpperCAmelCase  :  Any				       =CodeGenTokenizer.from_pretrained(self.tmpdirname    ,				bos_token=snake_case    ,				add_bos_token=snake_case)

          _UpperCAmelCase  :  Optional[Any]				       ='This is a simple input'
          _UpperCAmelCase  :  Any				       =['This is a simple input 1', 'This is a simple input 2']

          _UpperCAmelCase  :  int				       =tokenizer.bos_token_id

          _UpperCAmelCase  :  Optional[Any]				       =tokenizer(snake_case)
          _UpperCAmelCase  :  Tuple				       =tokenizer(snake_case)

          self.assertEqual(out_s.input_ids[0]    ,				snake_case)
          self.assertTrue(all(o[0] == bos_token_id for o in out_sa.input_ids))

          _UpperCAmelCase  :  Optional[int]				       =tokenizer.decode(out_s.input_ids)
          _UpperCAmelCase  :  Any				       =tokenizer.batch_decode(out_sa.input_ids)

          self.assertEqual(decode_s.split()[0]    ,				snake_case)
          self.assertTrue(all(d.split()[0] == bos_token for d in decode_sa))






       @slow
       def      lowerCAmelCase						(     self)		->    Optional[int]:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[Any]				       =CodeGenTokenizer.from_pretrained('Salesforce/codegen-350M-mono')

          _UpperCAmelCase  :  Optional[Any]				       ='\nif len_a > len_b:\n    result = a\nelse:\n    result = b\n\n\n\n#'
          _UpperCAmelCase  :  Tuple				       ='\nif len_a > len_b:      result = a\nelse:      result = b'

          _UpperCAmelCase  :  Optional[int]				       =tokenizer.encode(snake_case)
          _UpperCAmelCase  :  List[str]				       =['^#', re.escape('<|endoftext|>'), '^\'\'\'', '^"""', '\n\n\n']
          _UpperCAmelCase  :  Dict				       =tokenizer.decode(snake_case    ,				truncate_before_pattern=snake_case)
          self.assertEqual(snake_case    ,				snake_case)






       def      lowerCAmelCase						(     self)		->    Optional[Any]:




          '''simple docstring'''
          pass

