
"""simple docstring"""


import json
import re
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
						if is_torch_available():
												import torch
						if is_tf_available():
												import tensorflow as tf

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_codegen import CodeGenTokenizer


A_    							=					logging.get_logger(__name__)

A_    							=					{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

A_    							=					{
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """Salesforce/codegen-350M-mono""": (
            """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/tokenizer.json"""
        ),
    },
}

A_    							=					{
    """Salesforce/codegen-350M-mono""": 2048,
}




class  __lowerCamelCase  ( lowerCAmelCase			):
				a__:      List[str]          =    VOCAB_FILES_NAMES
				a__:      Any          =    PRETRAINED_VOCAB_FILES_MAP
				a__:      int          =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				a__:      Union[str, Any]          =    ['input_ids', 'attention_mask']
				a__:      List[str]          =    CodeGenTokenizer




				def __init__(				self							,						UpperCAmelCase=None							,						UpperCAmelCase=None							,						UpperCAmelCase=None							,						UpperCAmelCase="<|endoftext|>"							,						UpperCAmelCase="<|endoftext|>"							,						UpperCAmelCase="<|endoftext|>"							,						UpperCAmelCase=False							,						**UpperCAmelCase							,						):
							super().__init__(
							    UpperCAmelCase							,						UpperCAmelCase							,						tokenizer_file=UpperCAmelCase							,						unk_token=UpperCAmelCase							,						bos_token=UpperCAmelCase							,						eos_token=UpperCAmelCase							,						add_prefix_space=UpperCAmelCase							,						**UpperCAmelCase							,						)

							if kwargs.pop('''add_bos_token'''							,						UpperCAmelCase   ):
										lowerCamelCase_					=  kwargs.pop('''name_or_path'''							,						''''''   )
										raise ValueError(
										    '''Currenty GPT2\'s fast tokenizer does NOT support adding a BOS token.'''
										    '''Instead you should use GPT2\'s slow tokenizer class `CodeGenTokenizer` as follows: \n'''
										    f"`CodeGenTokenizer.from_pretrained('{model_id}')`\nor\n"
										    f"`AutoTokenizer.from_pretrained('{model_id}', use_fast=False)`\n"
										    '''This issue will be fixed soon, see: https://github.com/huggingface/tokenizers/pull/1005.'''
										    ''' so that the fast tokenizer works correctly.'''   )

							lowerCamelCase_					=  json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
							if pre_tok_state.get('''add_prefix_space'''							,						UpperCAmelCase   ) != add_prefix_space:
										lowerCamelCase_					=  getattr(UpperCAmelCase							,						pre_tok_state.pop('''type'''   )   )
										lowerCamelCase_					=  add_prefix_space
										lowerCamelCase_					=  pre_tok_class(**UpperCAmelCase   )

							lowerCamelCase_					=  add_prefix_space




				def        UpperCAmelCase__						(				self							,						*UpperCAmelCase							,						**UpperCAmelCase   ):
							lowerCamelCase_					=  kwargs.get('''is_split_into_words'''							,						UpperCAmelCase   )
							assert self.add_prefix_space or not is_split_into_words, (
							    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
							    "to use it with pretokenized inputs."
							)

							return super()._batch_encode_plus(*UpperCAmelCase							,						**UpperCAmelCase   )




				def        UpperCAmelCase__						(				self							,						*UpperCAmelCase							,						**UpperCAmelCase   ):
							lowerCamelCase_					=  kwargs.get('''is_split_into_words'''							,						UpperCAmelCase   )

							assert self.add_prefix_space or not is_split_into_words, (
							    f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
							    "to use it with pretokenized inputs."
							)

							return super()._encode_plus(*UpperCAmelCase							,						**UpperCAmelCase   )




				def        UpperCAmelCase__						(				self							,						UpperCAmelCase							,						UpperCAmelCase = None   ):
							lowerCamelCase_					=  self._tokenizer.model.save(UpperCAmelCase							,						name=UpperCAmelCase   )
							return tuple(UpperCAmelCase   )




				def        UpperCAmelCase__						(				self							,						UpperCAmelCase							,						UpperCAmelCase = False							,						UpperCAmelCase = None							,						UpperCAmelCase = None							,						**UpperCAmelCase							,						):
							lowerCamelCase_					=  super().decode(
							    token_ids=UpperCAmelCase							,						skip_special_tokens=UpperCAmelCase							,						clean_up_tokenization_spaces=UpperCAmelCase							,						**UpperCAmelCase							,						)

							if truncate_before_pattern is not None and len(UpperCAmelCase   ) > 0:
										lowerCamelCase_					=  self.truncate(UpperCAmelCase							,						UpperCAmelCase   )

							return decoded_text





				def        UpperCAmelCase__						(				self							,						UpperCAmelCase							,						UpperCAmelCase   ):
							def find_re(UpperCAmelCase							,						UpperCAmelCase							,						UpperCAmelCase   ):
										lowerCamelCase_					=  pattern.search(UpperCAmelCase							,						UpperCAmelCase   )
										return m.start() if m else -1

							lowerCamelCase_					=  [re.compile(UpperCAmelCase							,						re.MULTILINE   ) for pattern in truncate_before_pattern]

							lowerCamelCase_					=  list(re.finditer('''^print'''							,						UpperCAmelCase							,						re.MULTILINE   )   )

							if len(UpperCAmelCase   ) > 1:
										lowerCamelCase_					=  completion[: prints[1].start()]

							lowerCamelCase_					=  list(re.finditer('''^def'''							,						UpperCAmelCase							,						re.MULTILINE   )   )

							if len(UpperCAmelCase   ) > 1:
										lowerCamelCase_					=  completion[: defs[1].start()]

							lowerCamelCase_					=  0

							lowerCamelCase_					=  [
							    pos for pos in [find_re(UpperCAmelCase							,						UpperCAmelCase							,						UpperCAmelCase   ) for terminal in terminals] if pos != -1
							]

							if len(UpperCAmelCase   ) > 0:
										return completion[: min(UpperCAmelCase   )]
							else:
										return completion





"""simple docstring"""


import copy
import tempfile
import unittest

from huggingface_hub import HfFolder, delete_repo
from parameterized import parameterized
from requests.exceptions import HTTPError

from transformers import AutoConfig, GenerationConfig
from transformers.testing_utils import TOKEN, USER, is_staging_test




class  __lowerCamelCase  ( unittest.TestCase			):




				@parameterized.expand([(None,), ('''foo.json''',)]   )
				def        UpperCAmelCase__						(				self							,						UpperCAmelCase   ):
							lowerCamelCase_					=  GenerationConfig(
							    do_sample=UpperCAmelCase							,						temperature=0.7							,						length_penalty=1.0							,						bad_words_ids=[[1, 2, 3], [4, 5]]							,						)
							with tempfile.TemporaryDirectory() as tmp_dir:
										config.save_pretrained(UpperCAmelCase							,						config_name=UpperCAmelCase   )
										lowerCamelCase_					=  GenerationConfig.from_pretrained(UpperCAmelCase							,						config_name=UpperCAmelCase   )

							# Checks parameters that were specified
							self.assertEqual(loaded_config.do_sample							,						UpperCAmelCase   )
							self.assertEqual(loaded_config.temperature							,						0.7   )
							self.assertEqual(loaded_config.length_penalty							,						1.0   )
							self.assertEqual(loaded_config.bad_words_ids							,						[[1, 2, 3], [4, 5]]   )

							# Checks parameters that were not specified (defaults)
							self.assertEqual(loaded_config.top_k							,						50   )
							self.assertEqual(loaded_config.max_length							,						20   )
							self.assertEqual(loaded_config.max_time							,						UpperCAmelCase   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  AutoConfig.from_pretrained('''gpt2'''   )
							lowerCamelCase_					=  GenerationConfig.from_model_config(UpperCAmelCase   )
							lowerCamelCase_					=  GenerationConfig()

							# The generation config has loaded a few non-default parameters from the model config
							self.assertNotEqual(UpperCAmelCase							,						UpperCAmelCase   )

							# One of those parameters is eos_token_id -- check if it matches
							self.assertNotEqual(generation_config_from_model.eos_token_id							,						default_generation_config.eos_token_id   )
							self.assertEqual(generation_config_from_model.eos_token_id							,						model_config.eos_token_id   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  GenerationConfig()
							lowerCamelCase_					=  {
							    '''max_new_tokens''': 1024,
							    '''foo''': '''bar''',
							}
							lowerCamelCase_					=  copy.deepcopy(UpperCAmelCase   )
							lowerCamelCase_					=  generation_config.update(**UpperCAmelCase   )

							# update_kwargs was not modified (no side effects)
							self.assertEqual(UpperCAmelCase							,						UpperCAmelCase   )

							# update_kwargs was used to update the config on valid attributes
							self.assertEqual(generation_config.max_new_tokens							,						1024   )

							# `.update()` returns a dictionary of unused kwargs
							self.assertEqual(UpperCAmelCase							,						{'''foo''': '''bar'''}   )




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  GenerationConfig()
							lowerCamelCase_					=  '''bar'''

							with tempfile.TemporaryDirectory('''test-generation-config'''   ) as tmp_dir:
										generation_config.save_pretrained(UpperCAmelCase   )

										lowerCamelCase_					=  GenerationConfig.from_pretrained(UpperCAmelCase   )
							# update_kwargs was used to update the config on valid attributes
							self.assertEqual(new_config.foo							,						'''bar'''   )

							lowerCamelCase_					=  GenerationConfig.from_model_config(UpperCAmelCase   )
							assert not hasattr(UpperCAmelCase							,						'''foo'''   )  # no new kwargs should be initialized if from config





				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  GenerationConfig()
							self.assertEqual(default_config.temperature							,						1.0   )
							self.assertEqual(default_config.do_sample							,						UpperCAmelCase   )
							self.assertEqual(default_config.num_beams							,						1   )

							lowerCamelCase_					=  GenerationConfig(
							    do_sample=UpperCAmelCase							,						temperature=0.7							,						length_penalty=1.0							,						bad_words_ids=[[1, 2, 3], [4, 5]]							,						)
							self.assertEqual(config.temperature							,						0.7   )
							self.assertEqual(config.do_sample							,						UpperCAmelCase   )
							self.assertEqual(config.num_beams							,						1   )

							with tempfile.TemporaryDirectory() as tmp_dir:
										config.save_pretrained(UpperCAmelCase   )
										lowerCamelCase_					=  GenerationConfig.from_pretrained(UpperCAmelCase							,						temperature=1.0   )

							self.assertEqual(loaded_config.temperature							,						1.0   )
							self.assertEqual(loaded_config.do_sample							,						UpperCAmelCase   )
							self.assertEqual(loaded_config.num_beams							,						1   )  # default value




@is_staging_test
class  __lowerCamelCase  ( unittest.TestCase			):




				@classmethod
				def        UpperCAmelCase__						(				cls   ):
							lowerCamelCase_					=  TOKEN
							HfFolder.save_token(UpperCAmelCase   )




				@classmethod
				def        UpperCAmelCase__						(				cls   ):
							try:
										delete_repo(token=cls._token							,						repo_id='''test-generation-config'''   )
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token							,						repo_id='''valid_org/test-generation-config-org'''   )
							except HTTPError:
										pass




				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  GenerationConfig(
							    do_sample=UpperCAmelCase							,						temperature=0.7							,						length_penalty=1.0							,						)
							config.push_to_hub('''test-generation-config'''							,						use_auth_token=self._token   )

							lowerCamelCase_					=  GenerationConfig.from_pretrained(f"{USER}/test-generation-config"   )
							for k, v in config.to_dict().items():
										if k != "transformers_version":
													self.assertEqual(UpperCAmelCase							,						getattr(UpperCAmelCase							,						UpperCAmelCase   )   )

        # Reset repo
							delete_repo(token=self._token							,						repo_id='''test-generation-config'''   )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										config.save_pretrained(
										    UpperCAmelCase							,						repo_id='''test-generation-config'''							,						push_to_hub=UpperCAmelCase							,						use_auth_token=self._token   )

							lowerCamelCase_					=  GenerationConfig.from_pretrained(f"{USER}/test-generation-config"   )
							for k, v in config.to_dict().items():
										if k != "transformers_version":
													self.assertEqual(UpperCAmelCase							,						getattr(UpperCAmelCase							,						UpperCAmelCase   )   )





				def        UpperCAmelCase__						(				self   ):
							lowerCamelCase_					=  GenerationConfig(
							    do_sample=UpperCAmelCase							,						temperature=0.7							,						length_penalty=1.0							,						)
							config.push_to_hub('''valid_org/test-generation-config-org'''							,						use_auth_token=self._token   )

							lowerCamelCase_					=  GenerationConfig.from_pretrained('''valid_org/test-generation-config-org'''   )
							for k, v in config.to_dict().items():
										if k != "transformers_version":
													self.assertEqual(UpperCAmelCase							,						getattr(UpperCAmelCase							,						UpperCAmelCase   )   )

        # Reset repo
							delete_repo(token=self._token							,						repo_id='''valid_org/test-generation-config-org'''   )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										config.save_pretrained(
										    UpperCAmelCase							,						repo_id='''valid_org/test-generation-config-org'''							,						push_to_hub=UpperCAmelCase							,						use_auth_token=self._token   )

							lowerCamelCase_					=  GenerationConfig.from_pretrained('''valid_org/test-generation-config-org'''   )
							for k, v in config.to_dict().items():
										if k != "transformers_version":
													self.assertEqual(UpperCAmelCase							,						getattr(UpperCAmelCase							,						UpperCAmelCase   )   )



