'''simple docstring'''





from dataclasses import dataclass, field
from typing import Optional






@dataclass
class 				UpperCAmelCase :




    '''simple docstring'''







    snake_case__							:   Optional[str]							=						field(
        default="codeparrot/codeparrot"   ,    metadata={"help": "Model name or path of model to be trained."} )
    snake_case__							:   Optional[str]							=						field(
        default="./"   ,    metadata={"help": "Save dir where model repo is cloned and models updates are saved to."} )
    snake_case__							:   Optional[str]							=						field(
        default="codeparrot/codeparrot-clean-train"   ,    metadata={"help": "Name or path of training dataset."} )
    snake_case__							:   Optional[str]							=						field(
        default="codeparrot/codeparrot-clean-valid"   ,    metadata={"help": "Name or path of validation dataset."} )
    snake_case__							:   Optional[int]							=						field(default=2   ,    metadata={"help": "Batch size for training."} )
    snake_case__							:   Optional[int]							=						field(default=2   ,    metadata={"help": "Batch size for evaluation."} )
    snake_case__							:   Optional[float]							=						field(default=0.1   ,    metadata={"help": "Value of weight decay."} )
    snake_case__							:   Optional[int]							=						field(
        default=1_0_0_0_0   ,    metadata={"help": "Size of buffer used to shuffle streaming dataset."} )
    snake_case__							:   Optional[float]							=						field(default=2e-4   ,    metadata={"help": "Learning rate fo training."} )
    snake_case__							:   Optional[str]							=						field(default="cosine"   ,    metadata={"help": "Learning rate."} )
    snake_case__							:   Optional[int]							=						field(
        default=7_5_0   ,    metadata={"help": "Number of warmup steps in the learning rate schedule."} )
    snake_case__							:   Optional[int]							=						field(
        default=1_6   ,    metadata={"help": "Number of gradient accumulation steps."} )
    snake_case__							:   Optional[bool]							=						field(
        default=_SCREAMING_SNAKE_CASE   ,    metadata={"help": "Use gradient checkpointing to reduce memory footprint."} )
    snake_case__							:   Optional[int]							=						field(default=5_0_0_0_0   ,    metadata={"help": "Maximum number of training steps."} )
    snake_case__							:   Optional[int]							=						field(
        default=-1   ,    metadata={"help": "Maximum number of evaluation steps. If -1 the full dataset is evaluated."} )
    snake_case__							:   Optional[int]							=						field(default=1_0_2_4   ,    metadata={"help": "Sequence lengths used for training."} )
    snake_case__							:   Optional[int]							=						field(default=1   ,    metadata={"help": "Training seed."} )
    snake_case__							:   Optional[int]							=						field(
        default=1_0_2_4   ,    metadata={"help": "Interval to save checkpoints. Measured as number of forward passes not training steps."}   ,    )
    snake_case__							:   Optional[str]							=						field(
        default=_SCREAMING_SNAKE_CASE   ,    metadata={"help": "States path if the training should continue from a checkpoint folder."} )
    snake_case__							:   Optional[bool]							=						field(default=_SCREAMING_SNAKE_CASE   ,    metadata={"help": "If True the data is pretokenized."} )






@dataclass
class 				UpperCAmelCase :




    '''simple docstring'''







    snake_case__							:   Optional[str]							=						field(
        default="codeparrot/codeparrot"   ,    metadata={"help": "Model name or path of model to be evaluated."} )
    snake_case__							:   Optional[str]							=						field(
        default="codeparrot/codeparrot-clean-valid"   ,    metadata={"help": "Name or path of validation dataset."} )
    snake_case__							:   Optional[int]							=						field(default=2   ,    metadata={"help": "Batch size used for evaluation."} )
    snake_case__							:   Optional[int]							=						field(
        default=-1   ,    metadata={"help": "Maximum number of evaluation steps. If -1 the full dataset is evaluated."} )
    snake_case__							:   Optional[int]							=						field(default=1_0_2_4   ,    metadata={"help": "Length of sequences to be evaluated."} )
    snake_case__							:   Optional[int]							=						field(default=1   ,    metadata={"help": "Random seed used for evaluation."} )






@dataclass
class 				UpperCAmelCase :




    '''simple docstring'''







    snake_case__							:   Optional[str]							=						field(
        default="codeparrot/codeparrot"   ,    metadata={"help": "Model name or path of model to be evaluated."} )
    snake_case__							:   Optional[int]							=						field(default=_SCREAMING_SNAKE_CASE   ,    metadata={"help": "Number of workers used for code evaluation."} )
    snake_case__							:   Optional[int]							=						field(
        default=_SCREAMING_SNAKE_CASE   ,    metadata={"help": "The number of human-eval tasks to run. If not included all tasks are evaluated."}   ,    )
    snake_case__							:   Optional[bool]							=						field(
        default=_SCREAMING_SNAKE_CASE   ,    metadata={"help": "Sample from the language model's output distribution."} )
    snake_case__							:   Optional[float]							=						field(default=0.2   ,    metadata={"help": "Sampling temperature used for generation."} )
    snake_case__							:   Optional[int]							=						field(default=2_5_6   ,    metadata={"help": "Maximum number of newly generated tokens."} )
    snake_case__							:   Optional[int]							=						field(default=0   ,    metadata={"help": "Top-k parameter used for generation."} )
    snake_case__							:   Optional[float]							=						field(default=0.95   ,    metadata={"help": "Top-p parameter used for nucleus sampling."} )
    snake_case__							:   Optional[int]							=						field(default=1_0   ,    metadata={"help": "Number of generations to run in parallel."} )
    snake_case__							:   Optional[int]							=						field(
        default=2_0_0   ,    metadata={"help": "Number of completions to generate for each sample."} )
    snake_case__							:   Optional[int]							=						field(default=1   ,    metadata={"help": "Random seed used for evaluation."} )
    snake_case__							:   Optional[str]							=						field(
        default="eval_results.json"   ,    metadata={"help": "Random seed used for evaluation."} )
    snake_case__							:   Optional[str]							=						field(
        default="0"   ,    metadata={"help": "Allow `code_eval` to execute Python code on machine"} )
    snake_case__							:   Optional[int]							=						field(
        default=-1   ,    metadata={
            "help": (
                "Determine which device to run the `text-generation` Pipeline on. -1 is CPU and any zero or positive"
                " number corresponds to which GPU device id to run on."
            )
        }   ,    )






@dataclass
class 				UpperCAmelCase :




    '''simple docstring'''







    snake_case__							:   Optional[int]							=						field(
        default=_SCREAMING_SNAKE_CASE   ,    metadata={
            "help": "The number of CPU cores to use for parallel preprocessing. Default uses the maximum available."
        }   ,    )
    snake_case__							:   Optional[str]							=						field(
        default="transformersbook/codeparrot"   ,    metadata={"help": "Folder or name of dataset to process."} )
    snake_case__							:   Optional[str]							=						field(
        default="codeparrot-clean"   ,    metadata={"help": "Folder to save processed processed dataset."} )
    snake_case__							:   Optional[int]							=						field(
        default=1_0_0_0_0_0   ,    metadata={"help": "Number of files to save per JSON output file."} )
    snake_case__							:   Optional[str]							=						field(default="content"   ,    metadata={"help": "Column containing text data to process."} )
    snake_case__							:   Optional[float]							=						field(
        default=1_0_0_0   ,    metadata={"help": "Maximum line length in file, otherwise file is filtered."} )
    snake_case__							:   Optional[float]							=						field(
        default=1_0_0   ,    metadata={"help": "Maximum mean line length in file, otherwise file is filtered."} )
    snake_case__							:   Optional[float]							=						field(
        default=0.25   ,    metadata={"help": "Maximum fraction of non-alphanumeric characters, otherwise file is filtered."} )
    snake_case__							:   Optional[float]							=						field(
        default=1.5   ,    metadata={"help": "Minimum character token ratio for the file, otherwise file is filtered."} )
    snake_case__							:   Optional[float]							=						field(
        default=0.7   ,    metadata={"help": "Probability for filtering config, test and uncommon files."} )
    snake_case__							:   Optional[str]							=						field(
        default="codeparrot/codeparrot"   ,    metadata={"help": "Name or path to the tokenizer."}   ,    )
    snake_case__							:   Optional[bool]							=						field(
        default=_SCREAMING_SNAKE_CASE   ,    metadata={"help": "If True, near-duplicate samples are removed."} )
    snake_case__							:   Optional[float]							=						field(
        default=0.85   ,    metadata={"help": "Jaccard threshold for near-duplicate samples."} )






@dataclass
class 				UpperCAmelCase :




    '''simple docstring'''







    snake_case__							:   Optional[str]							=						field(
        default="gpt2"   ,    metadata={"help": "Base tokenizer to build new tokenizer from."} )
    snake_case__							:   Optional[str]							=						field(
        default="transformersbook/codeparrot-train"   ,    metadata={"help": "Dataset to train tokenizer on."} )
    snake_case__							:   Optional[str]							=						field(default="content"   ,    metadata={"help": "Column containing text data to process."} )
    snake_case__							:   Optional[int]							=						field(default=2_0_0_0_0_0   ,    metadata={"help": "Number of examples to train tokenizer on."} )
    snake_case__							:   Optional[int]							=						field(
        default=3_2_7_6_8   ,    metadata={"help": "Number of examples to train the tokenizer on."} )
    snake_case__							:   Optional[str]							=						field(default="codeparrot"   ,    metadata={"help": "Name of new tokenizer."} )
    snake_case__							:   Optional[bool]							=						field(default=_SCREAMING_SNAKE_CASE   ,    metadata={"help": "Push saved tokenizer to the hub."} )






@dataclass
class 				UpperCAmelCase :




    '''simple docstring'''







    snake_case__							:   Optional[str]							=						field(
        default="codeparrot/codeparrot"   ,    metadata={"help": "Name or path to the tokenizer."} )
    snake_case__							:   Optional[str]							=						field(
        default="codeparrot/codeparrot-clean-train"   ,    metadata={"help": "Name or path to the dataset to pretokenize."} )
    snake_case__							:   Optional[str]							=						field(
        default="tokenized-codeparrot-train"   ,    metadata={"help": "Repo name of the pretokenized data."} )
    snake_case__							:   Optional[int]							=						field(default=_SCREAMING_SNAKE_CASE   ,    metadata={"help": "Number of workers used for code evaluation."} )






@dataclass
class 				UpperCAmelCase :




    '''simple docstring'''







    snake_case__							:   Optional[str]							=						field(
        default="gpt2-large"   ,    metadata={"help": "Configuration to use for model initialization."} )
    snake_case__							:   Optional[str]							=						field(
        default="codeparrot/codeparrot"   ,    metadata={"help": "Tokenizer attached to model."} )
    snake_case__							:   Optional[str]							=						field(default="codeparrot"   ,    metadata={"help": "Name of the created model."} )
    snake_case__							:   Optional[bool]							=						field(default=_SCREAMING_SNAKE_CASE   ,    metadata={"help": "Push saved tokenizer to the hub."} )




'''simple docstring'''





import copy
import json
import os
import tempfile

from transformers import is_torch_available

from .test_configuration_utils import config_common_kwargs






class 				UpperCAmelCase (       _SCREAMING_SNAKE_CASE ):




    '''simple docstring'''







    def __init__(       self    ,							lowercase__    ,							lowercase__=None    ,							lowercase__=True    ,							lowercase__=None    ,							**lowercase__ )							-> Optional[Any]:
         SCREAMING_SNAKE_CASE					:			int    	=       parent
         SCREAMING_SNAKE_CASE					:			str    	=       config_class
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       has_text_modality
         SCREAMING_SNAKE_CASE					:			Any    	=       kwargs
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       common_properties






    def       _UpperCamelCase    (       self )							-> Union[str, Any]:
         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       self.config_class(**self.inputs_dict )
         SCREAMING_SNAKE_CASE					:			int    	=       (
             ['hidden_size', 'num_attention_heads', 'num_hidden_layers']
             if self.common_properties is None
             else self.common_properties
         )

         # Add common fields for text models
         if self.has_text_modality:
              common_properties.extend(['vocab_size'] )

         # Test that config has the common properties as getters
         for prop in common_properties:
              self.parent.assertTrue(hasattr(lowercase__    ,							lowercase__ )    ,							msg=F"""`{prop}` does not exist""" )

         # Test that config has the common properties as setter
         for idx, name in enumerate(lowercase__ ):
              try:
                   setattr(lowercase__    ,							lowercase__    ,							lowercase__ )
                   self.parent.assertEqual(
                       getattr(lowercase__    ,							lowercase__ )    ,							lowercase__    ,							msg=F"""`{name} value {idx} expected, but was {getattr(lowercase__    ,							lowercase__ )}""" )
              except NotImplementedError:
                   # Some models might not be able to implement setters for common_properties
                   # In that case, a NotImplementedError is raised
                   pass

        # Test if config class can be called with Config(prop_name=..)
         for idx, name in enumerate(lowercase__ ):
              try:
                   SCREAMING_SNAKE_CASE					:			Optional[int]    	=       self.config_class(**{name: idx} )
                   self.parent.assertEqual(
                       getattr(lowercase__    ,							lowercase__ )    ,							lowercase__    ,							msg=F"""`{name} value {idx} expected, but was {getattr(lowercase__    ,							lowercase__ )}""" )
              except NotImplementedError:
                   # Some models might not be able to implement setters for common_properties
                   # In that case, a NotImplementedError is raised
                   pass






    def       _UpperCamelCase    (       self )							-> List[str]:
         SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       self.config_class(**self.inputs_dict )
         SCREAMING_SNAKE_CASE					:			Any    	=       json.loads(config.to_json_string() )
         for key, value in self.inputs_dict.items():
              self.parent.assertEqual(obj[key]    ,							lowercase__ )






    def       _UpperCamelCase    (       self )							-> Optional[int]:
         SCREAMING_SNAKE_CASE					:			Tuple    	=       self.config_class(**self.inputs_dict )

         with tempfile.TemporaryDirectory() as tmpdirname:
              SCREAMING_SNAKE_CASE					:			List[str]    	=       os.path.join(lowercase__    ,							'config.json' )
              config_first.to_json_file(lowercase__ )
              SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       self.config_class.from_json_file(lowercase__ )

         self.parent.assertEqual(config_second.to_dict()    ,							config_first.to_dict() )






    def       _UpperCamelCase    (       self )							-> Dict:
         SCREAMING_SNAKE_CASE					:			Optional[Any]    	=       self.config_class(**self.inputs_dict )

         with tempfile.TemporaryDirectory() as tmpdirname:
              config_first.save_pretrained(lowercase__ )
              SCREAMING_SNAKE_CASE					:			Optional[int]    	=       self.config_class.from_pretrained(lowercase__ )

         self.parent.assertEqual(config_second.to_dict()    ,							config_first.to_dict() )






    def       _UpperCamelCase    (       self )							-> int:
         SCREAMING_SNAKE_CASE					:			List[Any]    	=       self.config_class(**self.inputs_dict )

         SCREAMING_SNAKE_CASE					:			Optional[int]    	=       'test'
         with tempfile.TemporaryDirectory() as tmpdirname:
              SCREAMING_SNAKE_CASE					:			Tuple    	=       os.path.join(lowercase__    ,							lowercase__ )
              config_first.save_pretrained(lowercase__ )
              SCREAMING_SNAKE_CASE					:			Dict    	=       self.config_class.from_pretrained(lowercase__    ,							subfolder=lowercase__ )

         self.parent.assertEqual(config_second.to_dict()    ,							config_first.to_dict() )






    def       _UpperCamelCase    (       self )							-> List[Any]:
         SCREAMING_SNAKE_CASE					:			Dict    	=       self.config_class(**self.inputs_dict    ,							num_labels=5 )
         self.parent.assertEqual(len(config.idalabel )    ,							5 )
         self.parent.assertEqual(len(config.labelaid )    ,							5 )

         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       3
         self.parent.assertEqual(len(config.idalabel )    ,							3 )
         self.parent.assertEqual(len(config.labelaid )    ,							3 )






    def       _UpperCamelCase    (       self )							-> Dict:
         if self.config_class.is_composition:
              return
         SCREAMING_SNAKE_CASE					:			int    	=       self.config_class()
         self.parent.assertIsNotNone(lowercase__ )






    def       _UpperCamelCase    (       self )							-> Tuple:
         SCREAMING_SNAKE_CASE					:			Union[str, Any]    	=       copy.deepcopy(lowercase__ )
         SCREAMING_SNAKE_CASE					:			Any    	=       self.config_class(**lowercase__ )
         SCREAMING_SNAKE_CASE					:			str    	=       []
         for key, value in config_common_kwargs.items():
              if key == "torch_dtype":
                   if not is_torch_available():
                        continue
                   else:
                        import torch

                        if config.torch_dtype != torch.floataa:
                             wrong_values.append(('torch_dtype', config.torch_dtype, torch.floataa) )
              elif getattr(lowercase__    ,							lowercase__ ) != value:
                   wrong_values.append((key, getattr(lowercase__    ,							lowercase__ ), value) )

         if len(lowercase__ ) > 0:
              SCREAMING_SNAKE_CASE					:			List[Any]    	=       '\n'.join([F"""- {v[0]}: got {v[1]} instead of {v[2]}""" for v in wrong_values] )
              raise ValueError(F"""The following keys were not properly set in the config:\n{errors}""" )






    def       _UpperCamelCase    (       self )							-> Optional[int]:
         self.create_and_test_config_common_properties()
         self.create_and_test_config_to_json_string()
         self.create_and_test_config_to_json_file()
         self.create_and_test_config_from_and_save_pretrained()
         self.create_and_test_config_from_and_save_pretrained_subfolder()
         self.create_and_test_config_with_num_labels()
         self.check_config_can_be_init_without_params()
         self.check_config_arguments_init()



