
"""simple docstring"""
from __future__ import annotations

import unittest

from transformers import FunnelConfig, is_tf_available
from transformers.testing_utils import require_tf

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers import (
          TFFunnelBaseModel,
          TFFunnelForMaskedLM,
          TFFunnelForMultipleChoice,
          TFFunnelForPreTraining,
          TFFunnelForQuestionAnswering,
          TFFunnelForSequenceClassification,
          TFFunnelForTokenClassification,
          TFFunnelModel,
      )
class 			UpperCamelCase      :
     def __init__(     self  ,     snake_case__  ,     snake_case__=13  ,     snake_case__=7  ,     snake_case__=True  ,     snake_case__=True  ,     snake_case__=True  ,     snake_case__=True  ,     snake_case__=99  ,     snake_case__=[1, 1, 2]  ,     snake_case__=1  ,     snake_case__=32  ,     snake_case__=4  ,     snake_case__=8  ,     snake_case__=37  ,     snake_case__="gelu_new"  ,     snake_case__=0.1  ,     snake_case__=0.1  ,     snake_case__=0.0  ,     snake_case__=512  ,     snake_case__=3  ,     snake_case__=0.02  ,     snake_case__=3  ,     snake_case__=4  ,     snake_case__=None  ,     snake_case__=False  ,     ):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				str							    = parent
       _SCREAMING_SNAKE_CASE   :				List[str]							    = batch_size
       _SCREAMING_SNAKE_CASE   :				Any							    = seq_length
       _SCREAMING_SNAKE_CASE   :				List[str]							    = is_training
       _SCREAMING_SNAKE_CASE   :				List[str]							    = use_input_mask
       _SCREAMING_SNAKE_CASE   :				List[str]							    = use_token_type_ids
       _SCREAMING_SNAKE_CASE   :				Any							    = use_labels
       _SCREAMING_SNAKE_CASE   :				str							    = vocab_size
       _SCREAMING_SNAKE_CASE   :				str							    = block_sizes
       _SCREAMING_SNAKE_CASE   :				int							    = num_decoder_layers
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = d_model
       _SCREAMING_SNAKE_CASE   :				int							    = n_head
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = d_head
       _SCREAMING_SNAKE_CASE   :				Tuple							    = d_inner
       _SCREAMING_SNAKE_CASE   :				List[str]							    = hidden_act
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = hidden_dropout
       _SCREAMING_SNAKE_CASE   :				str							    = attention_dropout
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = activation_dropout
       _SCREAMING_SNAKE_CASE   :				str							    = max_position_embeddings
       _SCREAMING_SNAKE_CASE   :				Tuple							    = type_vocab_size
       _SCREAMING_SNAKE_CASE   :				Any							    = 2
       _SCREAMING_SNAKE_CASE   :				Any							    = num_labels
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = num_choices
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = scope
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = initializer_std

       # Used in the tests to check the size of the first attention layer
       _SCREAMING_SNAKE_CASE   :				Dict							    = n_head
       # Used in the tests to check the size of the first hidden state
       _SCREAMING_SNAKE_CASE   :				List[str]							    = self.d_model
       # Used in the tests to check the number of output hidden states/attentions
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = sum(self.block_sizes				) + (0 if base else self.num_decoder_layers)
       # FunnelModel adds two hidden layers: input embeddings and the sum of the upsampled encoder hidden state with
       # the last hidden state of the first block (which is the first hidden state of the decoder).
       if not base:
         _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = self.num_hidden_layers + 2
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				int							    = ids_tensor([self.batch_size, self.seq_length]  ,     self.vocab_size				)

       _SCREAMING_SNAKE_CASE   :				Dict							    = None
       if self.use_input_mask:
         _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = random_attention_mask([self.batch_size, self.seq_length]				)

       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = None
       if self.use_token_type_ids:
         _SCREAMING_SNAKE_CASE   :				Tuple							    = ids_tensor([self.batch_size, self.seq_length]  ,     self.type_vocab_size				)

       _SCREAMING_SNAKE_CASE   :				List[str]							    = None
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = None
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = None
       if self.use_labels:
         _SCREAMING_SNAKE_CASE   :				str							    = ids_tensor([self.batch_size]  ,     self.type_sequence_label_size				)
         _SCREAMING_SNAKE_CASE   :				List[str]							    = ids_tensor([self.batch_size, self.seq_length]  ,     self.num_labels				)
         _SCREAMING_SNAKE_CASE   :				Any							    = ids_tensor([self.batch_size]  ,     self.num_choices				)

       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = FunnelConfig(
           vocab_size=self.vocab_size  ,     block_sizes=self.block_sizes  ,     num_decoder_layers=self.num_decoder_layers  ,     d_model=self.d_model  ,     n_head=self.n_head  ,     d_head=self.d_head  ,     d_inner=self.d_inner  ,     hidden_act=self.hidden_act  ,     hidden_dropout=self.hidden_dropout  ,     attention_dropout=self.attention_dropout  ,     activation_dropout=self.activation_dropout  ,     max_position_embeddings=self.max_position_embeddings  ,     type_vocab_size=self.type_vocab_size  ,     initializer_std=self.initializer_std  ,     )

       return (
           config,
           input_ids,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
           choice_labels,
       )
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     ):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Dict							    = TFFunnelModel(config=__A				)
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       _SCREAMING_SNAKE_CASE   :				Any							    = model(__A				)

       _SCREAMING_SNAKE_CASE   :				List[str]							    = [input_ids, input_mask]
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = model(__A				)

       _SCREAMING_SNAKE_CASE   :				Dict							    = model(__A				)
       self.parent.assertEqual(result.last_hidden_state.shape  ,     (self.batch_size, self.seq_length, self.d_model)				)

       _SCREAMING_SNAKE_CASE   :				Any							    = False
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = TFFunnelModel(config=__A				)
       _SCREAMING_SNAKE_CASE   :				Any							    = model(__A				)
       self.parent.assertEqual(result.last_hidden_state.shape  ,     (self.batch_size, self.seq_length, self.d_model)				)

       _SCREAMING_SNAKE_CASE   :				int							    = False
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = TFFunnelModel(config=__A				)
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = model(__A				)
       self.parent.assertEqual(result.last_hidden_state.shape  ,     (self.batch_size, self.seq_length, self.d_model)				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     ):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = TFFunnelBaseModel(config=__A				)
       _SCREAMING_SNAKE_CASE   :				Dict							    = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       _SCREAMING_SNAKE_CASE   :				Tuple							    = model(__A				)

       _SCREAMING_SNAKE_CASE   :				Any							    = [input_ids, input_mask]
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = model(__A				)

       _SCREAMING_SNAKE_CASE   :				List[Any]							    = model(__A				)
       self.parent.assertEqual(result.last_hidden_state.shape  ,     (self.batch_size, 2, self.d_model)				)

       _SCREAMING_SNAKE_CASE   :				str							    = False
       _SCREAMING_SNAKE_CASE   :				Dict							    = TFFunnelBaseModel(config=__A				)
       _SCREAMING_SNAKE_CASE   :				str							    = model(__A				)
       self.parent.assertEqual(result.last_hidden_state.shape  ,     (self.batch_size, 3, self.d_model)				)

       _SCREAMING_SNAKE_CASE   :				int							    = False
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = TFFunnelBaseModel(config=__A				)
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = model(__A				)
       self.parent.assertEqual(result.last_hidden_state.shape  ,     (self.batch_size, 2, self.d_model)				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     ):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				str							    = TFFunnelForPreTraining(config=__A				)
       _SCREAMING_SNAKE_CASE   :				List[str]							    = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       _SCREAMING_SNAKE_CASE   :				str							    = model(__A				)
       self.parent.assertEqual(result.logits.shape  ,     (self.batch_size, self.seq_length)				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     ):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				List[Any]							    = TFFunnelForMaskedLM(config=__A				)
       _SCREAMING_SNAKE_CASE   :				str							    = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = model(__A				)
       self.parent.assertEqual(result.logits.shape  ,     (self.batch_size, self.seq_length, self.vocab_size)				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     ):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = self.num_labels
       _SCREAMING_SNAKE_CASE   :				int							    = TFFunnelForSequenceClassification(config=__A				)
       _SCREAMING_SNAKE_CASE   :				List[str]							    = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       _SCREAMING_SNAKE_CASE   :				Dict							    = model(__A				)
       self.parent.assertEqual(result.logits.shape  ,     (self.batch_size, self.num_labels)				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     ):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = self.num_choices
       _SCREAMING_SNAKE_CASE   :				Tuple							    = TFFunnelForMultipleChoice(config=__A				)
       _SCREAMING_SNAKE_CASE   :				Tuple							    = tf.tile(tf.expand_dims(__A  ,     1				)  ,     (1, self.num_choices, 1)				)
       _SCREAMING_SNAKE_CASE   :				str							    = tf.tile(tf.expand_dims(__A  ,     1				)  ,     (1, self.num_choices, 1)				)
       _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = tf.tile(tf.expand_dims(__A  ,     1				)  ,     (1, self.num_choices, 1)				)
       _SCREAMING_SNAKE_CASE   :				Tuple							    = {
           "input_ids": multiple_choice_inputs_ids,
           "attention_mask": multiple_choice_input_mask,
           "token_type_ids": multiple_choice_token_type_ids,
       }
       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = model(__A				)
       self.parent.assertEqual(result.logits.shape  ,     (self.batch_size, self.num_choices)				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     ):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				List[Any]							    = self.num_labels
       _SCREAMING_SNAKE_CASE   :				Tuple							    = TFFunnelForTokenClassification(config=__A				)
       _SCREAMING_SNAKE_CASE   :				str							    = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       _SCREAMING_SNAKE_CASE   :				Tuple							    = model(__A				)
       self.parent.assertEqual(result.logits.shape  ,     (self.batch_size, self.seq_length, self.num_labels)				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     snake_case__  ,     ):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Tuple							    = TFFunnelForQuestionAnswering(config=__A				)
       _SCREAMING_SNAKE_CASE   :				Dict							    = {"input_ids": input_ids, "attention_mask": input_mask, "token_type_ids": token_type_ids}
       _SCREAMING_SNAKE_CASE   :				Any							    = model(__A				)
       self.parent.assertEqual(result.start_logits.shape  ,     (self.batch_size, self.seq_length)				)
       self.parent.assertEqual(result.end_logits.shape  ,     (self.batch_size, self.seq_length)				)




     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				int							    = self.prepare_config_and_inputs()
       (
           (
           _SCREAMING_SNAKE_CASE				
       )    ,		(
           _SCREAMING_SNAKE_CASE				
       )    ,		(
           _SCREAMING_SNAKE_CASE				
       )    ,		(
           _SCREAMING_SNAKE_CASE				
       )    ,		(
           _SCREAMING_SNAKE_CASE				
       )    ,		(
           _SCREAMING_SNAKE_CASE				
       )    ,		(
           _SCREAMING_SNAKE_CASE				
       )    ,						
       )   :				int							    = config_and_inputs
       _SCREAMING_SNAKE_CASE   :				Dict							    = {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": input_mask}
       return config, inputs_dict
@require_tf
class 			UpperCamelCase      (							__lowercase      ,       __lowercase      ,       unittest.TestCase  ):
     A__      =			(
         (
             TFFunnelModel,
             TFFunnelForMaskedLM,
             TFFunnelForPreTraining,
             TFFunnelForQuestionAnswering,
             TFFunnelForTokenClassification,
         )
         if is_tf_available()
         else ()
     )
     A__      =			(
         {
             'feature-extraction': (TFFunnelBaseModel, TFFunnelModel),
             'fill-mask': TFFunnelForMaskedLM,
             'question-answering': TFFunnelForQuestionAnswering,
             'text-classification': TFFunnelForSequenceClassification,
             'token-classification': TFFunnelForTokenClassification,
             'zero-shot': TFFunnelForSequenceClassification,
         }
         if is_tf_available()
         else {}
     )
     A__      =			False
     A__      =			False
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				str							    = TFFunnelModelTester(self				)
       _SCREAMING_SNAKE_CASE   :				int							    = ConfigTester(self  ,     config_class=__A				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       self.config_tester.run_common_tests()
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Dict							    = self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*__A				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Any							    = self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_pretraining(*__A				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				List[Any]							    = self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_masked_lm(*__A				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Tuple							    = self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*__A				)




     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				str							    = self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*__A				)
@require_tf
class 			UpperCamelCase      (							__lowercase      ,       unittest.TestCase  ):
     A__      =			(
         (TFFunnelBaseModel, TFFunnelForMultipleChoice, TFFunnelForSequenceClassification) if is_tf_available() else ()
     )
     A__      =			False
     A__      =			False
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				int							    = TFFunnelModelTester(self  ,     base=__A				)
       _SCREAMING_SNAKE_CASE   :				Tuple							    = ConfigTester(self  ,     config_class=__A				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       self.config_tester.run_common_tests()
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				str							    = self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_base_model(*__A				)
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*__A				)




     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				str							    = self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_multiple_choice(*__A				)





"""simple docstring"""
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Model name or path of model to be trained."""}  )
     A__      =			field(
         default="""./"""      ,       metadata={"""help""": """Save dir where model repo is cloned and models updates are saved to."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-train"""      ,       metadata={"""help""": """Name or path of training dataset."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-valid"""      ,       metadata={"""help""": """Name or path of validation dataset."""}  )
     A__      =			field(default=2      ,       metadata={"""help""": """Batch size for training."""}  )
     A__      =			field(default=2      ,       metadata={"""help""": """Batch size for evaluation."""}  )
     A__      =			field(default=0.1      ,       metadata={"""help""": """Value of weight decay."""}  )
     A__      =			field(
         default=10000      ,       metadata={"""help""": """Size of buffer used to shuffle streaming dataset."""}  )
     A__      =			field(default=2e-4      ,       metadata={"""help""": """Learning rate fo training."""}  )
     A__      =			field(default="""cosine"""      ,       metadata={"""help""": """Learning rate."""}  )
     A__      =			field(
         default=750      ,       metadata={"""help""": """Number of warmup steps in the learning rate schedule."""}  )
     A__      =			field(
         default=16      ,       metadata={"""help""": """Number of gradient accumulation steps."""}  )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Use gradient checkpointing to reduce memory footprint."""}  )
     A__      =			field(default=50000      ,       metadata={"""help""": """Maximum number of training steps."""}  )
     A__      =			field(
         default=-1      ,       metadata={"""help""": """Maximum number of evaluation steps. If -1 the full dataset is evaluated."""}  )
     A__      =			field(default=1024      ,       metadata={"""help""": """Sequence lengths used for training."""}  )
     A__      =			field(default=1      ,       metadata={"""help""": """Training seed."""}  )
     A__      =			field(
         default=1024      ,       metadata={"""help""": """Interval to save checkpoints. Measured as number of forward passes not training steps."""}      ,       )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """States path if the training should continue from a checkpoint folder."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """If True the data is pretokenized."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Model name or path of model to be evaluated."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-valid"""      ,       metadata={"""help""": """Name or path of validation dataset."""}  )
     A__      =			field(default=2      ,       metadata={"""help""": """Batch size used for evaluation."""}  )
     A__      =			field(
         default=-1      ,       metadata={"""help""": """Maximum number of evaluation steps. If -1 the full dataset is evaluated."""}  )
     A__      =			field(default=1024      ,       metadata={"""help""": """Length of sequences to be evaluated."""}  )
     A__      =			field(default=1      ,       metadata={"""help""": """Random seed used for evaluation."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Model name or path of model to be evaluated."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Number of workers used for code evaluation."""}  )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """The number of human-eval tasks to run. If not included all tasks are evaluated."""}      ,       )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Sample from the language model's output distribution."""}  )
     A__      =			field(default=0.2      ,       metadata={"""help""": """Sampling temperature used for generation."""}  )
     A__      =			field(default=256      ,       metadata={"""help""": """Maximum number of newly generated tokens."""}  )
     A__      =			field(default=0      ,       metadata={"""help""": """Top-k parameter used for generation."""}  )
     A__      =			field(default=0.95      ,       metadata={"""help""": """Top-p parameter used for nucleus sampling."""}  )
     A__      =			field(default=10      ,       metadata={"""help""": """Number of generations to run in parallel."""}  )
     A__      =			field(
         default=200      ,       metadata={"""help""": """Number of completions to generate for each sample."""}  )
     A__      =			field(default=1      ,       metadata={"""help""": """Random seed used for evaluation."""}  )
     A__      =			field(
         default="""eval_results.json"""      ,       metadata={"""help""": """Random seed used for evaluation."""}  )
     A__      =			field(
         default="""0"""      ,       metadata={"""help""": """Allow `code_eval` to execute Python code on machine"""}  )
     A__      =			field(
         default=-1      ,       metadata={
             """help""": (
                 """Determine which device to run the `text-generation` Pipeline on. -1 is CPU and any zero or positive"""
                 """ number corresponds to which GPU device id to run on."""
             )
         }      ,       )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={
             """help""": """The number of CPU cores to use for parallel preprocessing. Default uses the maximum available."""
         }      ,       )
     A__      =			field(
         default="""transformersbook/codeparrot"""      ,       metadata={"""help""": """Folder or name of dataset to process."""}  )
     A__      =			field(
         default="""codeparrot-clean"""      ,       metadata={"""help""": """Folder to save processed processed dataset."""}  )
     A__      =			field(
         default=100000      ,       metadata={"""help""": """Number of files to save per JSON output file."""}  )
     A__      =			field(default="""content"""      ,       metadata={"""help""": """Column containing text data to process."""}  )
     A__      =			field(
         default=1000      ,       metadata={"""help""": """Maximum line length in file, otherwise file is filtered."""}  )
     A__      =			field(
         default=100      ,       metadata={"""help""": """Maximum mean line length in file, otherwise file is filtered."""}  )
     A__      =			field(
         default=0.25      ,       metadata={"""help""": """Maximum fraction of non-alphanumeric characters, otherwise file is filtered."""}  )
     A__      =			field(
         default=1.5      ,       metadata={"""help""": """Minimum character token ratio for the file, otherwise file is filtered."""}  )
     A__      =			field(
         default=0.7      ,       metadata={"""help""": """Probability for filtering config, test and uncommon files."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Name or path to the tokenizer."""}      ,       )
     A__      =			field(
         default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """If True, near-duplicate samples are removed."""}  )
     A__      =			field(
         default=0.85      ,       metadata={"""help""": """Jaccard threshold for near-duplicate samples."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""gpt2"""      ,       metadata={"""help""": """Base tokenizer to build new tokenizer from."""}  )
     A__      =			field(
         default="""transformersbook/codeparrot-train"""      ,       metadata={"""help""": """Dataset to train tokenizer on."""}  )
     A__      =			field(default="""content"""      ,       metadata={"""help""": """Column containing text data to process."""}  )
     A__      =			field(default=200000      ,       metadata={"""help""": """Number of examples to train tokenizer on."""}  )
     A__      =			field(
         default=32768      ,       metadata={"""help""": """Number of examples to train the tokenizer on."""}  )
     A__      =			field(default="""codeparrot"""      ,       metadata={"""help""": """Name of new tokenizer."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Push saved tokenizer to the hub."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Name or path to the tokenizer."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot-clean-train"""      ,       metadata={"""help""": """Name or path to the dataset to pretokenize."""}  )
     A__      =			field(
         default="""tokenized-codeparrot-train"""      ,       metadata={"""help""": """Repo name of the pretokenized data."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Number of workers used for code evaluation."""}  )
@dataclass
class 			UpperCamelCase      :
     A__      =			field(
         default="""gpt2-large"""      ,       metadata={"""help""": """Configuration to use for model initialization."""}  )
     A__      =			field(
         default="""codeparrot/codeparrot"""      ,       metadata={"""help""": """Tokenizer attached to model."""}  )
     A__      =			field(default="""codeparrot"""      ,       metadata={"""help""": """Name of the created model."""}  )
     A__      =			field(default=__SCREAMING_SNAKE_CASE      ,       metadata={"""help""": """Push saved tokenizer to the hub."""}  )



