



"""simple docstring"""





import uuid
from typing import Any, Dict, List, Optional, Union

from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
  import tensorflow as tf

if is_torch_available():
  import torch


lowerCamelCase  			=		logging.get_logger(__name__)
class  lowercase__  :







       '''simple docstring'''


       def __init__(				self     :    List[Any]						,							_UpperCAmelCase     :    str = None						,							_UpperCAmelCase     :    uuid.UUID = None						,							_UpperCAmelCase     :    List[Any]=None						,							_UpperCAmelCase     :    List[str]=None       )					->		List[str]:


             '''simple docstring'''




             if not conversation_id:
                   UpperCAmelCase_     					=				uuid.uuida()
             if past_user_inputs is None:
                   UpperCAmelCase_     					=				[]
             if generated_responses is None:
                   UpperCAmelCase_     					=				[]

             UpperCAmelCase_     					=				conversation_id
             UpperCAmelCase_     					=				past_user_inputs
             UpperCAmelCase_     					=				generated_responses
             UpperCAmelCase_     					=				text



       def __eq__(				self     :    Optional[int]						,							_UpperCAmelCase     :    List[str]       )					->		Tuple:


             '''simple docstring'''




             if not isinstance(_UpperCAmelCase						,							_UpperCAmelCase       ):
                   return False
             if self.uuid == other.uuid:
                   return True
             return (
                 self.new_user_input == other.new_user_input
                 and self.past_user_inputs == other.past_user_inputs
                 and self.generated_responses == other.generated_responses
             )



       def 	lowercase__							(				self     :    Optional[int]						,							_UpperCAmelCase     :    str						,							_UpperCAmelCase     :    bool = False       )					->		Optional[Any]:


             '''simple docstring'''




             if self.new_user_input:
                   if overwrite:
                         logger.warning(
                             F"""User input added while unprocessed input was existing: \"{self.new_user_input}\" was overwritten """
                             F"""with: \"{text}\"."""       )
                         UpperCAmelCase_     					=				text
                   else:
                         logger.warning(
                             F"""User input added while unprocessed input was existing: \"{self.new_user_input}\" new input """
                             F"""ignored: \"{text}\". Set `overwrite` to True to overwrite unprocessed user input"""       )
             else:
                   UpperCAmelCase_     					=				text



       def 	lowercase__							(				self     :    int       )					->		Any:


             '''simple docstring'''




             if self.new_user_input:
                   self.past_user_inputs.append(self.new_user_input       )
             UpperCAmelCase_     					=				None



       def 	lowercase__							(				self     :    Any						,							_UpperCAmelCase     :    str       )					->		Union[str, Any]:


             '''simple docstring'''




             self.generated_responses.append(_UpperCAmelCase       )



       def 	lowercase__							(				self     :    str       )					->		List[Any]:


             '''simple docstring'''




             for user_input, generated_response in zip(self.past_user_inputs						,							self.generated_responses       ):
                   yield True, user_input
                   yield False, generated_response
             if self.new_user_input:
                   yield True, self.new_user_input







       def __repr__(				self     :    List[Any]       )					->		int:


             '''simple docstring'''




             UpperCAmelCase_     					=				F"""Conversation id: {self.uuid} \n"""
             for is_user, text in self.iter_texts():
                   UpperCAmelCase_     					=				"user" if is_user else "bot"
                   output += F"""{name} >> {text} \n"""
             return output



@add_end_docstrings(
    SCREAMING_SNAKE_CASE							,   R'''
        min_length_for_response (`int`, *optional*, defaults to 32):
            The minimum length (in number of tokens) for a response.
        minimum_tokens (`int`, *optional*, defaults to 10):
            The minimum length of tokens to leave for a response.
    '''							,   )
class  lowercase__  (       SCREAMING_SNAKE_CASE ):







       '''simple docstring'''


       def __init__(				self     :    Dict						,							*_UpperCAmelCase     :    List[Any]						,							**_UpperCAmelCase     :    Union[str, Any]       )					->		Union[str, Any]:


             '''simple docstring'''




             super().__init__(*_UpperCAmelCase						,							**_UpperCAmelCase       )
             if self.tokenizer.pad_token_id is None:
                   UpperCAmelCase_     					=				self.tokenizer.eos_token



       def 	lowercase__							(				self     :    int						,							_UpperCAmelCase     :    Union[str, Any]=None						,							_UpperCAmelCase     :    Dict=None						,							_UpperCAmelCase     :    Optional[Any]=None						,							**_UpperCAmelCase     :    Any       )					->		List[str]:


             '''simple docstring'''




             UpperCAmelCase_     					=				{}
             UpperCAmelCase_     					=				{}
             UpperCAmelCase_     					=				{}

             if min_length_for_response is not None:
                   UpperCAmelCase_     					=				min_length_for_response
             if minimum_tokens is not None:
                   UpperCAmelCase_     					=				minimum_tokens

             if "max_length" in generate_kwargs:
                   UpperCAmelCase_     					=				generate_kwargs["max_length"]
                   # self.max_length = generate_kwargs.get("max_length", self.model.config.max_length)
             if clean_up_tokenization_spaces is not None:
                   UpperCAmelCase_     					=				clean_up_tokenization_spaces

             if generate_kwargs:
                   forward_params.update(_UpperCAmelCase       )
             return preprocess_params, forward_params, postprocess_params



       def __call__(				self     :    Any						,							_UpperCAmelCase     :    Union[Conversation, List[Conversation]]						,							_UpperCAmelCase     :    Optional[int]=0						,							**_UpperCAmelCase     :    Dict       )					->		Tuple:


             '''simple docstring'''




             UpperCAmelCase_     					=				super().__call__(_UpperCAmelCase						,							num_workers=_UpperCAmelCase						,							**_UpperCAmelCase       )
             if isinstance(_UpperCAmelCase						,							_UpperCAmelCase       ) and len(_UpperCAmelCase       ) == 1:
                   return outputs[0]
             return outputs



       def 	lowercase__							(				self     :    str						,							_UpperCAmelCase     :    Conversation						,							_UpperCAmelCase     :    Dict=32       )					->		Dict[str, Any]:


             '''simple docstring'''




             if not isinstance(_UpperCAmelCase						,							_UpperCAmelCase       ):
                   raise ValueError("ConversationalPipeline, expects Conversation as inputs"       )
             if conversation.new_user_input is None:
                   raise ValueError(
                       F"""Conversation with UUID {type(conversation.uuid       )} does not contain new user input to process. """
                       "Add user inputs with the conversation's `add_user_input` method"       )
             if hasattr(self.tokenizer						,							"_build_conversation_input_ids"       ):
                   UpperCAmelCase_     					=				self.tokenizer._build_conversation_input_ids(_UpperCAmelCase       )
             else:
                   # If the tokenizer cannot handle conversations, we default to only the old version
                   UpperCAmelCase_     					=				self._legacy_parse_and_tokenize(_UpperCAmelCase       )

             if self.framework == "pt":
                   UpperCAmelCase_     					=				torch.LongTensor([input_ids]       )
             elif self.framework == "tf":
                   UpperCAmelCase_     					=				tf.constant([input_ids]       )
             return {"input_ids": input_ids, "conversation": conversation}



       def 	lowercase__							(				self     :    int						,							_UpperCAmelCase     :    str						,							_UpperCAmelCase     :    int=10						,							**_UpperCAmelCase     :    Optional[int]       )					->		Tuple:


             '''simple docstring'''




             UpperCAmelCase_     					=				generate_kwargs.get("max_length"						,							self.model.config.max_length       )

             UpperCAmelCase_     					=				model_inputs["input_ids"].shape[1]
             if max_length - minimum_tokens < n:
                   logger.warning(F"""Conversation input is to long ({n}), trimming it to ({max_length} - {minimum_tokens})"""       )
                   UpperCAmelCase_     					=				max_length - minimum_tokens
                   UpperCAmelCase_     					=				model_inputs["input_ids"][:, -trim:]
                   if "attention_mask" in model_inputs:
                         UpperCAmelCase_     					=				model_inputs["attention_mask"][:, -trim:]
             UpperCAmelCase_     					=				model_inputs.pop("conversation"       )
             UpperCAmelCase_     					=				max_length
             UpperCAmelCase_     					=				self.model.generate(**_UpperCAmelCase						,							**_UpperCAmelCase       )
             if self.model.config.is_encoder_decoder:
                   UpperCAmelCase_     					=				1
             else:
                   UpperCAmelCase_     					=				n
             return {"output_ids": output_ids[:, start_position:], "conversation": conversation}



       def 	lowercase__							(				self     :    Tuple						,							_UpperCAmelCase     :    Union[str, Any]						,							_UpperCAmelCase     :    Any=True       )					->		List[Any]:


             '''simple docstring'''




             UpperCAmelCase_     					=				model_outputs["output_ids"]
             UpperCAmelCase_     					=				self.tokenizer.decode(
                 output_ids[0]						,							skip_special_tokens=_UpperCAmelCase						,							clean_up_tokenization_spaces=_UpperCAmelCase						,							)
             UpperCAmelCase_     					=				model_outputs["conversation"]
             conversation.mark_processed()
             conversation.append_response(_UpperCAmelCase       )
             return conversation







       def 	lowercase__							(				self     :    Optional[int]						,							_UpperCAmelCase     :    Conversation       )					->		Dict:


             '''simple docstring'''




             UpperCAmelCase_     					=				self.tokenizer.eos_token_id
             UpperCAmelCase_     					=				[]
             for is_user, text in conversation.iter_texts():
                   if eos_token_id is not None:
                         input_ids.extend(self.tokenizer.encode(_UpperCAmelCase						,							add_special_tokens=_UpperCAmelCase       ) + [eos_token_id]       )
                   else:
                         input_ids.extend(self.tokenizer.encode(_UpperCAmelCase						,							add_special_tokens=_UpperCAmelCase       )       )

             if len(_UpperCAmelCase       ) > self.tokenizer.model_max_length:
                   UpperCAmelCase_     					=				input_ids[-self.tokenizer.model_max_length :]
             return input_ids






from dataclasses import dataclass, field
from typing import Optional




@dataclass
class   __lowercase						:
	_A       =   field(
	    default="codeparrot/codeparrot"   , metadata={"help": "Model name or path of model to be trained."}       )
	_A       =   field(
	    default="./"   , metadata={"help": "Save dir where model repo is cloned and models updates are saved to."}       )
	_A       =   field(
	    default="codeparrot/codeparrot-clean-train"   , metadata={"help": "Name or path of training dataset."}       )
	_A       =   field(
	    default="codeparrot/codeparrot-clean-valid"   , metadata={"help": "Name or path of validation dataset."}       )
	_A       =   field(default=2   , metadata={"help": "Batch size for training."}       )
	_A       =   field(default=2   , metadata={"help": "Batch size for evaluation."}       )
	_A       =   field(default=0.1   , metadata={"help": "Value of weight decay."}       )
	_A       =   field(
	    default=10000   , metadata={"help": "Size of buffer used to shuffle streaming dataset."}       )
	_A       =   field(default=2e-4   , metadata={"help": "Learning rate fo training."}       )
	_A       =   field(default="cosine"   , metadata={"help": "Learning rate."}       )
	_A       =   field(
	    default=750   , metadata={"help": "Number of warmup steps in the learning rate schedule."}       )
	_A       =   field(
	    default=16   , metadata={"help": "Number of gradient accumulation steps."}       )
	_A       =   field(
	    default=__snake_case   , metadata={"help": "Use gradient checkpointing to reduce memory footprint."}       )
	_A       =   field(default=50000   , metadata={"help": "Maximum number of training steps."}       )
	_A       =   field(
	    default=-1   , metadata={"help": "Maximum number of evaluation steps. If -1 the full dataset is evaluated."}       )
	_A       =   field(default=1024   , metadata={"help": "Sequence lengths used for training."}       )
	_A       =   field(default=1   , metadata={"help": "Training seed."}       )
	_A       =   field(
	    default=1024   , metadata={"help": "Interval to save checkpoints. Measured as number of forward passes not training steps."}   , )
	_A       =   field(
	    default=__snake_case   , metadata={"help": "States path if the training should continue from a checkpoint folder."}       )
	_A       =   field(default=__snake_case   , metadata={"help": "If True the data is pretokenized."}       )




@dataclass
class   __lowercase						:
	_A       =   field(
	    default="codeparrot/codeparrot"   , metadata={"help": "Model name or path of model to be evaluated."}       )
	_A       =   field(
	    default="codeparrot/codeparrot-clean-valid"   , metadata={"help": "Name or path of validation dataset."}       )
	_A       =   field(default=2   , metadata={"help": "Batch size used for evaluation."}       )
	_A       =   field(
	    default=-1   , metadata={"help": "Maximum number of evaluation steps. If -1 the full dataset is evaluated."}       )
	_A       =   field(default=1024   , metadata={"help": "Length of sequences to be evaluated."}       )
	_A       =   field(default=1   , metadata={"help": "Random seed used for evaluation."}       )




@dataclass
class   __lowercase						:
	_A       =   field(
	    default="codeparrot/codeparrot"   , metadata={"help": "Model name or path of model to be evaluated."}       )
	_A       =   field(default=__snake_case   , metadata={"help": "Number of workers used for code evaluation."}       )
	_A       =   field(
	    default=__snake_case   , metadata={"help": "The number of human-eval tasks to run. If not included all tasks are evaluated."}   , )
	_A       =   field(
	    default=__snake_case   , metadata={"help": "Sample from the language model's output distribution."}       )
	_A       =   field(default=0.2   , metadata={"help": "Sampling temperature used for generation."}       )
	_A       =   field(default=256   , metadata={"help": "Maximum number of newly generated tokens."}       )
	_A       =   field(default=0   , metadata={"help": "Top-k parameter used for generation."}       )
	_A       =   field(default=0.95   , metadata={"help": "Top-p parameter used for nucleus sampling."}       )
	_A       =   field(default=10   , metadata={"help": "Number of generations to run in parallel."}       )
	_A       =   field(
	    default=200   , metadata={"help": "Number of completions to generate for each sample."}       )
	_A       =   field(default=1   , metadata={"help": "Random seed used for evaluation."}       )
	_A       =   field(
	    default="eval_results.json"   , metadata={"help": "Random seed used for evaluation."}       )
	_A       =   field(
	    default="0"   , metadata={"help": "Allow `code_eval` to execute Python code on machine"}       )
	_A       =   field(
	    default=-1   , metadata={
	        "help": (
	            "Determine which device to run the `text-generation` Pipeline on. -1 is CPU and any zero or positive"
	            " number corresponds to which GPU device id to run on."
	        )
	    }   , )




@dataclass
class   __lowercase						:
	_A       =   field(
	    default=__snake_case   , metadata={
	        "help": "The number of CPU cores to use for parallel preprocessing. Default uses the maximum available."
	    }   , )
	_A       =   field(
	    default="transformersbook/codeparrot"   , metadata={"help": "Folder or name of dataset to process."}       )
	_A       =   field(
	    default="codeparrot-clean"   , metadata={"help": "Folder to save processed processed dataset."}       )
	_A       =   field(
	    default=100000   , metadata={"help": "Number of files to save per JSON output file."}       )
	_A       =   field(default="content"   , metadata={"help": "Column containing text data to process."}       )
	_A       =   field(
	    default=1000   , metadata={"help": "Maximum line length in file, otherwise file is filtered."}       )
	_A       =   field(
	    default=100   , metadata={"help": "Maximum mean line length in file, otherwise file is filtered."}       )
	_A       =   field(
	    default=0.25   , metadata={"help": "Maximum fraction of non-alphanumeric characters, otherwise file is filtered."}       )
	_A       =   field(
	    default=1.5   , metadata={"help": "Minimum character token ratio for the file, otherwise file is filtered."}       )
	_A       =   field(
	    default=0.7   , metadata={"help": "Probability for filtering config, test and uncommon files."}       )
	_A       =   field(
	    default="codeparrot/codeparrot"   , metadata={"help": "Name or path to the tokenizer."}   , )
	_A       =   field(
	    default=__snake_case   , metadata={"help": "If True, near-duplicate samples are removed."}       )
	_A       =   field(
	    default=0.85   , metadata={"help": "Jaccard threshold for near-duplicate samples."}       )




@dataclass
class   __lowercase						:
	_A       =   field(
	    default="gpt2"   , metadata={"help": "Base tokenizer to build new tokenizer from."}       )
	_A       =   field(
	    default="transformersbook/codeparrot-train"   , metadata={"help": "Dataset to train tokenizer on."}       )
	_A       =   field(default="content"   , metadata={"help": "Column containing text data to process."}       )
	_A       =   field(default=200000   , metadata={"help": "Number of examples to train tokenizer on."}       )
	_A       =   field(
	    default=32768   , metadata={"help": "Number of examples to train the tokenizer on."}       )
	_A       =   field(default="codeparrot"   , metadata={"help": "Name of new tokenizer."}       )
	_A       =   field(default=__snake_case   , metadata={"help": "Push saved tokenizer to the hub."}       )




@dataclass
class   __lowercase						:
	_A       =   field(
	    default="codeparrot/codeparrot"   , metadata={"help": "Name or path to the tokenizer."}       )
	_A       =   field(
	    default="codeparrot/codeparrot-clean-train"   , metadata={"help": "Name or path to the dataset to pretokenize."}       )
	_A       =   field(
	    default="tokenized-codeparrot-train"   , metadata={"help": "Repo name of the pretokenized data."}       )
	_A       =   field(default=__snake_case   , metadata={"help": "Number of workers used for code evaluation."}       )




@dataclass
class   __lowercase						:
	_A       =   field(
	    default="gpt2-large"   , metadata={"help": "Configuration to use for model initialization."}       )
	_A       =   field(
	    default="codeparrot/codeparrot"   , metadata={"help": "Tokenizer attached to model."}       )
	_A       =   field(default="codeparrot"   , metadata={"help": "Name of the created model."}       )
	_A       =   field(default=__snake_case   , metadata={"help": "Push saved tokenizer to the hub."}       )




