


'''simple docstring'''

import uuid
from typing import Any, Dict, List, Optional, Union

from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
     import tensorflow as tf

if is_torch_available():
     import torch


a_    :							Optional[int]         =					logging.get_logger(__name__)





class 	snake_case       :





  """simple docstring"""




  def __init__(       self     ,    UpperCamelCase = None     ,    UpperCamelCase = None     ,    UpperCamelCase=None     ,    UpperCamelCase=None	):







   """simple docstring"""



   if not conversation_id:
    lowerCamelCase_						=    uuid.uuida()
   if past_user_inputs is None:
    lowerCamelCase_						=    []
   if generated_responses is None:
    lowerCamelCase_						=    []

   lowerCamelCase_						=    conversation_id
   lowerCamelCase_						=    past_user_inputs
   lowerCamelCase_						=    generated_responses
   lowerCamelCase_						=    text






  def __eq__(       self     ,    UpperCamelCase	):







   """simple docstring"""



   if not isinstance(lowercase__     ,    lowercase__	):
    return False
   if self.uuid == other.uuid:
    return True
   return (
       self.new_user_input == other.new_user_input
       and self.past_user_inputs == other.past_user_inputs
       and self.generated_responses == other.generated_responses
   )






  def     snake_case						(       self     ,    UpperCamelCase     ,    UpperCamelCase = False	):







   """simple docstring"""



   if self.new_user_input:
    if overwrite:
     logger.warning(
         f'''User input added while unprocessed input was existing: "{self.new_user_input}" was overwritten '''
         f'''with: "{text}".'''	)
     lowerCamelCase_						=    text
    else:
     logger.warning(
         f'''User input added while unprocessed input was existing: "{self.new_user_input}" new input '''
         f'''ignored: "{text}". Set `overwrite` to True to overwrite unprocessed user input'''	)
   else:
    lowerCamelCase_						=    text






  def     snake_case						(       self	):







   """simple docstring"""



   if self.new_user_input:
    self.past_user_inputs.append(self.new_user_input	)
   lowerCamelCase_						=    None






  def     snake_case						(       self     ,    UpperCamelCase	):







   """simple docstring"""
   self.generated_responses.append(lowercase__	)






  def     snake_case						(       self	):







   """simple docstring"""



   for user_input, generated_response in zip(self.past_user_inputs     ,    self.generated_responses	):
    yield True, user_input
    yield False, generated_response
   if self.new_user_input:
    yield True, self.new_user_input







  def __repr__(       self	):







   """simple docstring"""
   lowerCamelCase_						=    f'''Conversation id: {self.uuid} \n'''
   for is_user, text in self.iter_texts():
    lowerCamelCase_						=    "user" if is_user else "bot"
    output += f'''{name} >> {text} \n'''
   return output





@add_end_docstrings(
    lowercase     ,		r"\n        min_length_for_response (`int`, *optional*, defaults to 32):\n            The minimum length (in number of tokens) for a response.\n        minimum_tokens (`int`, *optional*, defaults to 10):\n            The minimum length of tokens to leave for a response.\n    "     ,		)
class 	snake_case       (       lowercase     ):





  """simple docstring"""




  def __init__(       self     ,    *UpperCamelCase     ,    **UpperCamelCase	):







   """simple docstring"""
   super().__init__(*lowercase__     ,    **lowercase__	)
   if self.tokenizer.pad_token_id is None:
    lowerCamelCase_						=    self.tokenizer.eos_token






  def     snake_case						(       self     ,    UpperCamelCase=None     ,    UpperCamelCase=None     ,    UpperCamelCase=None     ,    **UpperCamelCase	):







   """simple docstring"""
   lowerCamelCase_						=    {}
   lowerCamelCase_						=    {}
   lowerCamelCase_						=    {}

   if min_length_for_response is not None:
    lowerCamelCase_						=    min_length_for_response
   if minimum_tokens is not None:
    lowerCamelCase_						=    minimum_tokens

   if "max_length" in generate_kwargs:
    lowerCamelCase_						=    generate_kwargs["max_length"]
    # self.max_length = generate_kwargs.get("max_length", self.model.config.max_length)
   if clean_up_tokenization_spaces is not None:
    lowerCamelCase_						=    clean_up_tokenization_spaces

   if generate_kwargs:
    forward_params.update(lowercase__	)
   return preprocess_params, forward_params, postprocess_params






  def __call__(       self     ,    UpperCamelCase     ,    UpperCamelCase=0     ,    **UpperCamelCase	):







   """simple docstring"""
   lowerCamelCase_						=    super().__call__(lowercase__     ,    num_workers=lowercase__     ,    **lowercase__	)
   if isinstance(lowercase__     ,    lowercase__	) and len(lowercase__	) == 1:
    return outputs[0]
   return outputs






  def     snake_case						(       self     ,    UpperCamelCase     ,    UpperCamelCase=32	):







   """simple docstring"""



   if not isinstance(lowercase__     ,    lowercase__	):
    raise ValueError("ConversationalPipeline, expects Conversation as inputs"	)
   if conversation.new_user_input is None:
    raise ValueError(
        f'''Conversation with UUID {type(conversation.uuid	)} does not contain new user input to process. '''
        "Add user inputs with the conversation\'s `add_user_input` method"	)
   if hasattr(self.tokenizer     ,    "_build_conversation_input_ids"	):
    lowerCamelCase_						=    self.tokenizer._build_conversation_input_ids(lowercase__	)
   else:
    # If the tokenizer cannot handle conversations, we default to only the old version
    lowerCamelCase_						=    self._legacy_parse_and_tokenize(lowercase__	)

   if self.framework == "pt":
    lowerCamelCase_						=    torch.LongTensor([input_ids]	)
   elif self.framework == "tf":
    lowerCamelCase_						=    tf.constant([input_ids]	)
   return {"input_ids": input_ids, "conversation": conversation}






  def     snake_case						(       self     ,    UpperCamelCase     ,    UpperCamelCase=10     ,    **UpperCamelCase	):







   """simple docstring"""
   lowerCamelCase_						=    generate_kwargs.get("max_length"     ,    self.model.config.max_length	)

   lowerCamelCase_						=    model_inputs["input_ids"].shape[1]
   if max_length - minimum_tokens < n:
    logger.warning(f'''Conversation input is to long ({n}), trimming it to ({max_length} - {minimum_tokens})'''	)
    lowerCamelCase_						=    max_length - minimum_tokens
    lowerCamelCase_						=    model_inputs["input_ids"][:, -trim:]
    if "attention_mask" in model_inputs:
     lowerCamelCase_						=    model_inputs["attention_mask"][:, -trim:]
   lowerCamelCase_						=    model_inputs.pop("conversation"	)
   lowerCamelCase_						=    max_length
   lowerCamelCase_						=    self.model.generate(**lowercase__     ,    **lowercase__	)
   if self.model.config.is_encoder_decoder:
    lowerCamelCase_						=    1
   else:
    lowerCamelCase_						=    n
   return {"output_ids": output_ids[:, start_position:], "conversation": conversation}






  def     snake_case						(       self     ,    UpperCamelCase     ,    UpperCamelCase=True	):







   """simple docstring"""
   lowerCamelCase_						=    model_outputs["output_ids"]
   lowerCamelCase_						=    self.tokenizer.decode(
       output_ids[0]     ,    skip_special_tokens=lowercase__     ,    clean_up_tokenization_spaces=lowercase__     ,    )
   lowerCamelCase_						=    model_outputs["conversation"]
   conversation.mark_processed()
   conversation.append_response(lowercase__	)
   return conversation







  def     snake_case						(       self     ,    UpperCamelCase	):







   """simple docstring"""
   lowerCamelCase_						=    self.tokenizer.eos_token_id
   lowerCamelCase_						=    []
   for is_user, text in conversation.iter_texts():
    if eos_token_id is not None:
     input_ids.extend(self.tokenizer.encode(lowercase__     ,    add_special_tokens=lowercase__	) + [eos_token_id]	)
    else:
     input_ids.extend(self.tokenizer.encode(lowercase__     ,    add_special_tokens=lowercase__	)	)

   if len(lowercase__	) > self.tokenizer.model_max_length:
    lowerCamelCase_						=    input_ids[-self.tokenizer.model_max_length :]
   return input_ids






import inspect
import re
from hashlib import shaaaa
from typing import Dict, List

from .arrow import arrow
from .audiofolder import audiofolder
from .csv import csv
from .imagefolder import imagefolder
from .json import json
from .pandas import pandas
from .parquet import parquet
from .sql import sql  # noqa F401
from .text import text





def      A__					(			__A  :						List[str]			)					->str:
      __A    =[]
      for line in lines:
            __A    =re.sub(r'''#.*'''     ,				''''''     ,				__A			)  # remove comments
            if line:
                  filtered_lines.append(__A			)
      __A    ='''\n'''.join(__A			)

      # Make a hash from all this code
      __A    =full_str.encode('''utf-8'''			)
      return shaaaa(__A			).hexdigest()


# get importable module names and hash for caching
_lowerCamelCase      :					Dict					    =      {
    '''csv''': (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
    '''json''': (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
    '''pandas''': (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
    '''parquet''': (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
    '''arrow''': (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
    '''text''': (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
    '''imagefolder''': (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
    '''audiofolder''': (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
}

# Used to infer the module to use based on the data files extensions
_lowerCamelCase      :					int					    =      {
    '''.csv''': ('''csv''', {}),
    '''.tsv''': ('''csv''', {'''sep''': '''\t'''}),
    '''.json''': ('''json''', {}),
    '''.jsonl''': ('''json''', {}),
    '''.parquet''': ('''parquet''', {}),
    '''.arrow''': ('''arrow''', {}),
    '''.txt''': ('''text''', {}),
}
_EXTENSION_TO_MODULE.update({ext: ('''imagefolder''', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('''imagefolder''', {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext: ('''audiofolder''', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ('''audiofolder''', {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_lowerCamelCase      :					List[str]					    =      {'''imagefolder''', '''audiofolder'''}

# Used to filter data files based on extensions given a module name
_lowerCamelCase      :					Dict[str, List[str]]					    =      {}
for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
   _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)

_MODULE_TO_EXTENSIONS["imagefolder"].append('''.zip''')


_MODULE_TO_EXTENSIONS["audiofolder"].append('''.zip''')


