





import uuid
from typing import Any, Dict, List, Optional, Union

from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
							import tensorflow as tf

if is_torch_available():
							import torch


_UpperCamelCase							=      logging.get_logger(__name__)
class 		SCREAMING_SNAKE_CASE_							:
				"""simple docstring"""




				def __init__(     self  :Dict						,    __lowercase  :str = None						,    __lowercase  :uuid.UUID = None						,    __lowercase  :List[Any]=None						,    __lowercase  :Any=None			):
											if not conversation_id:
																		__lowerCamelCase      :      Dict         =uuid.uuida()
											if past_user_inputs is None:
																		__lowerCamelCase      :      Optional[int]         =[]
											if generated_responses is None:
																		__lowerCamelCase      :      List[Any]         =[]

											__lowerCamelCase      :      uuid.UUID         =conversation_id
											__lowerCamelCase      :      List[str]         =past_user_inputs
											__lowerCamelCase      :      List[str]         =generated_responses
											__lowerCamelCase      :      Optional[str]         =text




				def __eq__(     self  :Optional[Any]						,    __lowercase  :Tuple			):
											if not isinstance(__lowercase						,    __lowercase			):
																		return False
											if self.uuid == other.uuid:
																		return True
											return (
											    self.new_user_input == other.new_user_input
											    and self.past_user_inputs == other.past_user_inputs
											    and self.generated_responses == other.generated_responses
											)




				def 					__lowercase  (     self  :Any						,    __lowercase  :str						,    __lowercase  :bool = False			):
											if self.new_user_input:
																		if overwrite:
																									logger.warning(
																									    f'User input added while unprocessed input was existing: "{self.new_user_input}" was overwritten '
																									    f'with: "{text}".'			)
																									__lowerCamelCase      :      List[Any]         =text
																		else:
																									logger.warning(
																									    f'User input added while unprocessed input was existing: "{self.new_user_input}" new input '
																									    f'ignored: "{text}". Set `overwrite` to True to overwrite unprocessed user input'			)
											else:
																		__lowerCamelCase      :      Optional[int]         =text




				def 					__lowercase  (     self  :List[str]			):
											if self.new_user_input:
																		self.past_user_inputs.append(self.new_user_input			)
											__lowerCamelCase      :      Dict         =None




				def 					__lowercase  (     self  :List[Any]						,    __lowercase  :str			):
											self.generated_responses.append(__lowercase			)




				def 					__lowercase  (     self  :int			):
											for user_input, generated_response in zip(self.past_user_inputs						,    self.generated_responses			):
																		yield True, user_input
																		yield False, generated_response
											if self.new_user_input:
																		yield True, self.new_user_input




				def __repr__(     self  :int			):
											__lowerCamelCase      :      Union[str, Any]         =f'Conversation id: {self.uuid} \n'
											for is_user, text in self.iter_texts():
																		__lowerCamelCase      :      Optional[Any]         ='''user''' if is_user else '''bot'''
																		output += f'{name} >> {text} \n'
											return output

@add_end_docstrings(
    snake_case__       ,  r"""
        min_length_for_response (`int`, *optional*, defaults to 32):
            The minimum length (in number of tokens) for a response.
        minimum_tokens (`int`, *optional*, defaults to 10):
            The minimum length of tokens to leave for a response.
    """       ,  )
class 		SCREAMING_SNAKE_CASE_							(    snake_case__   ):
				"""simple docstring"""




				def __init__(     self  :Tuple						,    *__lowercase  :int						,    **__lowercase  :Any			):
											super().__init__(*__lowercase						,    **__lowercase			)
											if self.tokenizer.pad_token_id is None:
																		__lowerCamelCase      :      int         =self.tokenizer.eos_token




				def 					__lowercase  (     self  :str						,    __lowercase  :int=None						,    __lowercase  :Dict=None						,    __lowercase  :Tuple=None						,    **__lowercase  :str			):
											__lowerCamelCase      :      Union[str, Any]         ={}
											__lowerCamelCase      :      str         ={}
											__lowerCamelCase      :      int         ={}

											if min_length_for_response is not None:
																		__lowerCamelCase      :      Optional[Any]         =min_length_for_response
											if minimum_tokens is not None:
																		__lowerCamelCase      :      Optional[int]         =minimum_tokens

											if "max_length" in generate_kwargs:
																		__lowerCamelCase      :      str         =generate_kwargs['''max_length''']
																		# self.max_length = generate_kwargs.get("max_length", self.model.config.max_length)
											if clean_up_tokenization_spaces is not None:
																		__lowerCamelCase      :      Optional[int]         =clean_up_tokenization_spaces

											if generate_kwargs:
																		forward_params.update(__lowercase			)
											return preprocess_params, forward_params, postprocess_params




				def __call__(     self  :Any						,    __lowercase  :Union[Conversation, List[Conversation]]						,    __lowercase  :List[str]=0						,    **__lowercase  :str			):
											__lowerCamelCase      :      List[str]         =super().__call__(__lowercase						,    num_workers=__lowercase						,    **__lowercase			)
											if isinstance(__lowercase						,    __lowercase			) and len(__lowercase			) == 1:
																		return outputs[0]
											return outputs




				def 					__lowercase  (     self  :List[Any]						,    __lowercase  :Conversation						,    __lowercase  :int=32			):
											if not isinstance(__lowercase						,    __lowercase			):
																		raise ValueError('''ConversationalPipeline, expects Conversation as inputs'''			)
											if conversation.new_user_input is None:
																		raise ValueError(
																		    f'Conversation with UUID {type(conversation.uuid			)} does not contain new user input to process. '
																		    '''Add user inputs with the conversation\'s `add_user_input` method'''			)
											if hasattr(self.tokenizer						,    '''_build_conversation_input_ids'''			):
																		__lowerCamelCase      :      Optional[Any]         =self.tokenizer._build_conversation_input_ids(__lowercase			)
											else:
																		# If the tokenizer cannot handle conversations, we default to only the old version
																		__lowerCamelCase      :      Optional[int]         =self._legacy_parse_and_tokenize(__lowercase			)

											if self.framework == "pt":
																		__lowerCamelCase      :      str         =torch.LongTensor([input_ids]			)
											elif self.framework == "tf":
																		__lowerCamelCase      :      Tuple         =tf.constant([input_ids]			)
											return {"input_ids": input_ids, "conversation": conversation}




				def 					__lowercase  (     self  :Optional[Any]						,    __lowercase  :Any						,    __lowercase  :List[str]=10						,    **__lowercase  :Union[str, Any]			):
											__lowerCamelCase      :      List[str]         =generate_kwargs.get('''max_length'''						,    self.model.config.max_length			)

											__lowerCamelCase      :      Union[str, Any]         =model_inputs['''input_ids'''].shape[1]
											if max_length - minimum_tokens < n:
																		logger.warning(f'Conversation input is to long ({n}), trimming it to ({max_length} - {minimum_tokens})'			)
																		__lowerCamelCase      :      Optional[int]         =max_length - minimum_tokens
																		__lowerCamelCase      :      Any         =model_inputs['''input_ids'''][:, -trim:]
																		if "attention_mask" in model_inputs:
																									__lowerCamelCase      :      Optional[Any]         =model_inputs['''attention_mask'''][:, -trim:]
											__lowerCamelCase      :      Optional[Any]         =model_inputs.pop('''conversation'''			)
											__lowerCamelCase      :      Union[str, Any]         =max_length
											__lowerCamelCase      :      Union[str, Any]         =self.model.generate(**__lowercase						,    **__lowercase			)
											if self.model.config.is_encoder_decoder:
																		__lowerCamelCase      :      Optional[Any]         =1
											else:
																		__lowerCamelCase      :      str         =n
											return {"output_ids": output_ids[:, start_position:], "conversation": conversation}




				def 					__lowercase  (     self  :List[Any]						,    __lowercase  :int						,    __lowercase  :List[str]=True			):
											__lowerCamelCase      :      Any         =model_outputs['''output_ids''']
											__lowerCamelCase      :      List[str]         =self.tokenizer.decode(
											    output_ids[0]						,    skip_special_tokens=__lowercase						,    clean_up_tokenization_spaces=__lowercase						,    )
											__lowerCamelCase      :      Optional[Any]         =model_outputs['''conversation''']
											conversation.mark_processed()
											conversation.append_response(__lowercase			)
											return conversation




				def 					__lowercase  (     self  :str						,    __lowercase  :Conversation			):
											__lowerCamelCase      :      str         =self.tokenizer.eos_token_id
											__lowerCamelCase      :      List[str]         =[]
											for is_user, text in conversation.iter_texts():
																		if eos_token_id is not None:
																									input_ids.extend(self.tokenizer.encode(__lowercase						,    add_special_tokens=__lowercase			) + [eos_token_id]			)
																		else:
																									input_ids.extend(self.tokenizer.encode(__lowercase						,    add_special_tokens=__lowercase			)			)

											if len(__lowercase			) > self.tokenizer.model_max_length:
																		__lowerCamelCase      :      int         =input_ids[-self.tokenizer.model_max_length :]
											return input_ids











"""simple docstring"""





from pickle import UnpicklingError

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict

from ..utils import logging


_UpperCamelCase							=      logging.get_logger(__name__)



def 				lowerCAmelCase_			(      SCREAMING_SNAKE_CASE       :							str    ,						SCREAMING_SNAKE_CASE       :							str   ):


							'''simple docstring'''







							try:
														with open(SCREAMING_SNAKE_CASE    ,						'''rb'''   ) as flax_state_f:
																					__lowerCamelCase      :      int         =from_bytes(SCREAMING_SNAKE_CASE    ,						flax_state_f.read()   )
							except UnpicklingError as e:
														try:
																					with open(SCREAMING_SNAKE_CASE   ) as f:
																												if f.read().startswith('''version'''   ):
																																			raise OSError(
																																			    '''You seem to have cloned a repository without having git-lfs installed. Please'''
																																			    ''' install git-lfs and run `git lfs install` followed by `git lfs pull` in the'''
																																			    ''' folder you cloned.'''   )
																												else:
																																			raise ValueError from e
														except (UnicodeDecodeError, ValueError):
																					raise EnvironmentError(F'Unable to convert {model_file} to Flax deserializable object. '   )

							return load_flax_weights_in_pytorch_model(SCREAMING_SNAKE_CASE    ,						SCREAMING_SNAKE_CASE   )



def 				lowerCAmelCase_			(      SCREAMING_SNAKE_CASE       :							int    ,						SCREAMING_SNAKE_CASE       :							Union[str, Any]   ):


							'''simple docstring'''







							try:
														import torch  # noqa: F401
							except ImportError:
														logger.error(
														    '''Loading Flax weights in PyTorch requires both PyTorch and Flax to be installed. Please see'''
														    ''' https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation'''
														    ''' instructions.'''   )
														raise

							# check if we have bf16 weights
							__lowerCamelCase      :      str         =flatten_dict(jax.tree_util.tree_map(lambda SCREAMING_SNAKE_CASE   : x.dtype == jnp.bfloataa    ,						SCREAMING_SNAKE_CASE   )   ).values()
							if any(SCREAMING_SNAKE_CASE   ):
														# convert all weights to fp32 if they are bf16 since torch.from_numpy can-not handle bf16

														# and bf16 is not fully supported in PT yet.
														logger.warning(
														    '''Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` '''
														    '''before loading those in PyTorch model.'''   )
														__lowerCamelCase      :      List[str]         =jax.tree_util.tree_map(
														    lambda SCREAMING_SNAKE_CASE   : params.astype(np.floataa   ) if params.dtype == jnp.bfloataa else params    ,						SCREAMING_SNAKE_CASE   )

							__lowerCamelCase      :      int         =''''''

							__lowerCamelCase      :      List[str]         =flatten_dict(SCREAMING_SNAKE_CASE    ,						sep='''.'''   )
							__lowerCamelCase      :      List[str]         =pt_model.state_dict()

							# keep track of unexpected & missing keys
							__lowerCamelCase      :      Union[str, Any]         =[]
							__lowerCamelCase      :      int         =set(pt_model_dict.keys()   )

							for flax_key_tuple, flax_tensor in flax_state_dict.items():
														__lowerCamelCase      :      Tuple         =flax_key_tuple.split('''.'''   )

														if flax_key_tuple_array[-1] == "kernel" and flax_tensor.ndim == 4:
																					__lowerCamelCase      :      Optional[int]         =flax_key_tuple_array[:-1] + ['''weight''']
																					__lowerCamelCase      :      Union[str, Any]         =jnp.transpose(SCREAMING_SNAKE_CASE    ,						(3, 2, 0, 1)   )
														elif flax_key_tuple_array[-1] == "kernel":
																					__lowerCamelCase      :      Optional[Any]         =flax_key_tuple_array[:-1] + ['''weight''']
																					__lowerCamelCase      :      Dict         =flax_tensor.T
														elif flax_key_tuple_array[-1] == "scale":
																					__lowerCamelCase      :      str         =flax_key_tuple_array[:-1] + ['''weight''']

														if "time_embedding" not in flax_key_tuple_array:
																					for i, flax_key_tuple_string in enumerate(SCREAMING_SNAKE_CASE   ):
																												__lowerCamelCase      :      Union[str, Any]         =(
																												    flax_key_tuple_string.replace('''_0'''    ,						'''.0'''   )
																												    .replace('''_1'''    ,						'''.1'''   )
																												    .replace('''_2'''    ,						'''.2'''   )
																												    .replace('''_3'''    ,						'''.3'''   )
																												    .replace('''_4'''    ,						'''.4'''   )
																												    .replace('''_5'''    ,						'''.5'''   )
																												    .replace('''_6'''    ,						'''.6'''   )
																												    .replace('''_7'''    ,						'''.7'''   )
																												    .replace('''_8'''    ,						'''.8'''   )
																												    .replace('''_9'''    ,						'''.9'''   )
																												)

														__lowerCamelCase      :      Optional[int]         ='''.'''.join(SCREAMING_SNAKE_CASE   )

														if flax_key in pt_model_dict:
																					if flax_tensor.shape != pt_model_dict[flax_key].shape:
																												raise ValueError(
																												    F'Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected '
																												    F'to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}.'   )
																					else:
																												# add weight to pytorch dict
																												__lowerCamelCase      :      Tuple         =np.asarray(SCREAMING_SNAKE_CASE   ) if not isinstance(SCREAMING_SNAKE_CASE    ,						np.ndarray   ) else flax_tensor
																												__lowerCamelCase      :      Any         =torch.from_numpy(SCREAMING_SNAKE_CASE   )
																												# remove from missing keys
																												missing_keys.remove(SCREAMING_SNAKE_CASE   )
														else:
																					# weight is not expected by PyTorch model
																					unexpected_keys.append(SCREAMING_SNAKE_CASE   )

							pt_model.load_state_dict(SCREAMING_SNAKE_CASE   )

							# re-transform missing_keys to list
							__lowerCamelCase      :      Optional[Any]         =list(SCREAMING_SNAKE_CASE   )

							if len(SCREAMING_SNAKE_CASE   ) > 0:
														logger.warning(
														    '''Some weights of the Flax model were not used when initializing the PyTorch model'''
														    F' {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing'
														    F' {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture'
														    ''' (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This'''
														    F' IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect'
														    ''' to be exactly identical (e.g. initializing a BertForSequenceClassification model from a'''
														    ''' FlaxBertForSequenceClassification model).'''   )
							if len(SCREAMING_SNAKE_CASE   ) > 0:
														logger.warning(
														    F'Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly'
														    F' initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to'
														    ''' use it for predictions and inference.'''   )

							return pt_model




