

import uuid
from typing import Any, Dict, List, Optional, Union

from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
   import tensorflow as tf

if is_torch_available():
   import torch


_lowerCamelCase      :					Optional[int]					    =      logging.get_logger(__name__)





class        lowerCAmelCase__  :



       '''simple docstring'''







       def __init__(							self ,   lowercase__ = None ,   lowercase__ = None ,   lowercase__=None ,   lowercase__=None     ):







             '''simple docstring'''



             if not conversation_id:
                   __A    =uuid.uuida()
             if past_user_inputs is None:
                   __A    =[]
             if generated_responses is None:
                   __A    =[]

             __A    =conversation_id
             __A    =past_user_inputs
             __A    =generated_responses
             __A    =text

       def __eq__(							self ,   lowercase__     ):







             '''simple docstring'''



             if not isinstance(lowercase__ ,   lowercase__     ):
                   return False
             if self.uuid == other.uuid:
                   return True
             return (
                 self.new_user_input == other.new_user_input
                 and self.past_user_inputs == other.past_user_inputs
                 and self.generated_responses == other.generated_responses
             )

       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__ = False     ):







             '''simple docstring'''



             if self.new_user_input:
                   if overwrite:
                         logger.warning(
                             f'''User input added while unprocessed input was existing: "{self.new_user_input}" was overwritten '''
                             f'''with: "{text}".'''     )
                         __A    =text
                   else:
                         logger.warning(
                             f'''User input added while unprocessed input was existing: "{self.new_user_input}" new input '''
                             f'''ignored: "{text}". Set `overwrite` to True to overwrite unprocessed user input'''     )
             else:
                   __A    =text

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             if self.new_user_input:
                   self.past_user_inputs.append(self.new_user_input     )
             __A    =None

       def     __UpperCamelCase     (							self ,   lowercase__     ):







             '''simple docstring'''



             self.generated_responses.append(lowercase__     )

       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             for user_input, generated_response in zip(self.past_user_inputs ,   self.generated_responses     ):
                   yield True, user_input
                   yield False, generated_response
             if self.new_user_input:
                   yield True, self.new_user_input


       def __repr__(							self     ):







             '''simple docstring'''



             __A    =f'''Conversation id: {self.uuid} \n'''
             for is_user, text in self.iter_texts():
                   __A    ='''user''' if is_user else '''bot'''
                   output += f'''{name} >> {text} \n'''
             return output





@add_end_docstrings(
    __magic_name__				,							r"""
        min_length_for_response (`int`, *optional*, defaults to 32):
            The minimum length (in number of tokens) for a response.
        minimum_tokens (`int`, *optional*, defaults to 10):
            The minimum length of tokens to leave for a response.
    """				,							)
class        lowerCAmelCase__  (					__magic_name__			):



       '''simple docstring'''







       def __init__(							self ,   *lowercase__ ,   **lowercase__     ):







             '''simple docstring'''



             super().__init__(*lowercase__ ,   **lowercase__     )
             if self.tokenizer.pad_token_id is None:
                   __A    =self.tokenizer.eos_token

       def     __UpperCamelCase     (							self ,   lowercase__=None ,   lowercase__=None ,   lowercase__=None ,   **lowercase__     ):







             '''simple docstring'''



             __A    ={}
             __A    ={}
             __A    ={}

             if min_length_for_response is not None:
                   __A    =min_length_for_response
             if minimum_tokens is not None:
                   __A    =minimum_tokens

             if "max_length" in generate_kwargs:
                   __A    =generate_kwargs['''max_length''']
                   # self.max_length = generate_kwargs.get("max_length", self.model.config.max_length)
             if clean_up_tokenization_spaces is not None:
                   __A    =clean_up_tokenization_spaces

             if generate_kwargs:
                   forward_params.update(lowercase__     )
             return preprocess_params, forward_params, postprocess_params

       def __call__(							self ,   lowercase__ ,   lowercase__=0 ,   **lowercase__     ):







             '''simple docstring'''



             __A    =super().__call__(lowercase__ ,   num_workers=lowercase__ ,   **lowercase__     )
             if isinstance(lowercase__ ,   lowercase__     ) and len(lowercase__     ) == 1:
                   return outputs[0]
             return outputs

       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__=3_2     ):







             '''simple docstring'''



             if not isinstance(lowercase__ ,   lowercase__     ):
                   raise ValueError('''ConversationalPipeline, expects Conversation as inputs'''     )
             if conversation.new_user_input is None:
                   raise ValueError(
                       f'''Conversation with UUID {type(conversation.uuid     )} does not contain new user input to process. '''
                       '''Add user inputs with the conversation\'s `add_user_input` method'''     )
             if hasattr(self.tokenizer ,   '''_build_conversation_input_ids'''     ):
                   __A    =self.tokenizer._build_conversation_input_ids(lowercase__     )
             else:
                   # If the tokenizer cannot handle conversations, we default to only the old version
                   __A    =self._legacy_parse_and_tokenize(lowercase__     )

             if self.framework == "pt":
                   __A    =torch.LongTensor([input_ids]     )
             elif self.framework == "tf":
                   __A    =tf.constant([input_ids]     )
             return {"input_ids": input_ids, "conversation": conversation}

       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__=1_0 ,   **lowercase__     ):







             '''simple docstring'''



             __A    =generate_kwargs.get('''max_length''' ,   self.model.config.max_length     )

             __A    =model_inputs['''input_ids'''].shape[1]
             if max_length - minimum_tokens < n:
                   logger.warning(f'''Conversation input is to long ({n}), trimming it to ({max_length} - {minimum_tokens})'''     )
                   __A    =max_length - minimum_tokens
                   __A    =model_inputs['''input_ids'''][:, -trim:]
                   if "attention_mask" in model_inputs:
                         __A    =model_inputs['''attention_mask'''][:, -trim:]
             __A    =model_inputs.pop('''conversation'''     )
             __A    =max_length
             __A    =self.model.generate(**lowercase__ ,   **lowercase__     )
             if self.model.config.is_encoder_decoder:
                   __A    =1
             else:
                   __A    =n
             return {"output_ids": output_ids[:, start_position:], "conversation": conversation}

       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__=True     ):







             '''simple docstring'''



             __A    =model_outputs['''output_ids''']
             __A    =self.tokenizer.decode(
                 output_ids[0] ,   skip_special_tokens=lowercase__ ,   clean_up_tokenization_spaces=lowercase__ ,   )
             __A    =model_outputs['''conversation''']
             conversation.mark_processed()
             conversation.append_response(lowercase__     )
             return conversation


       def     __UpperCamelCase     (							self ,   lowercase__     ):







             '''simple docstring'''



             __A    =self.tokenizer.eos_token_id
             __A    =[]
             for is_user, text in conversation.iter_texts():
                   if eos_token_id is not None:
                         input_ids.extend(self.tokenizer.encode(lowercase__ ,   add_special_tokens=lowercase__     ) + [eos_token_id]     )
                   else:
                         input_ids.extend(self.tokenizer.encode(lowercase__ ,   add_special_tokens=lowercase__     )     )

             if len(lowercase__     ) > self.tokenizer.model_max_length:
                   __A    =input_ids[-self.tokenizer.model_max_length :]
             return input_ids





from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase      :					Any					    =      logging.get_logger(__name__)


_lowerCamelCase      :					Tuple					    =      {
    '''nielsr/canine-s''': 2048,
}

# Unicode defines 1,114,112 total “codepoints”
_lowerCamelCase      :					List[str]					    =      111_4112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
_lowerCamelCase      :					List[str]					    =      0

_lowerCamelCase      :					Any					    =      0xe_0_0_0
_lowerCamelCase      :					Union[str, Any]					    =      0xe_0_0_1
_lowerCamelCase      :					Any					    =      0xe_0_0_2
_lowerCamelCase      :					List[str]					    =      0xe_0_0_3
_lowerCamelCase      :					Any					    =      0xe_0_0_4

# Maps special codepoints to human-readable names.
_lowerCamelCase      :					Dict[int, str]					    =      {
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
_lowerCamelCase      :					Dict[str, int]					    =      {name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}





class        lowerCAmelCase__  (					__magic_name__			):



       '''simple docstring'''







       lowercase_						=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

       def __init__(							self ,   lowercase__=chr(lowercase__     ) ,   lowercase__=chr(lowercase__     ) ,   lowercase__=chr(lowercase__     ) ,   lowercase__=chr(lowercase__     ) ,   lowercase__=chr(lowercase__     ) ,   lowercase__=chr(lowercase__     ) ,   lowercase__=False ,   lowercase__=2_0_4_8 ,   **lowercase__ ,   ):







             '''simple docstring'''



             __A    =AddedToken(lowercase__ ,   lstrip=lowercase__ ,   rstrip=lowercase__     ) if isinstance(lowercase__ ,   lowercase__     ) else bos_token
             __A    =AddedToken(lowercase__ ,   lstrip=lowercase__ ,   rstrip=lowercase__     ) if isinstance(lowercase__ ,   lowercase__     ) else eos_token
             __A    =AddedToken(lowercase__ ,   lstrip=lowercase__ ,   rstrip=lowercase__     ) if isinstance(lowercase__ ,   lowercase__     ) else sep_token
             __A    =AddedToken(lowercase__ ,   lstrip=lowercase__ ,   rstrip=lowercase__     ) if isinstance(lowercase__ ,   lowercase__     ) else cls_token
             __A    =AddedToken(lowercase__ ,   lstrip=lowercase__ ,   rstrip=lowercase__     ) if isinstance(lowercase__ ,   lowercase__     ) else pad_token

             # Mask token behave like a normal word, i.e. include the space before it
             __A    =AddedToken(lowercase__ ,   lstrip=lowercase__ ,   rstrip=lowercase__     ) if isinstance(lowercase__ ,   lowercase__     ) else mask_token

             super().__init__(
                 bos_token=lowercase__ ,   eos_token=lowercase__ ,   sep_token=lowercase__ ,   cls_token=lowercase__ ,   pad_token=lowercase__ ,   mask_token=lowercase__ ,   add_prefix_space=lowercase__ ,   model_max_length=lowercase__ ,   **lowercase__ ,   )

             # Creates a mapping for looking up the IDs of special symbols.
             __A    ={}
             for codepoint, name in SPECIAL_CODEPOINTS.items():
                   __A    =codepoint

             # Creates a mapping for looking up the string forms of special symbol IDs.
             __A    ={
                 codepoint: name for name, codepoint in self._special_codepoints.items()
             }

             __A    =UNICODE_VOCAB_SIZE
             __A    =len(self._special_codepoints     )

       @property
       def     __UpperCamelCase     (							self     ):







             '''simple docstring'''



             return self._unicode_vocab_size

       def     __UpperCamelCase     (							self ,   lowercase__     ):







             '''simple docstring'''



             return list(lowercase__     )

       def     __UpperCamelCase     (							self ,   lowercase__     ):







             '''simple docstring'''



             try:
                   return ord(lowercase__     )
             except TypeError:
                   raise ValueError(f'''invalid token: \'{token}\''''     )

       def     __UpperCamelCase     (							self ,   lowercase__     ):







             '''simple docstring'''



             try:
                   if index in SPECIAL_CODEPOINTS:
                         return SPECIAL_CODEPOINTS[index]
                   return chr(lowercase__     )
             except TypeError:
                   raise ValueError(f'''invalid id: {index}'''     )

       def     __UpperCamelCase     (							self ,   lowercase__     ):







             '''simple docstring'''



             return "".join(lowercase__     )

       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__ = None     ):







             '''simple docstring'''



             __A    =[self.sep_token_id]
             __A    =[self.cls_token_id]

             __A    =cls + token_ids_a + sep
             if token_ids_a is not None:
                   result += token_ids_a + sep
             return result

       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__ = None ,   lowercase__ = False     ):







             '''simple docstring'''



             if already_has_special_tokens:
                   return super().get_special_tokens_mask(
                       token_ids_a=lowercase__ ,   token_ids_a=lowercase__ ,   already_has_special_tokens=lowercase__     )

             __A    =[1] + ([0] * len(lowercase__     )) + [1]
             if token_ids_a is not None:
                   result += ([0] * len(lowercase__     )) + [1]
             return result

       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__ = None     ):







             '''simple docstring'''



             __A    =[self.sep_token_id]
             __A    =[self.cls_token_id]

             __A    =len(cls + token_ids_a + sep     ) * [0]
             if token_ids_a is not None:
                   result += len(token_ids_a + sep     ) * [1]
             return result


       def     __UpperCamelCase     (							self ,   lowercase__ ,   lowercase__ = None     ):







             '''simple docstring'''



             return ()


