




import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer



class 			a     (   tf.keras.layers.Layer			):

     '''simple docstring'''





     def __init__(	self   :			Optional[Any]					,    __snake_case   :			List					,    __snake_case   :			bool					,    __snake_case   :			int = None					,    __snake_case   :			int = None					,    __snake_case   :			int = None					,    __snake_case   :			str = "longest"					,    __snake_case   :			bool = True					,    __snake_case   :			int = 5_12					,    __snake_case   :			int = None					,    __snake_case   :			bool = True					,    __snake_case   :			bool = True					,    __snake_case   :			bool = True					,    **__snake_case   :			Dict					,    ):
         super().__init__()
         if use_fast_bert_tokenizer:
             UpperCAmelCase_							 =     FastBertTokenizer(
                 __snake_case					,    token_out_type=tf.intaa					,    lower_case_nfd_strip_accents=__snake_case					,    **__snake_case				)
         else:
             UpperCAmelCase_							 =     tf.lookup.StaticVocabularyTable(
                 tf.lookup.KeyValueTensorInitializer(
                     keys=__snake_case					,    key_dtype=tf.string					,    values=tf.range(tf.size(__snake_case					,    out_type=tf.intaa				)					,    dtype=tf.intaa				)					,    value_dtype=tf.intaa					,    )					,    num_oov_buckets=1					,    )
             UpperCAmelCase_							 =     BertTokenizerLayer(
                 __snake_case					,    token_out_type=tf.intaa					,    lower_case=__snake_case					,    **__snake_case				)

         UpperCAmelCase_							 =     vocab_list
         UpperCAmelCase_							 =     do_lower_case
         UpperCAmelCase_							 =     cls_token_id or vocab_list.index('''[CLS]'''				)
         UpperCAmelCase_							 =     sep_token_id or vocab_list.index('''[SEP]'''				)
         UpperCAmelCase_							 =     pad_token_id or vocab_list.index('''[PAD]'''				)
         UpperCAmelCase_							 =     ShrinkLongestTrimmer(max_length - 3					,    axis=1				)  # Allow room for special tokens
         UpperCAmelCase_							 =     max_length
         UpperCAmelCase_							 =     padding
         UpperCAmelCase_							 =     truncation
         UpperCAmelCase_							 =     pad_to_multiple_of
         UpperCAmelCase_							 =     return_token_type_ids
         UpperCAmelCase_							 =     return_attention_mask
     @classmethod
     def   lowerCamelCase_   (	cls   :			str					,    __snake_case   :			"PreTrainedTokenizerBase"					,    **__snake_case   :			Optional[int]				):  # noqa: F821
         UpperCAmelCase_							 =     kwargs.pop('''do_lower_case'''					,    __snake_case				)
         UpperCAmelCase_							 =     tokenizer.do_lower_case if do_lower_case is None else do_lower_case
         UpperCAmelCase_							 =     kwargs.pop('''cls_token_id'''					,    __snake_case				)
         UpperCAmelCase_							 =     tokenizer.cls_token_id if cls_token_id is None else cls_token_id
         UpperCAmelCase_							 =     kwargs.pop('''sep_token_id'''					,    __snake_case				)
         UpperCAmelCase_							 =     tokenizer.sep_token_id if sep_token_id is None else sep_token_id
         UpperCAmelCase_							 =     kwargs.pop('''pad_token_id'''					,    __snake_case				)
         UpperCAmelCase_							 =     tokenizer.pad_token_id if pad_token_id is None else pad_token_id

         UpperCAmelCase_							 =     tokenizer.get_vocab()
         UpperCAmelCase_							 =     sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]					,    key=lambda __snake_case				: x[1]				)
         UpperCAmelCase_							 =     [entry[0] for entry in vocab]
         return cls(
             vocab_list=__snake_case					,    do_lower_case=__snake_case					,    cls_token_id=__snake_case					,    sep_token_id=__snake_case					,    pad_token_id=__snake_case					,    **__snake_case					,    )
     @classmethod
     def   lowerCamelCase_   (	cls   :			Optional[Any]					,    __snake_case   :			Union[str, os.PathLike]					,    *__snake_case   :			List[str]					,    **__snake_case   :			str				):
         try:
             UpperCAmelCase_							 =     BertTokenizer.from_pretrained(__snake_case					,    *__snake_case					,    **__snake_case				)
         except:  # noqa: E722
             from .tokenization_bert_fast import BertTokenizerFast

             UpperCAmelCase_							 =     BertTokenizerFast.from_pretrained(__snake_case					,    *__snake_case					,    **__snake_case				)
         return cls.from_tokenizer(__snake_case					,    **__snake_case				)
     def   lowerCamelCase_   (	self   :			Optional[Any]					,    __snake_case   :			str				):
         if self.do_lower_case:
             UpperCAmelCase_							 =     case_fold_utfa(__snake_case				)
         UpperCAmelCase_							 =     self.tf_tokenizer.tokenize(__snake_case				)
         return tokens.merge_dims(1					,    -1				)
     def   lowerCamelCase_   (	self   :			str					,    __snake_case   :			Optional[Any]					,    __snake_case   :			Tuple=None					,    __snake_case   :			Optional[int]=None					,    __snake_case   :			Any=None					,    __snake_case   :			Dict=None					,    __snake_case   :			List[str]=None					,    __snake_case   :			Dict=None					,    __snake_case   :			Dict=None					,    ):
         if padding is None:
             UpperCAmelCase_							 =     self.padding
         if padding not in ("longest", "max_length"):
             raise ValueError('''Padding must be either \'longest\' or \'max_length\'!'''				)
         if max_length is not None and text_pair is not None:
             # Because we have to instantiate a Trimmer to do it properly
             raise ValueError('''max_length cannot be overridden at call time when truncating paired texts!'''				)
         if max_length is None:
             UpperCAmelCase_							 =     self.max_length
         if truncation is None:
             UpperCAmelCase_							 =     self.truncation
         if pad_to_multiple_of is None:
             UpperCAmelCase_							 =     self.pad_to_multiple_of
         if return_token_type_ids is None:
             UpperCAmelCase_							 =     self.return_token_type_ids
         if return_attention_mask is None:
             UpperCAmelCase_							 =     self.return_attention_mask
         if not isinstance(__snake_case					,    tf.Tensor				):
             UpperCAmelCase_							 =     tf.convert_to_tensor(__snake_case				)
         if text_pair is not None and not isinstance(__snake_case					,    tf.Tensor				):
             UpperCAmelCase_							 =     tf.convert_to_tensor(__snake_case				)
         if text_pair is not None:
             if text.shape.rank > 1:
                 raise ValueError('''text argument should not be multidimensional when a text pair is supplied!'''				)
             if text_pair.shape.rank > 1:
                 raise ValueError('''text_pair should not be multidimensional!'''				)
         if text.shape.rank == 2:
             UpperCAmelCase_ ,							UpperCAmelCase_							 =     text[:, 0], text[:, 1]
         UpperCAmelCase_							 =     self.unpaired_tokenize(__snake_case				)
         if text_pair is None:  # Unpaired text
             if truncation:
                 UpperCAmelCase_							 =     text[:, : max_length - 2]  # Allow room for special tokens
             UpperCAmelCase_ ,							UpperCAmelCase_							 =     combine_segments(
                 (text,)					,    start_of_sequence_id=self.cls_token_id					,    end_of_segment_id=self.sep_token_id				)
         else:  # Paired text
             UpperCAmelCase_							 =     self.unpaired_tokenize(__snake_case				)
             if truncation:
                 UpperCAmelCase_ ,							UpperCAmelCase_							 =     self.paired_trimmer.trim([text, text_pair]				)
             UpperCAmelCase_ ,							UpperCAmelCase_							 =     combine_segments(
                 (text, text_pair)					,    start_of_sequence_id=self.cls_token_id					,    end_of_segment_id=self.sep_token_id				)
         if padding == "longest":
             UpperCAmelCase_							 =     input_ids.bounding_shape(axis=1				)
             if pad_to_multiple_of is not None:
                 # No ceiling division in tensorflow, so we negate floordiv instead
                 UpperCAmelCase_							 =     pad_to_multiple_of * (-tf.math.floordiv(-pad_length					,    __snake_case				))
         else:
             UpperCAmelCase_							 =     max_length

         UpperCAmelCase_ ,							UpperCAmelCase_							 =     pad_model_inputs(__snake_case					,    max_seq_length=__snake_case					,    pad_value=self.pad_token_id				)
         UpperCAmelCase_							 =     {'''input_ids''': input_ids}
         if return_attention_mask:
             UpperCAmelCase_							 =     attention_mask
         if return_token_type_ids:
             UpperCAmelCase_ ,							UpperCAmelCase_							 =     pad_model_inputs(
                 __snake_case					,    max_seq_length=__snake_case					,    pad_value=self.pad_token_id				)
             UpperCAmelCase_							 =     token_type_ids
         return output




     def   lowerCamelCase_   (	self   :			List[Any]				):
         return {
             "vocab_list": self.vocab_list,
             "do_lower_case": self.do_lower_case,
             "cls_token_id": self.cls_token_id,
             "sep_token_id": self.sep_token_id,
             "pad_token_id": self.pad_token_id,
         }











from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...file_utils import TensorType, is_torch_available
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


_lowerCamelCase			 =						logging.get_logger(__name__)

_lowerCamelCase			 =						{
    'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/config.json',
    # See all Blenderbot models at https://huggingface.co/models?filter=blenderbot
}



class 			a     (   _A			):

     '''simple docstring'''





     lowerCAmelCase				:       List[Any]   		=  'blenderbot'
     lowerCAmelCase				:       int   		=  ['past_key_values']
     lowerCAmelCase				:       int   		=  {'num_attention_heads': 'encoder_attention_heads', 'hidden_size': 'd_model'}
     def __init__(	self   :			int					,    __snake_case   :			Optional[Any]=80_08					,    __snake_case   :			List[str]=1_28					,    __snake_case   :			List[str]=2					,    __snake_case   :			Tuple=1_02_40					,    __snake_case   :			List[Any]=32					,    __snake_case   :			int=24					,    __snake_case   :			str=1_02_40					,    __snake_case   :			Optional[int]=32					,    __snake_case   :			Dict=0.0					,    __snake_case   :			int=0.0					,    __snake_case   :			str=True					,    __snake_case   :			Dict=True					,    __snake_case   :			Dict="gelu"					,    __snake_case   :			List[str]=25_60					,    __snake_case   :			List[Any]=0.1					,    __snake_case   :			str=0.0					,    __snake_case   :			Dict=0.0					,    __snake_case   :			Tuple=0.02					,    __snake_case   :			Any=1					,    __snake_case   :			int=False					,    __snake_case   :			Any=0					,    __snake_case   :			Optional[Any]=1					,    __snake_case   :			Optional[int]=2					,    __snake_case   :			List[Any]=3					,    __snake_case   :			int=2					,    **__snake_case   :			Tuple					,    ):
         UpperCAmelCase_							 =     vocab_size
         UpperCAmelCase_							 =     max_position_embeddings
         UpperCAmelCase_							 =     d_model
         UpperCAmelCase_							 =     encoder_ffn_dim
         UpperCAmelCase_							 =     encoder_layers
         UpperCAmelCase_							 =     encoder_attention_heads
         UpperCAmelCase_							 =     decoder_ffn_dim
         UpperCAmelCase_							 =     decoder_layers
         UpperCAmelCase_							 =     decoder_attention_heads
         UpperCAmelCase_							 =     dropout
         UpperCAmelCase_							 =     attention_dropout
         UpperCAmelCase_							 =     activation_dropout
         UpperCAmelCase_							 =     activation_function
         UpperCAmelCase_							 =     init_std
         UpperCAmelCase_							 =     encoder_layerdrop
         UpperCAmelCase_							 =     decoder_layerdrop
         UpperCAmelCase_							 =     use_cache
         UpperCAmelCase_							 =     encoder_layers
         UpperCAmelCase_							 =     scale_embedding  # scale factor will be sqrt(d_model) if True

         super().__init__(
             pad_token_id=__snake_case					,    bos_token_id=__snake_case					,    eos_token_id=__snake_case					,    is_encoder_decoder=__snake_case					,    decoder_start_token_id=__snake_case					,    encoder_no_repeat_ngram_size=__snake_case					,    forced_eos_token_id=__snake_case					,    **__snake_case					,    )



class 			a     (   _A			):

     '''simple docstring'''





     @property
     def   lowerCamelCase_   (	self   :			Any				):
         if self.task in ["default", "seq2seq-lm"]:
             UpperCAmelCase_							 =     OrderedDict(
                 [
                     ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                     ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                 ]				)
             if self.use_past:
                 UpperCAmelCase_							 =     {0: '''batch'''}
                 UpperCAmelCase_							 =     {0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
             else:
                 UpperCAmelCase_							 =     {0: '''batch''', 1: '''decoder_sequence'''}
                 UpperCAmelCase_							 =     {0: '''batch''', 1: '''decoder_sequence'''}
             if self.use_past:
                 self.fill_with_past_key_values_(__snake_case					,    direction='''inputs'''				)
         elif self.task == "causal-lm":
             UpperCAmelCase_							 =     OrderedDict(
                 [
                     ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                     ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                 ]				)
             if self.use_past:
                 UpperCAmelCase_ ,							UpperCAmelCase_							 =     self.num_layers
                 for i in range(__snake_case				):
                     UpperCAmelCase_							 =     {0: '''batch''', 2: '''past_sequence + sequence'''}
                     UpperCAmelCase_							 =     {0: '''batch''', 2: '''past_sequence + sequence'''}
         else:
             UpperCAmelCase_							 =     OrderedDict(
                 [
                     ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                     ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                     ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
                     ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
                 ]				)

         return common_inputs
     @property
     # Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
     def   lowerCamelCase_   (	self   :			Optional[Any]				):
         if self.task in ["default", "seq2seq-lm"]:
             UpperCAmelCase_							 =     super().outputs
         else:
             UpperCAmelCase_							 =     super(__snake_case					,    self				).outputs
             if self.use_past:
                 UpperCAmelCase_ ,							UpperCAmelCase_							 =     self.num_layers
                 for i in range(__snake_case				):
                     UpperCAmelCase_							 =     {0: '''batch''', 2: '''past_sequence + sequence'''}
                     UpperCAmelCase_							 =     {0: '''batch''', 2: '''past_sequence + sequence'''}
         return common_outputs
     def   lowerCamelCase_   (	self   :			Optional[Any]					,    __snake_case   :			PreTrainedTokenizer					,    __snake_case   :			int = -1					,    __snake_case   :			int = -1					,    __snake_case   :			bool = False					,    __snake_case   :			Optional[TensorType] = None					,    ):
         UpperCAmelCase_							 =     self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
             __snake_case					,    __snake_case					,    __snake_case					,    __snake_case					,    __snake_case				)
         # Generate decoder inputs
         UpperCAmelCase_							 =     seq_length if not self.use_past else 1
         UpperCAmelCase_							 =     self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
             __snake_case					,    __snake_case					,    __snake_case					,    __snake_case					,    __snake_case				)
         UpperCAmelCase_							 =     {F'decoder_{name}': tensor for name, tensor in decoder_inputs.items()}
         UpperCAmelCase_							 =     dict(**__snake_case					,    **__snake_case				)

         if self.use_past:
             if not is_torch_available():
                 raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''				)
             else:
                 import torch
             UpperCAmelCase_ ,							UpperCAmelCase_							 =     common_inputs['''input_ids'''].shape
             UpperCAmelCase_							 =     common_inputs['''decoder_input_ids'''].shape[1]
             UpperCAmelCase_ ,							UpperCAmelCase_							 =     self.num_attention_heads
             UpperCAmelCase_							 =     (
                 batch,
                 num_encoder_attention_heads,
                 encoder_seq_length,
                 self._config.hidden_size // num_encoder_attention_heads,
             )
             UpperCAmelCase_							 =     decoder_seq_length
             UpperCAmelCase_							 =     (
                 batch,
                 num_decoder_attention_heads,
                 decoder_past_length,
                 self._config.hidden_size // num_decoder_attention_heads,
             )
             UpperCAmelCase_							 =     torch.cat(
                 [common_inputs['''decoder_attention_mask'''], torch.ones(__snake_case					,    __snake_case				)]					,    dim=1				)
             UpperCAmelCase_							 =     []
             UpperCAmelCase_ ,							UpperCAmelCase_							 =     self.num_layers

             for _ in range(__snake_case				):
                 common_inputs["past_key_values"].append(
                     (
                         torch.zeros(__snake_case				),
                         torch.zeros(__snake_case				),
                         torch.zeros(__snake_case				),
                         torch.zeros(__snake_case				),
                     )				)
         return common_inputs
     def   lowerCamelCase_   (	self   :			Optional[Any]					,    __snake_case   :			PreTrainedTokenizer					,    __snake_case   :			int = -1					,    __snake_case   :			int = -1					,    __snake_case   :			bool = False					,    __snake_case   :			Optional[TensorType] = None					,    ):
         UpperCAmelCase_							 =     self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
             __snake_case					,    __snake_case					,    __snake_case					,    __snake_case					,    __snake_case				)

         if self.use_past:
             if not is_torch_available():
                 raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''				)
             else:
                 import torch
             UpperCAmelCase_ ,							UpperCAmelCase_							 =     common_inputs['''input_ids'''].shape
             UpperCAmelCase_							 =     seqlen
             UpperCAmelCase_ ,							UpperCAmelCase_							 =     self.num_layers
             UpperCAmelCase_ ,							UpperCAmelCase_							 =     self.num_attention_heads
             UpperCAmelCase_							 =     (
                 batch,
                 num_encoder_attention_heads,
                 past_key_values_length,
                 self._config.hidden_size // num_encoder_attention_heads,
             )
             UpperCAmelCase_							 =     common_inputs['''attention_mask'''].dtype
             UpperCAmelCase_							 =     torch.cat(
                 [common_inputs['''attention_mask'''], torch.ones(__snake_case					,    __snake_case					,    dtype=__snake_case				)]					,    dim=1				)
             UpperCAmelCase_							 =     [
                 (torch.zeros(__snake_case				), torch.zeros(__snake_case				)) for _ in range(__snake_case				)
             ]
         return common_inputs
     def   lowerCamelCase_   (	self   :			Any					,    __snake_case   :			PreTrainedTokenizer					,    __snake_case   :			int = -1					,    __snake_case   :			int = -1					,    __snake_case   :			bool = False					,    __snake_case   :			Optional[TensorType] = None					,    ):
         # Copied from OnnxConfig.generate_dummy_inputs
         # Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
         # If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
         UpperCAmelCase_							 =     compute_effective_axis_dimension(
             __snake_case					,    fixed_dimension=OnnxConfig.default_fixed_batch					,    num_token_to_add=0				)

         # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
         UpperCAmelCase_							 =     tokenizer.num_special_tokens_to_add(__snake_case				)
         UpperCAmelCase_							 =     compute_effective_axis_dimension(
             __snake_case					,    fixed_dimension=OnnxConfig.default_fixed_sequence					,    num_token_to_add=__snake_case				)

         # Generate dummy inputs according to compute batch and sequence
         UpperCAmelCase_							 =     [''' '''.join([tokenizer.unk_token]				) * seq_length] * batch_size
         UpperCAmelCase_							 =     dict(tokenizer(__snake_case					,    return_tensors=__snake_case				)				)
         return common_inputs
     def   lowerCamelCase_   (	self   :			List[Any]					,    __snake_case   :			PreTrainedTokenizer					,    __snake_case   :			int = -1					,    __snake_case   :			int = -1					,    __snake_case   :			bool = False					,    __snake_case   :			Optional[TensorType] = None					,    ):
         if self.task in ["default", "seq2seq-lm"]:
             UpperCAmelCase_							 =     self._generate_dummy_inputs_for_default_and_seqaseq_lm(
                 __snake_case					,    batch_size=__snake_case					,    seq_length=__snake_case					,    is_pair=__snake_case					,    framework=__snake_case				)

         elif self.task == "causal-lm":
             UpperCAmelCase_							 =     self._generate_dummy_inputs_for_causal_lm(
                 __snake_case					,    batch_size=__snake_case					,    seq_length=__snake_case					,    is_pair=__snake_case					,    framework=__snake_case				)
         else:
             UpperCAmelCase_							 =     self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
                 __snake_case					,    batch_size=__snake_case					,    seq_length=__snake_case					,    is_pair=__snake_case					,    framework=__snake_case				)

         return common_inputs
     def   lowerCamelCase_   (	self   :			Dict					,    __snake_case   :			Optional[int]					,    __snake_case   :			int					,    __snake_case   :			str					,    __snake_case   :			Optional[int]				):
         if self.task in ["default", "seq2seq-lm"]:
             UpperCAmelCase_							 =     super()._flatten_past_key_values_(__snake_case					,    __snake_case					,    __snake_case					,    __snake_case				)
         else:
             UpperCAmelCase_							 =     super(__snake_case					,    self				)._flatten_past_key_values_(
                 __snake_case					,    __snake_case					,    __snake_case					,    __snake_case				)




     def   lowerCamelCase_   (	self   :			Dict					,    __snake_case   :			Mapping[str, Mapping[int, str]]					,    __snake_case   :			str				):
         if direction not in ["inputs", "outputs"]:
             raise ValueError(F'direction must either be "inputs" or "outputs", but {direction} was given'				)

         UpperCAmelCase_							 =     '''past_key_values''' if direction == '''inputs''' else '''present'''
         UpperCAmelCase_ ,							UpperCAmelCase_							 =     self.num_layers

         UpperCAmelCase_							 =     '''past_encoder_sequence'''
         UpperCAmelCase_							 =     '''past_decoder_sequence''' if direction == '''inputs''' else '''past_decoder_sequence + sequence'''

         for i in range(__snake_case				):
             UpperCAmelCase_							 =     {0: '''batch''', 2: decoder_sequence}
             UpperCAmelCase_							 =     {0: '''batch''', 2: decoder_sequence}
             UpperCAmelCase_							 =     {0: '''batch''', 2: encoder_sequence}
             UpperCAmelCase_							 =     {0: '''batch''', 2: encoder_sequence}





