How to Find the Average of the Input Vectors
Does this work for you? After the Bi-Directional LSTM layer, I sum the tensor over the batch axis and then divide by the batch size.

from tensorflow.keras import layers, Sequential
from tensorflow.keras.models import Model
from tensorflow.keras import backend as K
from tensorflow.keras.utils import plot_model

inputs = layers.Input(batch_shape=(4, 500), name="Input")
embed = layers.Embedding(input_dim=500, output_dim=30, name="Embedding")(inputs)
encoder = layers.Bidirectional(
                  layers.LSTM(units=20,return_sequences=True,dropout=0.3,name="LSTM")
                  ,name="Bi-LSTM")

hidden_states = encoder(embed)
total = layers.Lambda(lambda x: K.sum(x, axis=0)/K.cast(K.shape(x)[0],"float32"))(hidden_states)

model = Model(inputs, total)
model.summary()