How can I reshape the (1006,19) result of keras regressor predictions into a (1006,1) numpy array?
I'm trying to create a stock prediction model in botch PyTorch and Keras. I have already followed some tutorials online and modified to fit my data and it works fine.
Now I'm translating that code into a compatible Keras model. I've already created the model and did the predictions but the problem is that the regressor.predict() function from Keras returns a (1006,19) numpy array whereas when I do predictions = model(x_test) it returns a (1006,1) which is what I need for my following work so I can plot the results.
Here's my Keras code so far:
from keras.models import Sequential
from keras.layers import LSTM, Dense, Dropout

lookback = 20
x_train_keras, y_train_keras, x_test_keras, y_test_keras = split_data(price, lookback)
print('x_train.shape = ',x_train_keras.shape)  # x_train.shape =  (1006, 19, 1)
print('y_train.shape = ',y_train_keras.shape)  # y_train.shape =  (1006, 1)
print('x_test.shape = ',x_test_keras.shape)  # x_test.shape =  (252, 19, 1)
print('y_test.shape = ',y_test_keras.shape)  # y_test.shape =  (252, 1)

regression = Sequential()
regression.add(LSTM(units=50, return_sequences=True, kernel_initializer='glorot_uniform', input_shape=(x_train_keras.shape[1],1)))
regression.add(Dropout(0.2))

regression.add(LSTM(units=50,kernel_initializer='glorot_uniform',return_sequences=True))
regression.add(Dropout(0.2))

regression.add(LSTM(units=50,kernel_initializer='glorot_uniform',return_sequences=True))
regression.add(Dropout(0.2))

regression.add(LSTM(units=50,kernel_initializer='glorot_uniform',return_sequences=True))
regression.add(Dropout(0.2))

regression.add(Dense(units=1))

regression.compile(optimizer='adam', loss='mean_squared_error')

from keras.callbacks import History

history = History()
history = regression.fit(x_train_keras, y_train_keras, batch_size=30, epochs=100, callbacks=[history])

train_predict_keras = regression.predict(x_train_keras)
train_predict_keras = train_predict_keras.reshape((train_predict_keras.shape[0], train_predict_keras.shape[1]))

predict = pd.DataFrame(scaler.inverse_transform(train_predict_keras))
original = pd.DataFrame(scaler.inverse_transform(y_train_keras))

fig = plt.figure()
fig.subplots_adjust(hspace=0.2, wspace=0.2)

plt.subplot(1,2,1)
ax = sns.lineplot(x=original.index, y=original[0], label='Data', color='royalblue')
ax = sns.lineplot(x=predict.index, y=predict[0], label='Training Prediction', color='tomato')
ax.set_title('Stock Price', size=14, fontweight='bold')
ax.set_xlabel(&quot;Days&quot;, size = 14)
ax.set_ylabel(&quot;Cost (USD)&quot;, size = 14)
ax.set_xticklabels('', size=10)

plt.subplot(1,2,2)
ax = sns.lineplot(data=history.history.get('loss'), color='royalblue')
ax.set_xlabel(&quot;Epoch&quot;, size = 14)
ax.set_ylabel(&quot;Loss&quot;, size = 14)
ax.set_title(&quot;Training Loss&quot;, size = 14, fontweight='bold')
fig.set_figheight(6)
fig.set_figwidth(16)

# Make predictions
test_predict_keras = regression.predict(x_test_keras)

# Invert predictions
train_predict_keras = scaler.inverse_transform(train_predict_keras)
y_train_keras = scaler.inverse_transform(y_train_keras)
test_predict_keras = scaler.inverse_transform(test_predict_keras.reshape((test_predict_keras.shape[0], test_predict_keras.shape[1])))
y_test = scaler.inverse_transform(y_test_keras)

# Calculate root MSE
trainScore = math.sqrt(mean_squared_error(y_train[:,0], y_train_pred[:,0]))
print(f'Train score {trainScore:.2f} RMSE')
testScore = math.sqrt(mean_squared_error(y_test[:,0], y_test_pred[:,0]))
print(f'Test score {testScore:.2f} RMSE')

# shift train predictions for plotting
trainPredictPlot_keras = np.empty_like(price)
trainPredictPlot_keras[:, :] = np.nan
trainPredictPlot_keras[lookback:len(train_predict_keras)+lookback, :] = train_predict_keras

# shift test predictions for plotting
testPredictPlot_keras = np.empty_like(price)
testPredictPlot_keras[:, :] = np.nan
testPredictPlot_keras[len(train_predict_keras)+lookback-1:len(price)-1, :] = test_predict_keras

original = scaler.inverse_transform(price['Close'].values.reshape(-1,1))

predictions_keras = np.append(trainPredictPlot_keras, testPredictPlot_keras, axis=1)
predictions_keras = np.append(predictions_keras, original, axis=1)
result_keras = pd.DataFrame(predictions_keras)

The error occurs in the trainPredictPlot_keras[lookback:len(train_predict_keras)+lookback, :] = train_predict_keras line saying could not broadcast input array from shape (1006,19) into shape (1006,1)

Set the return_sequences to False for the last LSTM layer. You need to do as follows:
....
....
regression.add(LSTM(units=50,kernel_initializer='glorot_uniform',
                     return_sequences=False))
regression.add(Dropout(0.2))
regression.add(Dense(units=1))
regression.compile(optimizer='adam', loss='mean_squared_error')


Check doc:

return_sequences: Boolean. Whether to return the last output. in the output sequence, or the full sequence. Default: False.

