RuntimeError: Expected hidden[0] size (2, 1, 100), got (1, 1, 100)
I put together a LSTM model and it works. But only as long as I set num_layers = 1.
If I set it for example to 2 it gives my a long error message that tells me:
RuntimeError: Expected hidden[0] size (2, 1, 100), got (1, 1, 100)
I am pretty new at Python and deep learning in general, so I could need some advice how to fix my problem.
Code:
import torch
import torch.nn as nn
import numpy as np
import dataset
import time

amount_hidden_layers = 2
amount_neurons_hidden_layers = 100

input_layer_size = 12

output_layer_size = 48
small = True
hours = 20
learning_rate = 1/1000 #0,001
early_stoping = False

train_obs_dataset = dataset.return_observation_dataset(hours=hours, split=&quot;train&quot;, small=small)
val_obs_dataset = dataset.return_observation_dataset(hours=hours, split=&quot;val&quot;, small=small)
test_obs_dataset = dataset.return_observation_dataset(hours=hours, split=&quot;test&quot;, small=small)

class LSTM(nn.Module):
    def __init__(self, input_layer_size, hidden_layer_size, output_layer_size,
                 num_layers):
        super().__init__()
        self.hidden_layer_size = hidden_layer_size
        self.input_layer_size = input_layer_size
        self.num_layers = num_layers
        self.output_layer_size = output_layer_size

        # lstm_layers:
        self.lstm = nn.LSTM(input_size=self.input_layer_size, hidden_size=self.hidden_layer_size, num_layers=num_layers)

        self.hidden_cell = (torch.zeros(self.num_layers, 1, self.hidden_layer_size),
                            torch.zeros(self.num_layers, 1, self.hidden_layer_size))
        # output_layer:
        self.linear = nn.Linear(self.hidden_layer_size, self.output_layer_size)

    def forward(self, input_seq):
        lstm_out, self.hidden_cell = self.lstm(input_seq.view(len(input_seq), 1, -1), self.hidden_cell)
        predictions = self.linear(lstm_out.view(len(input_seq), -1))
        return predictions[-1]

model = LSTM(input_layer_size=input_layer_size, hidden_layer_size=amount_neurons_hidden_layers,
             output_layer_size=output_layer_size, num_layers=amount_hidden_layers)
loss_function = nn.MSELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

def training(dataset=train_obs_dataset, epochs=5):
    start_time = time.time()
    val_test = float(100000000)                                                     # for early stoping, saves the last evaluation value
    val_worse_counter = 0                                                           # for early stoping, couts how often tests with validation data get worse
    for i in range(epochs):                                                         # every epoch
        for datapoint in range(dataset[0].__len__()):                               # every datapoint
            optimizer.zero_grad()
            model.hidden_cell = (torch.zeros(1, 1, model.hidden_layer_size),
                                    torch.zeros(1, 1, model.hidden_layer_size))

            y_pred = model(torch.FloatTensor(dataset[0][datapoint]))                # model needs a tensor of shape [sequence][features]
            # squeeze: does not change data, prevents error,
            single_loss = loss_function(y_pred, torch.FloatTensor(dataset[1][datapoint]).squeeze(-1))
           
            single_loss.backward()
            optimizer.step()
            if datapoint%100 == 1:
                print(f' Time: {time.time() - start_time: 10.1f} sec,   Epoch: {i:4} Datapoint: {datapoint:3} Loss:  {single_loss.item():10.8f} ')
                #print(y_pred[0])
            &quot;early stoping: if tests with the validation data does not get better for 100 steps, the model stops&quot;
            if (early_stoping == True):
                if datapoint%20 == 1:
                    val = test(val_obs_dataset)
                    if (val[0] &lt;= val_test):
                        val_test = val[0]
                        val_worse_counter = 0
                    if(val[0] &gt; val_test):
                        val_worse_counter = val_worse_counter + 1
                    if(val[0] &gt;= 100):
                        break
                    print(&quot;Val: &quot;, val[0], val[1][0], &quot;val_test: &quot;, val_test, &quot; val_worse_counter: &quot;, val_worse_counter)

&quot;&quot;&quot;
returns a tupel with 1. the average loss and 2. an array with the losses for average losses seperated for all 48 hours
&quot;&quot;&quot;
def test(dataset=val_obs_dataset):
    with torch.no_grad():
        losses = []                 #sperated per hour
        loss = 0
        for i in range(48):
            losses.append(0)
        for datapoint in range(dataset.__len__()):
            val_pred = model(torch.FloatTensor(dataset[0][datapoint]))
            &quot;seperates the loss values per hour&quot;
            for i in range(48):
                losses[i] = losses[i] + float(loss_function(val_pred[i], torch.FloatTensor(dataset[1][datapoint]).squeeze(-1)[i]))
            loss = loss + float(loss_function(val_pred, torch.FloatTensor(dataset[1][datapoint]).squeeze(-1)))
        for i in range(48):                                         # calculates the average
            losses[i] = losses[i]/dataset.__len__()
            loss = loss/dataset.__len__()
    return (loss, losses)

print(model)

training(epochs=200)

Thanks in advance for every helpfull comment

That is because of this line in your training loop:
model.hidden_cell = (torch.zeros(1, 1, model.hidden_layer_size),
                     torch.zeros(1, 1, model.hidden_layer_size))

Even though you correctly defined hidden_cell in your model, here you hard coded num_layers to be 1 and replaced the one you did correctly.
To fix it, you can change it to this;
model.hidden_cell = (torch.zeros(model.num_layers, 1, model.hidden_layer_size),
                     torch.zeros(model.num_layers, 1, model.hidden_layer_size))

or even remove it totally as you are basically repeating what you've already done. And I don't suppose hidden_layer_size will change mid-training.
Maybe when you batch your data, then leaving it here would make more sense, but it is clear that your batch_size = 1.
