title: Built-in RNN model was constructed with a wrong shape in Virtual Donkey
body: <p>I use <a href="https://docs.donkeycar.com/guide/simulator/" rel="nofollow noreferrer">Donkey Simulator</a> to train and test a built-in recurrent neural network model with my own data, but the shape of the model and the shape of the input do not match. They don't have the same rank.</p>
  <p>This is one of the images, which I captured while collecting the data:</p>
  <p><a href="https://i.stack.imgur.com/2TSaz.jpg" rel="nofollow noreferrer"><img src="https://i.stack.imgur.com/2TSaz.jpg" alt="enter image description here" /></a></p>
  <p>There is the built-in model, which I wanted to use:</p>
  <pre><code>class KerasLSTM(KerasPilot):
      def __init__(self,
                   interpreter: Interpreter = KerasInterpreter(),
                   input_shape: Tuple[int, ...] = (120, 160, 3),
                   seq_length=3,
                   num_outputs=2):
          self.num_outputs = num_outputs
          self.seq_length = seq_length
          super().__init__(interpreter, input_shape)
          self.img_seq = deque()
          self.optimizer = &quot;rmsprop&quot;
  
      def seq_size(self) -&gt; int:
          return self.seq_length
  
      def create_model(self):
          return rnn_lstm(seq_length=self.seq_length,
                          num_outputs=self.num_outputs,
                          input_shape=self.input_shape)
  
      def compile(self):
          self.interpreter.compile(optimizer=self.optimizer, loss='mse')
  
      def x_transform(self, records: Union[TubRecord, List[TubRecord]]) -&gt; XY:
          &quot;&quot;&quot; Return x from record, here x = stacked images &quot;&quot;&quot;
          assert isinstance(records, list), 'List[TubRecord] expected'
          assert len(records) == self.seq_length, \
              f&quot;Record list of length {self.seq_length} required but &quot; \
              f&quot;{len(records)} was passed&quot;
          img_arrays = [rec.image(cached=True) for rec in records]
          return np.array(img_arrays)
  
      def x_translate(self, x: XY) -&gt; Dict[str, Union[float, np.ndarray]]:
          &quot;&quot;&quot; Translates x into dictionary where all model input layer's names
              must be matched by dictionary keys. &quot;&quot;&quot;
          img_arr = x
          return {'img_in': img_arr}
  
      def x_transform_and_process(
              self,
              records: Union[TubRecord, List[TubRecord]],
              img_processor: Callable[[np.ndarray], np.ndarray]) -&gt; XY:
          &quot;&quot;&quot; Transforms the record sequence into x for training the model to
              x, y. &quot;&quot;&quot;
          img_seq = self.x_transform(records)
          assert isinstance(img_seq, np.ndarray)
          # apply augmentation / normalisation on sequence of images
          x_process = [img_processor(img) for img in img_seq]
          return np.array(x_process)
  
      def y_transform(self, records: Union[TubRecord, List[TubRecord]]) -&gt; XY:
          &quot;&quot;&quot; Only return the last entry of angle/throttle&quot;&quot;&quot;
          assert isinstance(records, list), 'List[TubRecord] expected'
          angle = records[-1].underlying['user/angle']
          throttle = records[-1].underlying['user/throttle']
          return angle, throttle
  
      def y_translate(self, y: XY) -&gt; Dict[str, Union[float, List[float]]]:
          assert isinstance(y, tuple), 'Expected tuple'
          return {'model_outputs': list(y)}
  
      def run(self, img_arr, other_arr=None):
          if img_arr.shape[2] == 3 and self.input_shape[2] == 1:
              img_arr = dk.utils.rgb2gray(img_arr)
  
          while len(self.img_seq) &lt; self.seq_length:
              self.img_seq.append(img_arr)
  
          self.img_seq.popleft()
          self.img_seq.append(img_arr)
          new_shape = (self.seq_length, *self.input_shape)
          img_arr = np.array(self.img_seq).reshape(new_shape)
          img_arr_norm = normalize_image(img_arr)
          return self.inference(img_arr_norm, other_arr)
  
      def interpreter_to_output(self, interpreter_out) \
              -&gt; Tuple[Union[float, np.ndarray], ...]:
          steering = interpreter_out[0]
          throttle = interpreter_out[1]
          return steering, throttle
  
      def output_shapes(self):
          # need to cut off None from [None, 120, 160, 3] tensor shape
          img_shape = self.get_input_shapes()[0][1:]
          # the keys need to match the models input/output layers
          shapes = ({'img_in': tf.TensorShape(img_shape)},
                    {'model_outputs': tf.TensorShape([self.num_outputs])})
          return shapes
  
      def __str__(self) -&gt; str:
          &quot;&quot;&quot; For printing model initialisation &quot;&quot;&quot;
          return f'{super().__str__()}-L:{self.seq_length}'
  
  
  def rnn_lstm(seq_length=3, num_outputs=2, input_shape=(120, 160, 3)):
          # add sequence length dimensions as keras time-distributed expects shape
          # of (num_samples, seq_length, input_shape)
          img_seq_shape = (seq_length,) + input_shape
          img_in = Input(shape=img_seq_shape, name='img_in')
          drop_out = 0.3
      
          x = img_in
          x = TD(Convolution2D(24, (5, 5), strides=(2, 2), activation='relu'))(x)
          x = TD(Dropout(drop_out))(x)
          x = TD(Convolution2D(32, (5, 5), strides=(2, 2), activation='relu'))(x)
          x = TD(Dropout(drop_out))(x)
          x = TD(Convolution2D(32, (3, 3), strides=(2, 2), activation='relu'))(x)
          x = TD(Dropout(drop_out))(x)
          x = TD(Convolution2D(32, (3, 3), strides=(1, 1), activation='relu'))(x)
          x = TD(Dropout(drop_out))(x)
          x = TD(MaxPooling2D(pool_size=(2, 2)))(x)
          x = TD(Flatten(name='flattened'))(x)
          x = TD(Dense(100, activation='relu'))(x)
          x = TD(Dropout(drop_out))(x)
      
          x = LSTM(128, return_sequences=True, name=&quot;LSTM_seq&quot;)(x)
          x = Dropout(.1)(x)
          x = LSTM(128, return_sequences=False, name=&quot;LSTM_fin&quot;)(x)
          x = Dropout(.1)(x)
          x = Dense(128, activation='relu')(x)
          x = Dropout(.1)(x)
          x = Dense(64, activation='relu')(x)
          x = Dense(10, activation='relu')(x)
          out = Dense(num_outputs, activation='linear', name='model_outputs')(x)
          model = Model(inputs=[img_in], outputs=[out], name='lstm')
          return model
  </code></pre>
  <p>And this is the error message, which I get:</p>
  <pre><code>WARNING:tensorflow:Model was constructed with shape (None, 3, 120, 160, 3) for input 
  Tensor(&quot;img_in_1:0&quot;, shape=(None, 3, 120, 160, 3), dtype=float32), but it was called on an input with incompatible shape (1, 120, 160, 3).
  Traceback (most recent call last):
    File &quot;C:\Users\kovad\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\framework\tensor_shape.py&quot;, line 930, in merge_with
      self.assert_same_rank(other)
    File &quot;C:\Users\kovad\AppData\Roaming\Python\Python37\site-packages\tensorflow\python\framework\tensor_shape.py&quot;, line 985, in assert_same_rank
      (self, other))
  ValueError: Shapes (120, 160, 3) and (None, None, None, None) must have the same rank
  </code></pre>