@Test public void when_callingGetTargetExpectedQValuesOfLast_expect_INDArrayWithCorrectShape() { List<StateActionPair<Integer>> experience = new ArrayList<StateActionPair<Integer>>() { { add(new StateActionPair<Integer>(new Observation(Nd4j.create(new double[] { 1.1, 1.2 }).reshape(1, 2, 1)), 0, 1.0, false)); add(new StateActionPair<Integer>(new Observation(Nd4j.create(new double[] { 2.1, 2.2 }).reshape(1, 2, 1)), 1, 2.0, false)); } }; INDArray features = Nd4j.create(new double[] { 1.0, 2.0, 3.0, 4.0 }).reshape(1, 2, 2); IOutputNeuralNet targetMock = mock(IOutputNeuralNet.class); final NeuralNetOutput neuralNetOutput = new NeuralNetOutput(); neuralNetOutput.put(CommonOutputNames.QValues, Nd4j.create(new double[] { -4.1, -4.2 }).reshape(1, 1, 2)); when(targetMock.output(any(INDArray.class))).thenReturn(neuralNetOutput); INDArray result = sut.getTargetExpectedQValuesOfLast(targetMock, experience, features); ArgumentCaptor<INDArray> arrayCaptor = ArgumentCaptor.forClass(INDArray.class); verify(targetMock, times(1)).output(arrayCaptor.capture()); INDArray array = arrayCaptor.getValue(); assertEquals(1.0, array.getDouble(0, 0, 0), 0.00001); assertEquals(2.0, array.getDouble(0, 0, 1), 0.00001); assertEquals(3.0, array.getDouble(0, 1, 0), 0.00001); assertEquals(4.0, array.getDouble(0, 1, 1), 0.00001); assertEquals(-4.2, result.getDouble(0, 0, 0), 0.00001); }
@Override public INDArray getTargetExpectedQValuesOfLast(IOutputNeuralNet target, List<StateActionPair<Integer>> trainingBatch, INDArray features) { return getElementAtIndex(target.output(features).get(CommonOutputNames.QValues), trainingBatch.size() - 1); }
RecurrentNStepQLearningHelper extends NStepQLearningHelper { @Override public INDArray getTargetExpectedQValuesOfLast(IOutputNeuralNet target, List<StateActionPair<Integer>> trainingBatch, INDArray features) { return getElementAtIndex(target.output(features).get(CommonOutputNames.QValues), trainingBatch.size() - 1); } }
RecurrentNStepQLearningHelper extends NStepQLearningHelper { @Override public INDArray getTargetExpectedQValuesOfLast(IOutputNeuralNet target, List<StateActionPair<Integer>> trainingBatch, INDArray features) { return getElementAtIndex(target.output(features).get(CommonOutputNames.QValues), trainingBatch.size() - 1); } RecurrentNStepQLearningHelper(int actionSpaceSize); }
RecurrentNStepQLearningHelper extends NStepQLearningHelper { @Override public INDArray getTargetExpectedQValuesOfLast(IOutputNeuralNet target, List<StateActionPair<Integer>> trainingBatch, INDArray features) { return getElementAtIndex(target.output(features).get(CommonOutputNames.QValues), trainingBatch.size() - 1); } RecurrentNStepQLearningHelper(int actionSpaceSize); @Override INDArray createLabels(int trainingBatchSize); @Override INDArray getExpectedQValues(INDArray allExpectedQValues, int idx); @Override void setLabels(INDArray labels, long idx, INDArray data); @Override INDArray getTargetExpectedQValuesOfLast(IOutputNeuralNet target, List<StateActionPair<Integer>> trainingBatch, INDArray features); }
RecurrentNStepQLearningHelper extends NStepQLearningHelper { @Override public INDArray getTargetExpectedQValuesOfLast(IOutputNeuralNet target, List<StateActionPair<Integer>> trainingBatch, INDArray features) { return getElementAtIndex(target.output(features).get(CommonOutputNames.QValues), trainingBatch.size() - 1); } RecurrentNStepQLearningHelper(int actionSpaceSize); @Override INDArray createLabels(int trainingBatchSize); @Override INDArray getExpectedQValues(INDArray allExpectedQValues, int idx); @Override void setLabels(INDArray labels, long idx, INDArray data); @Override INDArray getTargetExpectedQValuesOfLast(IOutputNeuralNet target, List<StateActionPair<Integer>> trainingBatch, INDArray features);  }