@Test public void when_callingWithMultipleExperiences_expect_gradientsAreValid() { double gamma = 0.9; setup(gamma); List<StateActionPair<Integer>> experience = new ArrayList<StateActionPair<Integer>>() { { add(new StateActionPair<Integer>(new Observation(Nd4j.create(new double[] { -1.1, -1.2 }).reshape(1, 2)), 0, 1.0, false)); add(new StateActionPair<Integer>(new Observation(Nd4j.create(new double[] { -2.1, -2.2 }).reshape(1, 2)), 1, 2.0, true)); } }; sut.computeGradients(dqnMock, experience); ArgumentCaptor<INDArray> features = ArgumentCaptor.forClass(INDArray.class); ArgumentCaptor<INDArray> targets = ArgumentCaptor.forClass(INDArray.class); verify(dqnMock, times(1)).gradient(features.capture(), targets.capture()); INDArray featuresValues = features.getValue(); assertEquals(-1.1, featuresValues.getDouble(0, 0), 0.00001); assertEquals(-1.2, featuresValues.getDouble(0, 1), 0.00001); assertEquals(-2.1, featuresValues.getDouble(1, 0), 0.00001); assertEquals(-2.2, featuresValues.getDouble(1, 1), 0.00001); INDArray targetsValues = targets.getValue(); assertEquals(1.0 + gamma * 2.0, targetsValues.getDouble(0, 0), 0.00001); assertEquals(1.2, targetsValues.getDouble(0, 1), 0.00001); assertEquals(2.1, targetsValues.getDouble(1, 0), 0.00001); assertEquals(2.0, targetsValues.getDouble(1, 1), 0.00001); }
@Override public Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience) { int size = experience.size(); StateActionPair<Integer> stateActionPair = experience.get(size - 1); INDArray data = stateActionPair.getObservation().getData(); INDArray features = INDArrayHelper.createBatchForShape(size, data.shape()); INDArray targets = Nd4j.create(size, actionSpaceSize); double r; if (stateActionPair.isTerminal()) { r = 0; } else { INDArray[] output = null; output = current.outputAll(data); r = Nd4j.max(output[0]).getDouble(0); } for (int i = size - 1; i >= 0; i--) { stateActionPair = experience.get(i); data = stateActionPair.getObservation().getData(); features.putRow(i, data); r = stateActionPair.getReward() + gamma * r; INDArray[] output = current.outputAll(data); INDArray row = output[0]; row = row.putScalar(stateActionPair.getAction(), r); targets.putRow(i, row); } return current.gradient(features, targets); }
QLearningUpdateAlgorithm implements UpdateAlgorithm<IDQN> { @Override public Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience) { int size = experience.size(); StateActionPair<Integer> stateActionPair = experience.get(size - 1); INDArray data = stateActionPair.getObservation().getData(); INDArray features = INDArrayHelper.createBatchForShape(size, data.shape()); INDArray targets = Nd4j.create(size, actionSpaceSize); double r; if (stateActionPair.isTerminal()) { r = 0; } else { INDArray[] output = null; output = current.outputAll(data); r = Nd4j.max(output[0]).getDouble(0); } for (int i = size - 1; i >= 0; i--) { stateActionPair = experience.get(i); data = stateActionPair.getObservation().getData(); features.putRow(i, data); r = stateActionPair.getReward() + gamma * r; INDArray[] output = current.outputAll(data); INDArray row = output[0]; row = row.putScalar(stateActionPair.getAction(), r); targets.putRow(i, row); } return current.gradient(features, targets); } }
QLearningUpdateAlgorithm implements UpdateAlgorithm<IDQN> { @Override public Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience) { int size = experience.size(); StateActionPair<Integer> stateActionPair = experience.get(size - 1); INDArray data = stateActionPair.getObservation().getData(); INDArray features = INDArrayHelper.createBatchForShape(size, data.shape()); INDArray targets = Nd4j.create(size, actionSpaceSize); double r; if (stateActionPair.isTerminal()) { r = 0; } else { INDArray[] output = null; output = current.outputAll(data); r = Nd4j.max(output[0]).getDouble(0); } for (int i = size - 1; i >= 0; i--) { stateActionPair = experience.get(i); data = stateActionPair.getObservation().getData(); features.putRow(i, data); r = stateActionPair.getReward() + gamma * r; INDArray[] output = current.outputAll(data); INDArray row = output[0]; row = row.putScalar(stateActionPair.getAction(), r); targets.putRow(i, row); } return current.gradient(features, targets); } QLearningUpdateAlgorithm(int actionSpaceSize,
                                    double gamma); }
QLearningUpdateAlgorithm implements UpdateAlgorithm<IDQN> { @Override public Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience) { int size = experience.size(); StateActionPair<Integer> stateActionPair = experience.get(size - 1); INDArray data = stateActionPair.getObservation().getData(); INDArray features = INDArrayHelper.createBatchForShape(size, data.shape()); INDArray targets = Nd4j.create(size, actionSpaceSize); double r; if (stateActionPair.isTerminal()) { r = 0; } else { INDArray[] output = null; output = current.outputAll(data); r = Nd4j.max(output[0]).getDouble(0); } for (int i = size - 1; i >= 0; i--) { stateActionPair = experience.get(i); data = stateActionPair.getObservation().getData(); features.putRow(i, data); r = stateActionPair.getReward() + gamma * r; INDArray[] output = current.outputAll(data); INDArray row = output[0]; row = row.putScalar(stateActionPair.getAction(), r); targets.putRow(i, row); } return current.gradient(features, targets); } QLearningUpdateAlgorithm(int actionSpaceSize,
                                    double gamma); @Override Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience); }
QLearningUpdateAlgorithm implements UpdateAlgorithm<IDQN> { @Override public Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience) { int size = experience.size(); StateActionPair<Integer> stateActionPair = experience.get(size - 1); INDArray data = stateActionPair.getObservation().getData(); INDArray features = INDArrayHelper.createBatchForShape(size, data.shape()); INDArray targets = Nd4j.create(size, actionSpaceSize); double r; if (stateActionPair.isTerminal()) { r = 0; } else { INDArray[] output = null; output = current.outputAll(data); r = Nd4j.max(output[0]).getDouble(0); } for (int i = size - 1; i >= 0; i--) { stateActionPair = experience.get(i); data = stateActionPair.getObservation().getData(); features.putRow(i, data); r = stateActionPair.getReward() + gamma * r; INDArray[] output = current.outputAll(data); INDArray row = output[0]; row = row.putScalar(stateActionPair.getAction(), r); targets.putRow(i, row); } return current.gradient(features, targets); } QLearningUpdateAlgorithm(int actionSpaceSize,
                                    double gamma); @Override Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience);  }