@Test public void when_terminalAndNoTargetUpdate_expect_initRewardWithMaxQFromCurrent() { setup(1.0); final Observation observation = new Observation(Nd4j.create(new double[] { -123.0, -234.0 }).reshape(1, 2)); List<StateActionPair<Integer>> experience = new ArrayList<StateActionPair<Integer>>() { { add(new StateActionPair<Integer>(observation, 0, 0.0, false)); } }; sut.computeGradients(dqnMock, experience); ArgumentCaptor<INDArray> argument = ArgumentCaptor.forClass(INDArray.class); verify(dqnMock, times(2)).outputAll(argument.capture()); List<INDArray> values = argument.getAllValues(); assertEquals(-123.0, values.get(0).getDouble(0, 0), 0.00001); assertEquals(-123.0, values.get(1).getDouble(0, 0), 0.00001); verify(dqnMock, times(1)).gradient(any(INDArray.class), argThat((INDArray x) -> x.getDouble(0) == 234.0)); }
@Override public Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience) { int size = experience.size(); StateActionPair<Integer> stateActionPair = experience.get(size - 1); INDArray data = stateActionPair.getObservation().getData(); INDArray features = INDArrayHelper.createBatchForShape(size, data.shape()); INDArray targets = Nd4j.create(size, actionSpaceSize); double r; if (stateActionPair.isTerminal()) { r = 0; } else { INDArray[] output = null; output = current.outputAll(data); r = Nd4j.max(output[0]).getDouble(0); } for (int i = size - 1; i >= 0; i--) { stateActionPair = experience.get(i); data = stateActionPair.getObservation().getData(); features.putRow(i, data); r = stateActionPair.getReward() + gamma * r; INDArray[] output = current.outputAll(data); INDArray row = output[0]; row = row.putScalar(stateActionPair.getAction(), r); targets.putRow(i, row); } return current.gradient(features, targets); }
QLearningUpdateAlgorithm implements UpdateAlgorithm<IDQN> { @Override public Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience) { int size = experience.size(); StateActionPair<Integer> stateActionPair = experience.get(size - 1); INDArray data = stateActionPair.getObservation().getData(); INDArray features = INDArrayHelper.createBatchForShape(size, data.shape()); INDArray targets = Nd4j.create(size, actionSpaceSize); double r; if (stateActionPair.isTerminal()) { r = 0; } else { INDArray[] output = null; output = current.outputAll(data); r = Nd4j.max(output[0]).getDouble(0); } for (int i = size - 1; i >= 0; i--) { stateActionPair = experience.get(i); data = stateActionPair.getObservation().getData(); features.putRow(i, data); r = stateActionPair.getReward() + gamma * r; INDArray[] output = current.outputAll(data); INDArray row = output[0]; row = row.putScalar(stateActionPair.getAction(), r); targets.putRow(i, row); } return current.gradient(features, targets); } }
QLearningUpdateAlgorithm implements UpdateAlgorithm<IDQN> { @Override public Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience) { int size = experience.size(); StateActionPair<Integer> stateActionPair = experience.get(size - 1); INDArray data = stateActionPair.getObservation().getData(); INDArray features = INDArrayHelper.createBatchForShape(size, data.shape()); INDArray targets = Nd4j.create(size, actionSpaceSize); double r; if (stateActionPair.isTerminal()) { r = 0; } else { INDArray[] output = null; output = current.outputAll(data); r = Nd4j.max(output[0]).getDouble(0); } for (int i = size - 1; i >= 0; i--) { stateActionPair = experience.get(i); data = stateActionPair.getObservation().getData(); features.putRow(i, data); r = stateActionPair.getReward() + gamma * r; INDArray[] output = current.outputAll(data); INDArray row = output[0]; row = row.putScalar(stateActionPair.getAction(), r); targets.putRow(i, row); } return current.gradient(features, targets); } QLearningUpdateAlgorithm(int actionSpaceSize,
                                    double gamma); }
QLearningUpdateAlgorithm implements UpdateAlgorithm<IDQN> { @Override public Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience) { int size = experience.size(); StateActionPair<Integer> stateActionPair = experience.get(size - 1); INDArray data = stateActionPair.getObservation().getData(); INDArray features = INDArrayHelper.createBatchForShape(size, data.shape()); INDArray targets = Nd4j.create(size, actionSpaceSize); double r; if (stateActionPair.isTerminal()) { r = 0; } else { INDArray[] output = null; output = current.outputAll(data); r = Nd4j.max(output[0]).getDouble(0); } for (int i = size - 1; i >= 0; i--) { stateActionPair = experience.get(i); data = stateActionPair.getObservation().getData(); features.putRow(i, data); r = stateActionPair.getReward() + gamma * r; INDArray[] output = current.outputAll(data); INDArray row = output[0]; row = row.putScalar(stateActionPair.getAction(), r); targets.putRow(i, row); } return current.gradient(features, targets); } QLearningUpdateAlgorithm(int actionSpaceSize,
                                    double gamma); @Override Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience); }
QLearningUpdateAlgorithm implements UpdateAlgorithm<IDQN> { @Override public Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience) { int size = experience.size(); StateActionPair<Integer> stateActionPair = experience.get(size - 1); INDArray data = stateActionPair.getObservation().getData(); INDArray features = INDArrayHelper.createBatchForShape(size, data.shape()); INDArray targets = Nd4j.create(size, actionSpaceSize); double r; if (stateActionPair.isTerminal()) { r = 0; } else { INDArray[] output = null; output = current.outputAll(data); r = Nd4j.max(output[0]).getDouble(0); } for (int i = size - 1; i >= 0; i--) { stateActionPair = experience.get(i); data = stateActionPair.getObservation().getData(); features.putRow(i, data); r = stateActionPair.getReward() + gamma * r; INDArray[] output = current.outputAll(data); INDArray row = output[0]; row = row.putScalar(stateActionPair.getAction(), r); targets.putRow(i, row); } return current.gradient(features, targets); } QLearningUpdateAlgorithm(int actionSpaceSize,
                                    double gamma); @Override Gradient[] computeGradients(IDQN current, List<StateActionPair<Integer>> experience);  }