@Test public void refac_calcGradient_non_terminal() { int[] observationShape = new int[]{5}; double gamma = 0.9; AdvantageActorCriticUpdateAlgorithm algorithm = new AdvantageActorCriticUpdateAlgorithm(false, observationShape, 1, gamma); INDArray[] originalObservations = new INDArray[]{ Nd4j.create(new double[]{0.0, 0.1, 0.2, 0.3, 0.4}), Nd4j.create(new double[]{1.0, 1.1, 1.2, 1.3, 1.4}), Nd4j.create(new double[]{2.0, 2.1, 2.2, 2.3, 2.4}), Nd4j.create(new double[]{3.0, 3.1, 3.2, 3.3, 3.4}), }; int[] actions = new int[]{0, 1, 2, 1}; double[] rewards = new double[]{0.1, 1.0, 10.0, 100.0}; List<StateActionPair<Integer>> experience = new ArrayList<StateActionPair<Integer>>(); for (int i = 0; i < originalObservations.length; ++i) { experience.add(new StateActionPair<>(new Observation(originalObservations[i]), actions[i], rewards[i], false)); } when(mockActorCritic.outputAll(any(INDArray.class))).thenAnswer(invocation -> { INDArray batch = invocation.getArgument(0); return new INDArray[]{batch.mul(-1.0)}; }); ArgumentCaptor<INDArray> inputArgumentCaptor = ArgumentCaptor.forClass(INDArray.class); ArgumentCaptor<INDArray[]> criticActorArgumentCaptor = ArgumentCaptor.forClass(INDArray[].class); algorithm.computeGradients(mockActorCritic, experience); verify(mockActorCritic, times(1)).gradient(inputArgumentCaptor.capture(), criticActorArgumentCaptor.capture()); assertEquals(Nd4j.stack(0, originalObservations), inputArgumentCaptor.getValue()); }
@Override public Gradient[] computeGradients(IActorCritic current, List<StateActionPair<Integer>> experience) { int size = experience.size(); int[] nshape = recurrent ? Learning.makeShape(1, shape, size) : Learning.makeShape(size, shape); INDArray input = Nd4j.create(nshape); INDArray targets = recurrent ? Nd4j.create(1, 1, size) : Nd4j.create(size, 1); INDArray logSoftmax = recurrent ? Nd4j.zeros(1, actionSpaceSize, size) : Nd4j.zeros(size, actionSpaceSize); StateActionPair<Integer> stateActionPair = experience.get(size - 1); double value; if (stateActionPair.isTerminal()) { value = 0; } else { INDArray[] output = current.outputAll(stateActionPair.getObservation().getData()); value = output[0].getDouble(0); } for (int i = size - 1; i >= 0; --i) { stateActionPair = experience.get(i); INDArray observationData = stateActionPair.getObservation().getData(); INDArray[] output = current.outputAll(observationData); value = stateActionPair.getReward() + gamma * value; if (recurrent) { input.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.point(i)).assign(observationData); } else { input.putRow(i, observationData); } targets.putScalar(i, value); double expectedV = output[0].getDouble(0); double advantage = value - expectedV; if (recurrent) { logSoftmax.putScalar(0, stateActionPair.getAction(), i, advantage); } else { logSoftmax.putScalar(i, stateActionPair.getAction(), advantage); } } return current.gradient(input, new INDArray[]{targets, logSoftmax}); }
AdvantageActorCriticUpdateAlgorithm implements UpdateAlgorithm<IActorCritic> { @Override public Gradient[] computeGradients(IActorCritic current, List<StateActionPair<Integer>> experience) { int size = experience.size(); int[] nshape = recurrent ? Learning.makeShape(1, shape, size) : Learning.makeShape(size, shape); INDArray input = Nd4j.create(nshape); INDArray targets = recurrent ? Nd4j.create(1, 1, size) : Nd4j.create(size, 1); INDArray logSoftmax = recurrent ? Nd4j.zeros(1, actionSpaceSize, size) : Nd4j.zeros(size, actionSpaceSize); StateActionPair<Integer> stateActionPair = experience.get(size - 1); double value; if (stateActionPair.isTerminal()) { value = 0; } else { INDArray[] output = current.outputAll(stateActionPair.getObservation().getData()); value = output[0].getDouble(0); } for (int i = size - 1; i >= 0; --i) { stateActionPair = experience.get(i); INDArray observationData = stateActionPair.getObservation().getData(); INDArray[] output = current.outputAll(observationData); value = stateActionPair.getReward() + gamma * value; if (recurrent) { input.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.point(i)).assign(observationData); } else { input.putRow(i, observationData); } targets.putScalar(i, value); double expectedV = output[0].getDouble(0); double advantage = value - expectedV; if (recurrent) { logSoftmax.putScalar(0, stateActionPair.getAction(), i, advantage); } else { logSoftmax.putScalar(i, stateActionPair.getAction(), advantage); } } return current.gradient(input, new INDArray[]{targets, logSoftmax}); } }
AdvantageActorCriticUpdateAlgorithm implements UpdateAlgorithm<IActorCritic> { @Override public Gradient[] computeGradients(IActorCritic current, List<StateActionPair<Integer>> experience) { int size = experience.size(); int[] nshape = recurrent ? Learning.makeShape(1, shape, size) : Learning.makeShape(size, shape); INDArray input = Nd4j.create(nshape); INDArray targets = recurrent ? Nd4j.create(1, 1, size) : Nd4j.create(size, 1); INDArray logSoftmax = recurrent ? Nd4j.zeros(1, actionSpaceSize, size) : Nd4j.zeros(size, actionSpaceSize); StateActionPair<Integer> stateActionPair = experience.get(size - 1); double value; if (stateActionPair.isTerminal()) { value = 0; } else { INDArray[] output = current.outputAll(stateActionPair.getObservation().getData()); value = output[0].getDouble(0); } for (int i = size - 1; i >= 0; --i) { stateActionPair = experience.get(i); INDArray observationData = stateActionPair.getObservation().getData(); INDArray[] output = current.outputAll(observationData); value = stateActionPair.getReward() + gamma * value; if (recurrent) { input.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.point(i)).assign(observationData); } else { input.putRow(i, observationData); } targets.putScalar(i, value); double expectedV = output[0].getDouble(0); double advantage = value - expectedV; if (recurrent) { logSoftmax.putScalar(0, stateActionPair.getAction(), i, advantage); } else { logSoftmax.putScalar(i, stateActionPair.getAction(), advantage); } } return current.gradient(input, new INDArray[]{targets, logSoftmax}); } AdvantageActorCriticUpdateAlgorithm(boolean recurrent,
                                               int[] shape,
                                               int actionSpaceSize,
                                               double gamma); }
AdvantageActorCriticUpdateAlgorithm implements UpdateAlgorithm<IActorCritic> { @Override public Gradient[] computeGradients(IActorCritic current, List<StateActionPair<Integer>> experience) { int size = experience.size(); int[] nshape = recurrent ? Learning.makeShape(1, shape, size) : Learning.makeShape(size, shape); INDArray input = Nd4j.create(nshape); INDArray targets = recurrent ? Nd4j.create(1, 1, size) : Nd4j.create(size, 1); INDArray logSoftmax = recurrent ? Nd4j.zeros(1, actionSpaceSize, size) : Nd4j.zeros(size, actionSpaceSize); StateActionPair<Integer> stateActionPair = experience.get(size - 1); double value; if (stateActionPair.isTerminal()) { value = 0; } else { INDArray[] output = current.outputAll(stateActionPair.getObservation().getData()); value = output[0].getDouble(0); } for (int i = size - 1; i >= 0; --i) { stateActionPair = experience.get(i); INDArray observationData = stateActionPair.getObservation().getData(); INDArray[] output = current.outputAll(observationData); value = stateActionPair.getReward() + gamma * value; if (recurrent) { input.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.point(i)).assign(observationData); } else { input.putRow(i, observationData); } targets.putScalar(i, value); double expectedV = output[0].getDouble(0); double advantage = value - expectedV; if (recurrent) { logSoftmax.putScalar(0, stateActionPair.getAction(), i, advantage); } else { logSoftmax.putScalar(i, stateActionPair.getAction(), advantage); } } return current.gradient(input, new INDArray[]{targets, logSoftmax}); } AdvantageActorCriticUpdateAlgorithm(boolean recurrent,
                                               int[] shape,
                                               int actionSpaceSize,
                                               double gamma); @Override Gradient[] computeGradients(IActorCritic current, List<StateActionPair<Integer>> experience); }
AdvantageActorCriticUpdateAlgorithm implements UpdateAlgorithm<IActorCritic> { @Override public Gradient[] computeGradients(IActorCritic current, List<StateActionPair<Integer>> experience) { int size = experience.size(); int[] nshape = recurrent ? Learning.makeShape(1, shape, size) : Learning.makeShape(size, shape); INDArray input = Nd4j.create(nshape); INDArray targets = recurrent ? Nd4j.create(1, 1, size) : Nd4j.create(size, 1); INDArray logSoftmax = recurrent ? Nd4j.zeros(1, actionSpaceSize, size) : Nd4j.zeros(size, actionSpaceSize); StateActionPair<Integer> stateActionPair = experience.get(size - 1); double value; if (stateActionPair.isTerminal()) { value = 0; } else { INDArray[] output = current.outputAll(stateActionPair.getObservation().getData()); value = output[0].getDouble(0); } for (int i = size - 1; i >= 0; --i) { stateActionPair = experience.get(i); INDArray observationData = stateActionPair.getObservation().getData(); INDArray[] output = current.outputAll(observationData); value = stateActionPair.getReward() + gamma * value; if (recurrent) { input.get(NDArrayIndex.point(0), NDArrayIndex.all(), NDArrayIndex.point(i)).assign(observationData); } else { input.putRow(i, observationData); } targets.putScalar(i, value); double expectedV = output[0].getDouble(0); double advantage = value - expectedV; if (recurrent) { logSoftmax.putScalar(0, stateActionPair.getAction(), i, advantage); } else { logSoftmax.putScalar(i, stateActionPair.getAction(), advantage); } } return current.gradient(input, new INDArray[]{targets, logSoftmax}); } AdvantageActorCriticUpdateAlgorithm(boolean recurrent,
                                               int[] shape,
                                               int actionSpaceSize,
                                               double gamma); @Override Gradient[] computeGradients(IActorCritic current, List<StateActionPair<Integer>> experience);  }