@Test public void when_singleTrainStep_expect_correctValues() { mockTestContext(100,0,2,1.0, 10, null); Observation observation = new Observation(Nd4j.zeros(observationShape)); NeuralNetOutput netOutputResult = new NeuralNetOutput(); netOutputResult.put(CommonOutputNames.QValues, Nd4j.create(new float[] {1.0f, 0.5f})); when(mockDQN.output(eq(observation))).thenReturn(netOutputResult); when(mockMDP.step(anyInt())).thenReturn(new StepReply<>(new Observation(Nd4j.zeros(observationShape)), 0, false, null)); QLearning.QLStepReturn<Observation> stepReturn = qLearningDiscrete.trainStep(observation); assertEquals(1.0, stepReturn.getMaxQ(), 1e-5); StepReply<Observation> stepReply = stepReturn.getStepReply(); assertEquals(0, stepReply.getReward(), 1e-5); assertFalse(stepReply.isDone()); assertFalse(stepReply.getObservation().isSkipped()); assertEquals(observation.getData().reshape(observationShape), stepReply.getObservation().getData().reshape(observationShape)); }
protected QLStepReturn<Observation> trainStep(Observation obs) { Double maxQ = Double.NaN; if (!obs.isSkipped()) { INDArray qs = getQNetwork().output(obs).get(CommonOutputNames.QValues); int maxAction = Learning.getMaxAction(qs); maxQ = qs.getDouble(maxAction); lastAction = getEgPolicy().nextAction(obs); } StepReply<Observation> stepReply = mdp.step(lastAction); accuReward += stepReply.getReward() * configuration.getRewardFactor(); if (!obs.isSkipped()) { learningBehavior.handleNewExperience(obs, lastAction, accuReward, stepReply.isDone()); accuReward = 0; } return new QLStepReturn<>(maxQ, getQNetwork().getLatestScore(), stepReply); }
QLearningDiscrete extends QLearning<O, Integer, DiscreteSpace> { protected QLStepReturn<Observation> trainStep(Observation obs) { Double maxQ = Double.NaN; if (!obs.isSkipped()) { INDArray qs = getQNetwork().output(obs).get(CommonOutputNames.QValues); int maxAction = Learning.getMaxAction(qs); maxQ = qs.getDouble(maxAction); lastAction = getEgPolicy().nextAction(obs); } StepReply<Observation> stepReply = mdp.step(lastAction); accuReward += stepReply.getReward() * configuration.getRewardFactor(); if (!obs.isSkipped()) { learningBehavior.handleNewExperience(obs, lastAction, accuReward, stepReply.isDone()); accuReward = 0; } return new QLStepReturn<>(maxQ, getQNetwork().getLatestScore(), stepReply); } }
QLearningDiscrete extends QLearning<O, Integer, DiscreteSpace> { protected QLStepReturn<Observation> trainStep(Observation obs) { Double maxQ = Double.NaN; if (!obs.isSkipped()) { INDArray qs = getQNetwork().output(obs).get(CommonOutputNames.QValues); int maxAction = Learning.getMaxAction(qs); maxQ = qs.getDouble(maxAction); lastAction = getEgPolicy().nextAction(obs); } StepReply<Observation> stepReply = mdp.step(lastAction); accuReward += stepReply.getReward() * configuration.getRewardFactor(); if (!obs.isSkipped()) { learningBehavior.handleNewExperience(obs, lastAction, accuReward, stepReply.isDone()); accuReward = 0; } return new QLStepReturn<>(maxQ, getQNetwork().getLatestScore(), stepReply); } QLearningDiscrete(MDP<O, Integer, DiscreteSpace> mdp, IDQN dqn, QLearningConfiguration conf, int epsilonNbStep);  QLearningDiscrete(MDP<O, Integer, DiscreteSpace> mdp, IDQN dqn, QLearningConfiguration conf, int epsilonNbStep, Random random);  QLearningDiscrete(MDP<O, Integer, DiscreteSpace> mdp, IDQN dqn, QLearningConfiguration conf,
                             int epsilonNbStep, ILearningBehavior<Integer> learningBehavior, Random random); }
QLearningDiscrete extends QLearning<O, Integer, DiscreteSpace> { protected QLStepReturn<Observation> trainStep(Observation obs) { Double maxQ = Double.NaN; if (!obs.isSkipped()) { INDArray qs = getQNetwork().output(obs).get(CommonOutputNames.QValues); int maxAction = Learning.getMaxAction(qs); maxQ = qs.getDouble(maxAction); lastAction = getEgPolicy().nextAction(obs); } StepReply<Observation> stepReply = mdp.step(lastAction); accuReward += stepReply.getReward() * configuration.getRewardFactor(); if (!obs.isSkipped()) { learningBehavior.handleNewExperience(obs, lastAction, accuReward, stepReply.isDone()); accuReward = 0; } return new QLStepReturn<>(maxQ, getQNetwork().getLatestScore(), stepReply); } QLearningDiscrete(MDP<O, Integer, DiscreteSpace> mdp, IDQN dqn, QLearningConfiguration conf, int epsilonNbStep);  QLearningDiscrete(MDP<O, Integer, DiscreteSpace> mdp, IDQN dqn, QLearningConfiguration conf, int epsilonNbStep, Random random);  QLearningDiscrete(MDP<O, Integer, DiscreteSpace> mdp, IDQN dqn, QLearningConfiguration conf,
                             int epsilonNbStep, ILearningBehavior<Integer> learningBehavior, Random random); MDP<O, Integer, DiscreteSpace> getMdp(); void postEpoch(); void preEpoch(); @Override void setHistoryProcessor(IHistoryProcessor historyProcessor); }
QLearningDiscrete extends QLearning<O, Integer, DiscreteSpace> { protected QLStepReturn<Observation> trainStep(Observation obs) { Double maxQ = Double.NaN; if (!obs.isSkipped()) { INDArray qs = getQNetwork().output(obs).get(CommonOutputNames.QValues); int maxAction = Learning.getMaxAction(qs); maxQ = qs.getDouble(maxAction); lastAction = getEgPolicy().nextAction(obs); } StepReply<Observation> stepReply = mdp.step(lastAction); accuReward += stepReply.getReward() * configuration.getRewardFactor(); if (!obs.isSkipped()) { learningBehavior.handleNewExperience(obs, lastAction, accuReward, stepReply.isDone()); accuReward = 0; } return new QLStepReturn<>(maxQ, getQNetwork().getLatestScore(), stepReply); } QLearningDiscrete(MDP<O, Integer, DiscreteSpace> mdp, IDQN dqn, QLearningConfiguration conf, int epsilonNbStep);  QLearningDiscrete(MDP<O, Integer, DiscreteSpace> mdp, IDQN dqn, QLearningConfiguration conf, int epsilonNbStep, Random random);  QLearningDiscrete(MDP<O, Integer, DiscreteSpace> mdp, IDQN dqn, QLearningConfiguration conf,
                             int epsilonNbStep, ILearningBehavior<Integer> learningBehavior, Random random); MDP<O, Integer, DiscreteSpace> getMdp(); void postEpoch(); void preEpoch(); @Override void setHistoryProcessor(IHistoryProcessor historyProcessor);  }