@Test public void when_episodeLongerThanNsteps_expect_returnNStepLength() { int episodeRemaining = 5; int remainingTrainingSteps = 4; when(mockMDP.isDone()).thenAnswer(invocation -> asyncThreadDiscrete.getStepCount() == episodeRemaining ); when(mockLegacyMDPWrapper.step(0)).thenReturn(new StepReply<>(mockObservation, 0.0, false, null)); AsyncThread.SubEpochReturn subEpochReturn = asyncThreadDiscrete.trainSubEpoch(mockObservation, remainingTrainingSteps); assertFalse(subEpochReturn.isEpisodeComplete()); assertEquals(remainingTrainingSteps, subEpochReturn.getSteps()); }
public SubEpochReturn trainSubEpoch(Observation sObs, int trainingSteps) { current.copyFrom(getAsyncGlobal().getTarget()); Observation obs = sObs; IPolicy<Integer> policy = getPolicy(current); Integer action = getMdp().getActionSpace().noOp(); double reward = 0; double accuReward = 0; while (!getMdp().isDone() && experienceHandler.getTrainingBatchSize() != trainingSteps) { if (!obs.isSkipped()) { action = policy.nextAction(obs); } StepReply<Observation> stepReply = getLegacyMDPWrapper().step(action); accuReward += stepReply.getReward() * getConfiguration().getRewardFactor(); if (!obs.isSkipped()) { experienceHandler.addExperience(obs, action, accuReward, stepReply.isDone()); accuReward = 0; incrementSteps(); } obs = stepReply.getObservation(); reward += stepReply.getReward(); } boolean episodeComplete = getMdp().isDone() || getConfiguration().getMaxEpochStep() == currentEpisodeStepCount; if (episodeComplete && experienceHandler.getTrainingBatchSize() != trainingSteps) { experienceHandler.setFinalObservation(obs); } int experienceSize = experienceHandler.getTrainingBatchSize(); getAsyncGlobal().applyGradient(updateAlgorithm.computeGradients(current, experienceHandler.generateTrainingBatch()), experienceSize); return new SubEpochReturn(experienceSize, obs, reward, current.getLatestScore(), episodeComplete); }
AsyncThreadDiscrete extends AsyncThread<OBSERVATION, Integer, DiscreteSpace, NN> { public SubEpochReturn trainSubEpoch(Observation sObs, int trainingSteps) { current.copyFrom(getAsyncGlobal().getTarget()); Observation obs = sObs; IPolicy<Integer> policy = getPolicy(current); Integer action = getMdp().getActionSpace().noOp(); double reward = 0; double accuReward = 0; while (!getMdp().isDone() && experienceHandler.getTrainingBatchSize() != trainingSteps) { if (!obs.isSkipped()) { action = policy.nextAction(obs); } StepReply<Observation> stepReply = getLegacyMDPWrapper().step(action); accuReward += stepReply.getReward() * getConfiguration().getRewardFactor(); if (!obs.isSkipped()) { experienceHandler.addExperience(obs, action, accuReward, stepReply.isDone()); accuReward = 0; incrementSteps(); } obs = stepReply.getObservation(); reward += stepReply.getReward(); } boolean episodeComplete = getMdp().isDone() || getConfiguration().getMaxEpochStep() == currentEpisodeStepCount; if (episodeComplete && experienceHandler.getTrainingBatchSize() != trainingSteps) { experienceHandler.setFinalObservation(obs); } int experienceSize = experienceHandler.getTrainingBatchSize(); getAsyncGlobal().applyGradient(updateAlgorithm.computeGradients(current, experienceHandler.generateTrainingBatch()), experienceSize); return new SubEpochReturn(experienceSize, obs, reward, current.getLatestScore(), episodeComplete); } }
AsyncThreadDiscrete extends AsyncThread<OBSERVATION, Integer, DiscreteSpace, NN> { public SubEpochReturn trainSubEpoch(Observation sObs, int trainingSteps) { current.copyFrom(getAsyncGlobal().getTarget()); Observation obs = sObs; IPolicy<Integer> policy = getPolicy(current); Integer action = getMdp().getActionSpace().noOp(); double reward = 0; double accuReward = 0; while (!getMdp().isDone() && experienceHandler.getTrainingBatchSize() != trainingSteps) { if (!obs.isSkipped()) { action = policy.nextAction(obs); } StepReply<Observation> stepReply = getLegacyMDPWrapper().step(action); accuReward += stepReply.getReward() * getConfiguration().getRewardFactor(); if (!obs.isSkipped()) { experienceHandler.addExperience(obs, action, accuReward, stepReply.isDone()); accuReward = 0; incrementSteps(); } obs = stepReply.getObservation(); reward += stepReply.getReward(); } boolean episodeComplete = getMdp().isDone() || getConfiguration().getMaxEpochStep() == currentEpisodeStepCount; if (episodeComplete && experienceHandler.getTrainingBatchSize() != trainingSteps) { experienceHandler.setFinalObservation(obs); } int experienceSize = experienceHandler.getTrainingBatchSize(); getAsyncGlobal().applyGradient(updateAlgorithm.computeGradients(current, experienceHandler.generateTrainingBatch()), experienceSize); return new SubEpochReturn(experienceSize, obs, reward, current.getLatestScore(), episodeComplete); } AsyncThreadDiscrete(IAsyncGlobal<NN> asyncGlobal,
                               MDP<OBSERVATION, Integer, DiscreteSpace> mdp,
                               TrainingListenerList listeners,
                               int threadNumber,
                               int deviceNum); }
AsyncThreadDiscrete extends AsyncThread<OBSERVATION, Integer, DiscreteSpace, NN> { public SubEpochReturn trainSubEpoch(Observation sObs, int trainingSteps) { current.copyFrom(getAsyncGlobal().getTarget()); Observation obs = sObs; IPolicy<Integer> policy = getPolicy(current); Integer action = getMdp().getActionSpace().noOp(); double reward = 0; double accuReward = 0; while (!getMdp().isDone() && experienceHandler.getTrainingBatchSize() != trainingSteps) { if (!obs.isSkipped()) { action = policy.nextAction(obs); } StepReply<Observation> stepReply = getLegacyMDPWrapper().step(action); accuReward += stepReply.getReward() * getConfiguration().getRewardFactor(); if (!obs.isSkipped()) { experienceHandler.addExperience(obs, action, accuReward, stepReply.isDone()); accuReward = 0; incrementSteps(); } obs = stepReply.getObservation(); reward += stepReply.getReward(); } boolean episodeComplete = getMdp().isDone() || getConfiguration().getMaxEpochStep() == currentEpisodeStepCount; if (episodeComplete && experienceHandler.getTrainingBatchSize() != trainingSteps) { experienceHandler.setFinalObservation(obs); } int experienceSize = experienceHandler.getTrainingBatchSize(); getAsyncGlobal().applyGradient(updateAlgorithm.computeGradients(current, experienceHandler.generateTrainingBatch()), experienceSize); return new SubEpochReturn(experienceSize, obs, reward, current.getLatestScore(), episodeComplete); } AsyncThreadDiscrete(IAsyncGlobal<NN> asyncGlobal,
                               MDP<OBSERVATION, Integer, DiscreteSpace> mdp,
                               TrainingListenerList listeners,
                               int threadNumber,
                               int deviceNum); @Override void setHistoryProcessor(IHistoryProcessor historyProcessor); SubEpochReturn trainSubEpoch(Observation sObs, int trainingSteps); }
AsyncThreadDiscrete extends AsyncThread<OBSERVATION, Integer, DiscreteSpace, NN> { public SubEpochReturn trainSubEpoch(Observation sObs, int trainingSteps) { current.copyFrom(getAsyncGlobal().getTarget()); Observation obs = sObs; IPolicy<Integer> policy = getPolicy(current); Integer action = getMdp().getActionSpace().noOp(); double reward = 0; double accuReward = 0; while (!getMdp().isDone() && experienceHandler.getTrainingBatchSize() != trainingSteps) { if (!obs.isSkipped()) { action = policy.nextAction(obs); } StepReply<Observation> stepReply = getLegacyMDPWrapper().step(action); accuReward += stepReply.getReward() * getConfiguration().getRewardFactor(); if (!obs.isSkipped()) { experienceHandler.addExperience(obs, action, accuReward, stepReply.isDone()); accuReward = 0; incrementSteps(); } obs = stepReply.getObservation(); reward += stepReply.getReward(); } boolean episodeComplete = getMdp().isDone() || getConfiguration().getMaxEpochStep() == currentEpisodeStepCount; if (episodeComplete && experienceHandler.getTrainingBatchSize() != trainingSteps) { experienceHandler.setFinalObservation(obs); } int experienceSize = experienceHandler.getTrainingBatchSize(); getAsyncGlobal().applyGradient(updateAlgorithm.computeGradients(current, experienceHandler.generateTrainingBatch()), experienceSize); return new SubEpochReturn(experienceSize, obs, reward, current.getLatestScore(), episodeComplete); } AsyncThreadDiscrete(IAsyncGlobal<NN> asyncGlobal,
                               MDP<OBSERVATION, Integer, DiscreteSpace> mdp,
                               TrainingListenerList listeners,
                               int threadNumber,
                               int deviceNum); @Override void setHistoryProcessor(IHistoryProcessor historyProcessor); SubEpochReturn trainSubEpoch(Observation sObs, int trainingSteps);  }