@Test public void when_runIsCalled_expect_experienceHandledWithLearningBehavior() { AgentLearner.Configuration configuration = AgentLearner.Configuration.builder() .maxEpisodeSteps(4) .build(); AgentLearner<Integer> sut = new AgentLearner(environmentMock, transformProcessMock, policyMock, configuration, null, learningBehaviorMock); Schema schema = new Schema(new IntegerActionSchema(0, -1)); when(environmentMock.getSchema()).thenReturn(schema); when(environmentMock.reset()).thenReturn(new HashMap<>()); double[] reward = new double[] { 0.0 }; when(environmentMock.step(any(Integer.class))) .thenAnswer(a -> new StepResult(new HashMap<>(), ++reward[0], reward[0] == 4.0)); when(environmentMock.isEpisodeFinished()).thenAnswer(x -> reward[0] == 4.0); when(transformProcessMock.transform(any(Map.class), anyInt(), anyBoolean())) .thenAnswer(new Answer<Observation>() { public Observation answer(InvocationOnMock invocation) throws Throwable { int step = (int)invocation.getArgument(1); boolean isTerminal = (boolean)invocation.getArgument(2); return (step % 2 == 0 || isTerminal) ? new Observation(Nd4j.create(new double[] { step * 1.1 })) : Observation.SkippedObservation; } }); when(policyMock.nextAction(any(Observation.class))).thenAnswer(x -> (int)reward[0]); sut.run(); ArgumentCaptor<Observation> observationCaptor = ArgumentCaptor.forClass(Observation.class); ArgumentCaptor<Integer> actionCaptor = ArgumentCaptor.forClass(Integer.class); ArgumentCaptor<Double> rewardCaptor = ArgumentCaptor.forClass(Double.class); ArgumentCaptor<Boolean> isTerminalCaptor = ArgumentCaptor.forClass(Boolean.class); verify(learningBehaviorMock, times(2)).handleNewExperience(observationCaptor.capture(), actionCaptor.capture(), rewardCaptor.capture(), isTerminalCaptor.capture()); List<Observation> observations = observationCaptor.getAllValues(); List<Integer> actions = actionCaptor.getAllValues(); List<Double> rewards = rewardCaptor.getAllValues(); List<Boolean> isTerminalList = isTerminalCaptor.getAllValues(); assertEquals(0.0, observations.get(0).getData().getDouble(0), 0.00001); assertEquals(0, (int)actions.get(0)); assertEquals(0.0 + 1.0, rewards.get(0), 0.00001); assertFalse(isTerminalList.get(0)); assertEquals(2.2, observations.get(1).getData().getDouble(0), 0.00001); assertEquals(2, (int)actions.get(1)); assertEquals(2.0 + 3.0, rewards.get(1), 0.00001); assertFalse(isTerminalList.get(1)); ArgumentCaptor<Observation> finalObservationCaptor = ArgumentCaptor.forClass(Observation.class); verify(learningBehaviorMock, times(1)).handleEpisodeEnd(finalObservationCaptor.capture()); assertEquals(4.4, finalObservationCaptor.getValue().getData().getDouble(0), 0.00001); }
@Override protected void reset() { super.reset(); rewardAtLastExperience = 0; }
AgentLearner extends Agent<ACTION> implements IAgentLearner<ACTION> { @Override protected void reset() { super.reset(); rewardAtLastExperience = 0; } }
AgentLearner extends Agent<ACTION> implements IAgentLearner<ACTION> { @Override protected void reset() { super.reset(); rewardAtLastExperience = 0; } AgentLearner(Environment<ACTION> environment,
                        TransformProcess transformProcess,
                        IPolicy<ACTION> policy,
                        Configuration configuration,
                        String id,
                        @NonNull ILearningBehavior<ACTION> learningBehavior); }
AgentLearner extends Agent<ACTION> implements IAgentLearner<ACTION> { @Override protected void reset() { super.reset(); rewardAtLastExperience = 0; } AgentLearner(Environment<ACTION> environment,
                        TransformProcess transformProcess,
                        IPolicy<ACTION> policy,
                        Configuration configuration,
                        String id,
                        @NonNull ILearningBehavior<ACTION> learningBehavior);  }
AgentLearner extends Agent<ACTION> implements IAgentLearner<ACTION> { @Override protected void reset() { super.reset(); rewardAtLastExperience = 0; } AgentLearner(Environment<ACTION> environment,
                        TransformProcess transformProcess,
                        IPolicy<ACTION> policy,
                        Configuration configuration,
                        String id,
                        @NonNull ILearningBehavior<ACTION> learningBehavior);   }