JobDispatcher { protected void submitNextNewJob(HarvestChannel channel) { Job jobToSubmit = prepareNextJobForSubmission(channel); if (jobToSubmit == null) { log.trace("No {} jobs to be run at this time", channel.getName()); } else { log.debug("Submitting new {} job {}", channel.getName(), jobToSubmit.getJobID()); try { List<MetadataEntry> metadata = createMetadata(jobToSubmit); String hName = harvestDefinitionDAO.getHarvestName(jobToSubmit.getOrigHarvestDefinitionID()); String schedule = ""; String hdComments = ""; String hdAudience = ""; SparseFullHarvest fh = harvestDefinitionDAO.getSparseFullHarvest(hName); if (fh != null) { hdComments = fh.getComments(); } else { SparsePartialHarvest ph = harvestDefinitionDAO.getSparsePartialHarvest(hName); if (ph == null) { throw new ArgumentNotValid("No harvest definition found for id '" + jobToSubmit.getOrigHarvestDefinitionID() + "', named '" + hName + "'"); } schedule = ph.getScheduleName(); hdComments = ph.getComments(); hdAudience = ph.getAudience(); } doOneCrawl(jobToSubmit, hName, hdComments, schedule, channel, hdAudience, metadata); log.info("Job #{} submitted", jobToSubmit.getJobID()); } catch (Throwable t) { String message = "Error while dispatching job " + jobToSubmit.getJobID() + ". Job status changed to FAILED"; log.warn(message, t); if (jobToSubmit != null) { jobToSubmit.setStatus(JobStatus.FAILED); jobToSubmit.appendHarvestErrors(message); jobToSubmit.appendHarvestErrorDetails(ExceptionUtils.getStackTrace(t)); jobDao.update(jobToSubmit); } } } } JobDispatcher(JMSConnection jmsConnection, HarvestDefinitionDAO hDao, JobDAO jobDao); void doOneCrawl(Job job, String origHarvestName, String origHarvestDesc, String origHarvestSchedule,
            HarvestChannel channel, String origHarvestAudience, List<MetadataEntry> metadata);  }
@Test public void testSubmitNewJobs() throws DocumentException { prepareDefaultMockAnswers(SELECTIVE_HARVEST_CHANNEL, jobMock, false); jobDispatcher.submitNextNewJob(SELECTIVE_HARVEST_CHANNEL); verify(jobMock).setStatus(JobStatus.SUBMITTED); verify(jobMock).setSubmittedDate(any(Date.class)); verify(jobDAO, new Times(1)).update(jobMock); verify(jmsConnection).send(crawlMessageCaptor.capture()); assertTrue(jobMock == crawlMessageCaptor.getValue().getJob()); assertEquals(HarvesterChannels.getHarvestJobChannelId(SELECTIVE_HARVEST_CHANNEL), crawlMessageCaptor.getValue() .getTo()); assertEquals(harvest.getName(), crawlMessageCaptor.getValue().getOrigHarvestInfo().getOrigHarvestName()); }
@Test public void testSubmitNewJobsMakesAliasInfo() throws SQLException { prepareDefaultMockAnswers(SELECTIVE_HARVEST_CHANNEL, jobMock, false); String originalDomain = "netarkiv.dk"; String aliasDomain = "netatarkivalias.dk"; when(jobDAO.getJobAliasInfo(any(Job.class))).thenReturn( Arrays.asList(new AliasInfo[] {new AliasInfo("netatarkivalias.dk", "netarkiv.dk", new Date())})); jobDispatcher.submitNextNewJob(SELECTIVE_HARVEST_CHANNEL); verify(jmsConnection).send(crawlMessageCaptor.capture()); assertEquals("Should have 1 metadata entry in last received message", 1, crawlMessageCaptor.getValue() .getMetadata().size()); assertEquals(aliasDomain + " is an alias for " + originalDomain + "\n", new String(crawlMessageCaptor .getValue().getMetadata().get(0).getData())); }
@Test public void testSubmitNewJobsMakesDuplicateReductionInfo() throws DocumentException { prepareDefaultMockAnswers(SELECTIVE_HARVEST_CHANNEL, jobMock, true); List<Long> jobIDsForDuplicateReduction = Arrays.asList(new Long[] {1L}); when(jobDAO.getJobIDsForDuplicateReduction(jobMock.getJobID())).thenReturn(jobIDsForDuplicateReduction); jobDispatcher.submitNextNewJob(SELECTIVE_HARVEST_CHANNEL); verify(jmsConnection).send(crawlMessageCaptor.capture()); DoOneCrawlMessage crawlMessage = crawlMessageCaptor.getValue(); assertEquals("Should have 1 metadata entry in the crawl request", 1, crawlMessage.getMetadata().size()); MetadataEntry metadataEntry = crawlMessage.getMetadata().get(0); assertNotNull("Should have 1 metadata entry", metadataEntry); assertEquals("Should have mimetype text/plain", "text/plain", metadataEntry.getMimeType()); assertEquals("Should have right url", "metadata: + "&harvestid=" + harvest.getOid() + "&harvestnum=" + jobMock.getHarvestNum() + "&jobid=" + jobMock.getJobID(), metadataEntry.getURL()); assertEquals("Should have right data", jobIDsForDuplicateReduction.get(0) + "", new String(metadataEntry.getData())); }
@Test public void testSendingToCorrectQueue() { prepareDefaultMockAnswers(SELECTIVE_HARVEST_CHANNEL, jobMock, false); jobDispatcher.submitNextNewJob(SELECTIVE_HARVEST_CHANNEL); verify(jmsConnection).send(crawlMessageCaptor.capture()); assertTrue(jobMock == crawlMessageCaptor.getValue().getJob()); assertEquals(HarvesterChannels.getHarvestJobChannelId(SELECTIVE_HARVEST_CHANNEL), crawlMessageCaptor.getValue() .getTo()); reset(jmsConnection); Job snapshotJob = createJob(2); prepareDefaultMockAnswers(SNAPSHOT, snapshotJob, false); jobDispatcher.submitNextNewJob(SNAPSHOT); verify(jmsConnection).send(crawlMessageCaptor.capture()); assertTrue(snapshotJob == crawlMessageCaptor.getValue().getJob()); assertEquals(HarvesterChannels.getHarvestJobChannelId(SNAPSHOT), crawlMessageCaptor.getValue().getTo()); }