@Test public void testSubmitNewJobsMakesDuplicateReductionInfo() throws DocumentException { prepareDefaultMockAnswers(SELECTIVE_HARVEST_CHANNEL, jobMock, true); List<Long> jobIDsForDuplicateReduction = Arrays.asList(new Long[] {1L}); when(jobDAO.getJobIDsForDuplicateReduction(jobMock.getJobID())).thenReturn(jobIDsForDuplicateReduction); jobDispatcher.submitNextNewJob(SELECTIVE_HARVEST_CHANNEL); verify(jmsConnection).send(crawlMessageCaptor.capture()); DoOneCrawlMessage crawlMessage = crawlMessageCaptor.getValue(); assertEquals("Should have 1 metadata entry in the crawl request", 1, crawlMessage.getMetadata().size()); MetadataEntry metadataEntry = crawlMessage.getMetadata().get(0); assertNotNull("Should have 1 metadata entry", metadataEntry); assertEquals("Should have mimetype text/plain", "text/plain", metadataEntry.getMimeType()); assertEquals("Should have right url", "metadata: + "&harvestid=" + harvest.getOid() + "&harvestnum=" + jobMock.getHarvestNum() + "&jobid=" + jobMock.getJobID(), metadataEntry.getURL()); assertEquals("Should have right data", jobIDsForDuplicateReduction.get(0) + "", new String(metadataEntry.getData())); }
protected void submitNextNewJob(HarvestChannel channel) { Job jobToSubmit = prepareNextJobForSubmission(channel); if (jobToSubmit == null) { log.trace("No {} jobs to be run at this time", channel.getName()); } else { log.debug("Submitting new {} job {}", channel.getName(), jobToSubmit.getJobID()); try { List<MetadataEntry> metadata = createMetadata(jobToSubmit); String hName = harvestDefinitionDAO.getHarvestName(jobToSubmit.getOrigHarvestDefinitionID()); String schedule = ""; String hdComments = ""; String hdAudience = ""; SparseFullHarvest fh = harvestDefinitionDAO.getSparseFullHarvest(hName); if (fh != null) { hdComments = fh.getComments(); } else { SparsePartialHarvest ph = harvestDefinitionDAO.getSparsePartialHarvest(hName); if (ph == null) { throw new ArgumentNotValid("No harvest definition found for id '" + jobToSubmit.getOrigHarvestDefinitionID() + "', named '" + hName + "'"); } schedule = ph.getScheduleName(); hdComments = ph.getComments(); hdAudience = ph.getAudience(); } doOneCrawl(jobToSubmit, hName, hdComments, schedule, channel, hdAudience, metadata); log.info("Job #{} submitted", jobToSubmit.getJobID()); } catch (Throwable t) { String message = "Error while dispatching job " + jobToSubmit.getJobID() + ". Job status changed to FAILED"; log.warn(message, t); if (jobToSubmit != null) { jobToSubmit.setStatus(JobStatus.FAILED); jobToSubmit.appendHarvestErrors(message); jobToSubmit.appendHarvestErrorDetails(ExceptionUtils.getStackTrace(t)); jobDao.update(jobToSubmit); } } } }
JobDispatcher { protected void submitNextNewJob(HarvestChannel channel) { Job jobToSubmit = prepareNextJobForSubmission(channel); if (jobToSubmit == null) { log.trace("No {} jobs to be run at this time", channel.getName()); } else { log.debug("Submitting new {} job {}", channel.getName(), jobToSubmit.getJobID()); try { List<MetadataEntry> metadata = createMetadata(jobToSubmit); String hName = harvestDefinitionDAO.getHarvestName(jobToSubmit.getOrigHarvestDefinitionID()); String schedule = ""; String hdComments = ""; String hdAudience = ""; SparseFullHarvest fh = harvestDefinitionDAO.getSparseFullHarvest(hName); if (fh != null) { hdComments = fh.getComments(); } else { SparsePartialHarvest ph = harvestDefinitionDAO.getSparsePartialHarvest(hName); if (ph == null) { throw new ArgumentNotValid("No harvest definition found for id '" + jobToSubmit.getOrigHarvestDefinitionID() + "', named '" + hName + "'"); } schedule = ph.getScheduleName(); hdComments = ph.getComments(); hdAudience = ph.getAudience(); } doOneCrawl(jobToSubmit, hName, hdComments, schedule, channel, hdAudience, metadata); log.info("Job #{} submitted", jobToSubmit.getJobID()); } catch (Throwable t) { String message = "Error while dispatching job " + jobToSubmit.getJobID() + ". Job status changed to FAILED"; log.warn(message, t); if (jobToSubmit != null) { jobToSubmit.setStatus(JobStatus.FAILED); jobToSubmit.appendHarvestErrors(message); jobToSubmit.appendHarvestErrorDetails(ExceptionUtils.getStackTrace(t)); jobDao.update(jobToSubmit); } } } } }
JobDispatcher { protected void submitNextNewJob(HarvestChannel channel) { Job jobToSubmit = prepareNextJobForSubmission(channel); if (jobToSubmit == null) { log.trace("No {} jobs to be run at this time", channel.getName()); } else { log.debug("Submitting new {} job {}", channel.getName(), jobToSubmit.getJobID()); try { List<MetadataEntry> metadata = createMetadata(jobToSubmit); String hName = harvestDefinitionDAO.getHarvestName(jobToSubmit.getOrigHarvestDefinitionID()); String schedule = ""; String hdComments = ""; String hdAudience = ""; SparseFullHarvest fh = harvestDefinitionDAO.getSparseFullHarvest(hName); if (fh != null) { hdComments = fh.getComments(); } else { SparsePartialHarvest ph = harvestDefinitionDAO.getSparsePartialHarvest(hName); if (ph == null) { throw new ArgumentNotValid("No harvest definition found for id '" + jobToSubmit.getOrigHarvestDefinitionID() + "', named '" + hName + "'"); } schedule = ph.getScheduleName(); hdComments = ph.getComments(); hdAudience = ph.getAudience(); } doOneCrawl(jobToSubmit, hName, hdComments, schedule, channel, hdAudience, metadata); log.info("Job #{} submitted", jobToSubmit.getJobID()); } catch (Throwable t) { String message = "Error while dispatching job " + jobToSubmit.getJobID() + ". Job status changed to FAILED"; log.warn(message, t); if (jobToSubmit != null) { jobToSubmit.setStatus(JobStatus.FAILED); jobToSubmit.appendHarvestErrors(message); jobToSubmit.appendHarvestErrorDetails(ExceptionUtils.getStackTrace(t)); jobDao.update(jobToSubmit); } } } } JobDispatcher(JMSConnection jmsConnection, HarvestDefinitionDAO hDao, JobDAO jobDao); }
JobDispatcher { protected void submitNextNewJob(HarvestChannel channel) { Job jobToSubmit = prepareNextJobForSubmission(channel); if (jobToSubmit == null) { log.trace("No {} jobs to be run at this time", channel.getName()); } else { log.debug("Submitting new {} job {}", channel.getName(), jobToSubmit.getJobID()); try { List<MetadataEntry> metadata = createMetadata(jobToSubmit); String hName = harvestDefinitionDAO.getHarvestName(jobToSubmit.getOrigHarvestDefinitionID()); String schedule = ""; String hdComments = ""; String hdAudience = ""; SparseFullHarvest fh = harvestDefinitionDAO.getSparseFullHarvest(hName); if (fh != null) { hdComments = fh.getComments(); } else { SparsePartialHarvest ph = harvestDefinitionDAO.getSparsePartialHarvest(hName); if (ph == null) { throw new ArgumentNotValid("No harvest definition found for id '" + jobToSubmit.getOrigHarvestDefinitionID() + "', named '" + hName + "'"); } schedule = ph.getScheduleName(); hdComments = ph.getComments(); hdAudience = ph.getAudience(); } doOneCrawl(jobToSubmit, hName, hdComments, schedule, channel, hdAudience, metadata); log.info("Job #{} submitted", jobToSubmit.getJobID()); } catch (Throwable t) { String message = "Error while dispatching job " + jobToSubmit.getJobID() + ". Job status changed to FAILED"; log.warn(message, t); if (jobToSubmit != null) { jobToSubmit.setStatus(JobStatus.FAILED); jobToSubmit.appendHarvestErrors(message); jobToSubmit.appendHarvestErrorDetails(ExceptionUtils.getStackTrace(t)); jobDao.update(jobToSubmit); } } } } JobDispatcher(JMSConnection jmsConnection, HarvestDefinitionDAO hDao, JobDAO jobDao); void doOneCrawl(Job job, String origHarvestName, String origHarvestDesc, String origHarvestSchedule,
            HarvestChannel channel, String origHarvestAudience, List<MetadataEntry> metadata); }
JobDispatcher { protected void submitNextNewJob(HarvestChannel channel) { Job jobToSubmit = prepareNextJobForSubmission(channel); if (jobToSubmit == null) { log.trace("No {} jobs to be run at this time", channel.getName()); } else { log.debug("Submitting new {} job {}", channel.getName(), jobToSubmit.getJobID()); try { List<MetadataEntry> metadata = createMetadata(jobToSubmit); String hName = harvestDefinitionDAO.getHarvestName(jobToSubmit.getOrigHarvestDefinitionID()); String schedule = ""; String hdComments = ""; String hdAudience = ""; SparseFullHarvest fh = harvestDefinitionDAO.getSparseFullHarvest(hName); if (fh != null) { hdComments = fh.getComments(); } else { SparsePartialHarvest ph = harvestDefinitionDAO.getSparsePartialHarvest(hName); if (ph == null) { throw new ArgumentNotValid("No harvest definition found for id '" + jobToSubmit.getOrigHarvestDefinitionID() + "', named '" + hName + "'"); } schedule = ph.getScheduleName(); hdComments = ph.getComments(); hdAudience = ph.getAudience(); } doOneCrawl(jobToSubmit, hName, hdComments, schedule, channel, hdAudience, metadata); log.info("Job #{} submitted", jobToSubmit.getJobID()); } catch (Throwable t) { String message = "Error while dispatching job " + jobToSubmit.getJobID() + ". Job status changed to FAILED"; log.warn(message, t); if (jobToSubmit != null) { jobToSubmit.setStatus(JobStatus.FAILED); jobToSubmit.appendHarvestErrors(message); jobToSubmit.appendHarvestErrorDetails(ExceptionUtils.getStackTrace(t)); jobDao.update(jobToSubmit); } } } } JobDispatcher(JMSConnection jmsConnection, HarvestDefinitionDAO hDao, JobDAO jobDao); void doOneCrawl(Job job, String origHarvestName, String origHarvestDesc, String origHarvestSchedule,
            HarvestChannel channel, String origHarvestAudience, List<MetadataEntry> metadata);  }