@Test public void testAddConfiguration() { Job job = createDefaultJob(); DomainConfiguration anotherConfig = DomainConfigurationTest.createDefaultDomainConfiguration("kaarefc.dk"); job.addConfiguration(anotherConfig); assertEquals(2, job.getDomainConfigurationMap().size()); }
public void addConfiguration(DomainConfiguration cfg) { ArgumentNotValid.checkNotNull(cfg, "cfg"); if (domainConfigurationMap.containsKey(cfg.getDomainName())) { throw new ArgumentNotValid("Job already has a configuration for Domain " + cfg.getDomainName()); } if (log.isTraceEnabled()) { log.trace("Adding configuration '{}' to job '{}'", cfg, cfg.getName()); } if (!underConstruction) { final String msg = "Cannot modify job " + this + " as it is no longer under construction"; log.debug(msg); throw new IllegalState(msg); } if (!cfg.getOrderXmlName().equals(getOrderXMLName())) { throw new ArgumentNotValid("Job requires the orderxml file:'" + getOrderXMLName() + "' not:'" + cfg.getOrderXmlName() + "' used by the configuration:'" + cfg.getName()); } domainConfigurationMap.put(cfg.getDomainName(), cfg.getName()); for (Iterator<SeedList> itt = cfg.getSeedLists(); itt.hasNext();) { SeedList seed = itt.next(); List<String> seeds = seed.getSeeds(); for (String seedUrl : seeds) { seedListSet.add(seedUrl); try { String seedASCII = seedUrl; if (seedUrl.contains(":") || seedUrl.contains("/")) { String normalizedUrl = seedUrl; if (!normalizedUrl.matches("^[a-zA-Z]+:.*")) { normalizedUrl = "http: } URL url = new URL(normalizedUrl); String domainName = url.getHost(); String domainNameASCII = IDNA.toASCII(domainName); if (!domainName.equals(domainNameASCII)) { seedASCII = seedUrl.replaceFirst(Pattern.quote(domainName), domainNameASCII); } } else { seedASCII = IDNA.toASCII(seedUrl); } if (!seedASCII.equals(seedUrl)) { log.trace("Converted {} to {}", seedUrl, seedASCII); seedListSet.add(seedASCII); } } catch (IDNAException e) { log.trace("Cannot convert seed {} to ASCII", seedUrl, e); } catch (MalformedURLException e) { log.trace("Cannot convert seed {} to ASCII", seedUrl, e); } } } orderXMLdoc.editOrderXMLAddPerDomainCrawlerTraps(cfg); long expectation = cfg.getExpectedNumberOfObjects(forceMaxObjectsPerDomain, forceMaxBytesPerDomain); maxCountObjects = Math.max(expectation, maxCountObjects); minCountObjects = Math.min(expectation, minCountObjects); totalCountObjects += expectation; configsChanged = true; assert (maxCountObjects >= minCountObjects) : "basic invariant"; }
Job implements Serializable, JobInfo { public void addConfiguration(DomainConfiguration cfg) { ArgumentNotValid.checkNotNull(cfg, "cfg"); if (domainConfigurationMap.containsKey(cfg.getDomainName())) { throw new ArgumentNotValid("Job already has a configuration for Domain " + cfg.getDomainName()); } if (log.isTraceEnabled()) { log.trace("Adding configuration '{}' to job '{}'", cfg, cfg.getName()); } if (!underConstruction) { final String msg = "Cannot modify job " + this + " as it is no longer under construction"; log.debug(msg); throw new IllegalState(msg); } if (!cfg.getOrderXmlName().equals(getOrderXMLName())) { throw new ArgumentNotValid("Job requires the orderxml file:'" + getOrderXMLName() + "' not:'" + cfg.getOrderXmlName() + "' used by the configuration:'" + cfg.getName()); } domainConfigurationMap.put(cfg.getDomainName(), cfg.getName()); for (Iterator<SeedList> itt = cfg.getSeedLists(); itt.hasNext();) { SeedList seed = itt.next(); List<String> seeds = seed.getSeeds(); for (String seedUrl : seeds) { seedListSet.add(seedUrl); try { String seedASCII = seedUrl; if (seedUrl.contains(":") || seedUrl.contains("/")) { String normalizedUrl = seedUrl; if (!normalizedUrl.matches("^[a-zA-Z]+:.*")) { normalizedUrl = "http: } URL url = new URL(normalizedUrl); String domainName = url.getHost(); String domainNameASCII = IDNA.toASCII(domainName); if (!domainName.equals(domainNameASCII)) { seedASCII = seedUrl.replaceFirst(Pattern.quote(domainName), domainNameASCII); } } else { seedASCII = IDNA.toASCII(seedUrl); } if (!seedASCII.equals(seedUrl)) { log.trace("Converted {} to {}", seedUrl, seedASCII); seedListSet.add(seedASCII); } } catch (IDNAException e) { log.trace("Cannot convert seed {} to ASCII", seedUrl, e); } catch (MalformedURLException e) { log.trace("Cannot convert seed {} to ASCII", seedUrl, e); } } } orderXMLdoc.editOrderXMLAddPerDomainCrawlerTraps(cfg); long expectation = cfg.getExpectedNumberOfObjects(forceMaxObjectsPerDomain, forceMaxBytesPerDomain); maxCountObjects = Math.max(expectation, maxCountObjects); minCountObjects = Math.min(expectation, minCountObjects); totalCountObjects += expectation; configsChanged = true; assert (maxCountObjects >= minCountObjects) : "basic invariant"; } }
Job implements Serializable, JobInfo { public void addConfiguration(DomainConfiguration cfg) { ArgumentNotValid.checkNotNull(cfg, "cfg"); if (domainConfigurationMap.containsKey(cfg.getDomainName())) { throw new ArgumentNotValid("Job already has a configuration for Domain " + cfg.getDomainName()); } if (log.isTraceEnabled()) { log.trace("Adding configuration '{}' to job '{}'", cfg, cfg.getName()); } if (!underConstruction) { final String msg = "Cannot modify job " + this + " as it is no longer under construction"; log.debug(msg); throw new IllegalState(msg); } if (!cfg.getOrderXmlName().equals(getOrderXMLName())) { throw new ArgumentNotValid("Job requires the orderxml file:'" + getOrderXMLName() + "' not:'" + cfg.getOrderXmlName() + "' used by the configuration:'" + cfg.getName()); } domainConfigurationMap.put(cfg.getDomainName(), cfg.getName()); for (Iterator<SeedList> itt = cfg.getSeedLists(); itt.hasNext();) { SeedList seed = itt.next(); List<String> seeds = seed.getSeeds(); for (String seedUrl : seeds) { seedListSet.add(seedUrl); try { String seedASCII = seedUrl; if (seedUrl.contains(":") || seedUrl.contains("/")) { String normalizedUrl = seedUrl; if (!normalizedUrl.matches("^[a-zA-Z]+:.*")) { normalizedUrl = "http: } URL url = new URL(normalizedUrl); String domainName = url.getHost(); String domainNameASCII = IDNA.toASCII(domainName); if (!domainName.equals(domainNameASCII)) { seedASCII = seedUrl.replaceFirst(Pattern.quote(domainName), domainNameASCII); } } else { seedASCII = IDNA.toASCII(seedUrl); } if (!seedASCII.equals(seedUrl)) { log.trace("Converted {} to {}", seedUrl, seedASCII); seedListSet.add(seedASCII); } } catch (IDNAException e) { log.trace("Cannot convert seed {} to ASCII", seedUrl, e); } catch (MalformedURLException e) { log.trace("Cannot convert seed {} to ASCII", seedUrl, e); } } } orderXMLdoc.editOrderXMLAddPerDomainCrawlerTraps(cfg); long expectation = cfg.getExpectedNumberOfObjects(forceMaxObjectsPerDomain, forceMaxBytesPerDomain); maxCountObjects = Math.max(expectation, maxCountObjects); minCountObjects = Math.min(expectation, minCountObjects); totalCountObjects += expectation; configsChanged = true; assert (maxCountObjects >= minCountObjects) : "basic invariant"; } protected  Job();  Job(Long harvestID, DomainConfiguration cfg, HeritrixTemplate orderXMLdoc, HarvestChannel channel,
            long forceMaxObjectsPerDomain,
            long forceMaxBytesPerDomain, long forceMaxJobRunningTime, int harvestNum);   Job(Long harvestID, Map<String, String> configurations, String channel, boolean snapshot,
            long forceMaxObjectsPerDomain, long forceMaxBytesPerDomain, long forceMaxJobRunningTime, JobStatus status,
            String orderXMLname, HeritrixTemplate orderXMLdoc, String seedlist, int harvestNum, Long continuationOf); }
Job implements Serializable, JobInfo { public void addConfiguration(DomainConfiguration cfg) { ArgumentNotValid.checkNotNull(cfg, "cfg"); if (domainConfigurationMap.containsKey(cfg.getDomainName())) { throw new ArgumentNotValid("Job already has a configuration for Domain " + cfg.getDomainName()); } if (log.isTraceEnabled()) { log.trace("Adding configuration '{}' to job '{}'", cfg, cfg.getName()); } if (!underConstruction) { final String msg = "Cannot modify job " + this + " as it is no longer under construction"; log.debug(msg); throw new IllegalState(msg); } if (!cfg.getOrderXmlName().equals(getOrderXMLName())) { throw new ArgumentNotValid("Job requires the orderxml file:'" + getOrderXMLName() + "' not:'" + cfg.getOrderXmlName() + "' used by the configuration:'" + cfg.getName()); } domainConfigurationMap.put(cfg.getDomainName(), cfg.getName()); for (Iterator<SeedList> itt = cfg.getSeedLists(); itt.hasNext();) { SeedList seed = itt.next(); List<String> seeds = seed.getSeeds(); for (String seedUrl : seeds) { seedListSet.add(seedUrl); try { String seedASCII = seedUrl; if (seedUrl.contains(":") || seedUrl.contains("/")) { String normalizedUrl = seedUrl; if (!normalizedUrl.matches("^[a-zA-Z]+:.*")) { normalizedUrl = "http: } URL url = new URL(normalizedUrl); String domainName = url.getHost(); String domainNameASCII = IDNA.toASCII(domainName); if (!domainName.equals(domainNameASCII)) { seedASCII = seedUrl.replaceFirst(Pattern.quote(domainName), domainNameASCII); } } else { seedASCII = IDNA.toASCII(seedUrl); } if (!seedASCII.equals(seedUrl)) { log.trace("Converted {} to {}", seedUrl, seedASCII); seedListSet.add(seedASCII); } } catch (IDNAException e) { log.trace("Cannot convert seed {} to ASCII", seedUrl, e); } catch (MalformedURLException e) { log.trace("Cannot convert seed {} to ASCII", seedUrl, e); } } } orderXMLdoc.editOrderXMLAddPerDomainCrawlerTraps(cfg); long expectation = cfg.getExpectedNumberOfObjects(forceMaxObjectsPerDomain, forceMaxBytesPerDomain); maxCountObjects = Math.max(expectation, maxCountObjects); minCountObjects = Math.min(expectation, minCountObjects); totalCountObjects += expectation; configsChanged = true; assert (maxCountObjects >= minCountObjects) : "basic invariant"; } protected  Job();  Job(Long harvestID, DomainConfiguration cfg, HeritrixTemplate orderXMLdoc, HarvestChannel channel,
            long forceMaxObjectsPerDomain,
            long forceMaxBytesPerDomain, long forceMaxJobRunningTime, int harvestNum);   Job(Long harvestID, Map<String, String> configurations, String channel, boolean snapshot,
            long forceMaxObjectsPerDomain, long forceMaxBytesPerDomain, long forceMaxJobRunningTime, JobStatus status,
            String orderXMLname, HeritrixTemplate orderXMLdoc, String seedlist, int harvestNum, Long continuationOf); void setAttributes(List<AttributeAndType> attributesAndTypes); void addConfiguration(DomainConfiguration cfg); String getOrderXMLName(); Date getActualStop(); Date getActualStart(); Date getSubmittedDate(); Date getCreationDate(); File[] getSettingsXMLfiles(); Long getOrigHarvestDefinitionID(); Long getJobID(); void setJobID(Long id); int getCountDomains(); void setActualStart(Date actualStart); void setActualStop(Date actualStop); void setOrderXMLDoc(HeritrixTemplate doc); HeritrixTemplate getOrderXMLdoc(); void setSeedList(String seedList); String getSeedListAsString(); JobStatus getStatus(); void setStatus(JobStatus newStatus); Map<String, String> getDomainConfigurationMap(); long getMaxObjectsPerDomain(); long getMaxBytesPerDomain(); void setHarvestChannel(HarvestChannel harvestChannel); String getChannel(); void setChannel(String channel); boolean isSnapshot(); void setSnapshot(boolean isSnapshot); @Override String toString(); long getForceMaxObjectsPerDomain(); long getMaxJobRunningTime(); int getHarvestNum(); void setHarvestNum(int harvestNum); String getHarvestErrors(); void appendHarvestErrors(String harvestErrors); String getHarvestErrorDetails(); void appendHarvestErrorDetails(String harvestErrorDetails); String getUploadErrors(); void appendUploadErrors(String uploadErrors); String getUploadErrorDetails(); void appendUploadErrorDetails(String uploadErrorDetails); Long getResubmittedAsJob(); void setSubmittedDate(Date submittedDate); void setCreationDate(Date creationDate); void setResubmittedAsJob(Long resubmittedAsJob); Long getContinuationOf(); @Override String getHarvestFilenamePrefix(); void setHarvestFilenamePrefix(String prefix); long getForceMaxBytesPerDomain(); boolean isConfigurationSetsObjectLimit(); boolean isConfigurationSetsByteLimit(); long getMinCountObjects(); long getMaxCountObjects(); long getTotalCountObjects(); String getHarvestAudience(); void setHarvestAudience(String theAudience); List<String> getSortedSeedList(); }
Job implements Serializable, JobInfo { public void addConfiguration(DomainConfiguration cfg) { ArgumentNotValid.checkNotNull(cfg, "cfg"); if (domainConfigurationMap.containsKey(cfg.getDomainName())) { throw new ArgumentNotValid("Job already has a configuration for Domain " + cfg.getDomainName()); } if (log.isTraceEnabled()) { log.trace("Adding configuration '{}' to job '{}'", cfg, cfg.getName()); } if (!underConstruction) { final String msg = "Cannot modify job " + this + " as it is no longer under construction"; log.debug(msg); throw new IllegalState(msg); } if (!cfg.getOrderXmlName().equals(getOrderXMLName())) { throw new ArgumentNotValid("Job requires the orderxml file:'" + getOrderXMLName() + "' not:'" + cfg.getOrderXmlName() + "' used by the configuration:'" + cfg.getName()); } domainConfigurationMap.put(cfg.getDomainName(), cfg.getName()); for (Iterator<SeedList> itt = cfg.getSeedLists(); itt.hasNext();) { SeedList seed = itt.next(); List<String> seeds = seed.getSeeds(); for (String seedUrl : seeds) { seedListSet.add(seedUrl); try { String seedASCII = seedUrl; if (seedUrl.contains(":") || seedUrl.contains("/")) { String normalizedUrl = seedUrl; if (!normalizedUrl.matches("^[a-zA-Z]+:.*")) { normalizedUrl = "http: } URL url = new URL(normalizedUrl); String domainName = url.getHost(); String domainNameASCII = IDNA.toASCII(domainName); if (!domainName.equals(domainNameASCII)) { seedASCII = seedUrl.replaceFirst(Pattern.quote(domainName), domainNameASCII); } } else { seedASCII = IDNA.toASCII(seedUrl); } if (!seedASCII.equals(seedUrl)) { log.trace("Converted {} to {}", seedUrl, seedASCII); seedListSet.add(seedASCII); } } catch (IDNAException e) { log.trace("Cannot convert seed {} to ASCII", seedUrl, e); } catch (MalformedURLException e) { log.trace("Cannot convert seed {} to ASCII", seedUrl, e); } } } orderXMLdoc.editOrderXMLAddPerDomainCrawlerTraps(cfg); long expectation = cfg.getExpectedNumberOfObjects(forceMaxObjectsPerDomain, forceMaxBytesPerDomain); maxCountObjects = Math.max(expectation, maxCountObjects); minCountObjects = Math.min(expectation, minCountObjects); totalCountObjects += expectation; configsChanged = true; assert (maxCountObjects >= minCountObjects) : "basic invariant"; } protected  Job();  Job(Long harvestID, DomainConfiguration cfg, HeritrixTemplate orderXMLdoc, HarvestChannel channel,
            long forceMaxObjectsPerDomain,
            long forceMaxBytesPerDomain, long forceMaxJobRunningTime, int harvestNum);   Job(Long harvestID, Map<String, String> configurations, String channel, boolean snapshot,
            long forceMaxObjectsPerDomain, long forceMaxBytesPerDomain, long forceMaxJobRunningTime, JobStatus status,
            String orderXMLname, HeritrixTemplate orderXMLdoc, String seedlist, int harvestNum, Long continuationOf); void setAttributes(List<AttributeAndType> attributesAndTypes); void addConfiguration(DomainConfiguration cfg); String getOrderXMLName(); Date getActualStop(); Date getActualStart(); Date getSubmittedDate(); Date getCreationDate(); File[] getSettingsXMLfiles(); Long getOrigHarvestDefinitionID(); Long getJobID(); void setJobID(Long id); int getCountDomains(); void setActualStart(Date actualStart); void setActualStop(Date actualStop); void setOrderXMLDoc(HeritrixTemplate doc); HeritrixTemplate getOrderXMLdoc(); void setSeedList(String seedList); String getSeedListAsString(); JobStatus getStatus(); void setStatus(JobStatus newStatus); Map<String, String> getDomainConfigurationMap(); long getMaxObjectsPerDomain(); long getMaxBytesPerDomain(); void setHarvestChannel(HarvestChannel harvestChannel); String getChannel(); void setChannel(String channel); boolean isSnapshot(); void setSnapshot(boolean isSnapshot); @Override String toString(); long getForceMaxObjectsPerDomain(); long getMaxJobRunningTime(); int getHarvestNum(); void setHarvestNum(int harvestNum); String getHarvestErrors(); void appendHarvestErrors(String harvestErrors); String getHarvestErrorDetails(); void appendHarvestErrorDetails(String harvestErrorDetails); String getUploadErrors(); void appendUploadErrors(String uploadErrors); String getUploadErrorDetails(); void appendUploadErrorDetails(String uploadErrorDetails); Long getResubmittedAsJob(); void setSubmittedDate(Date submittedDate); void setCreationDate(Date creationDate); void setResubmittedAsJob(Long resubmittedAsJob); Long getContinuationOf(); @Override String getHarvestFilenamePrefix(); void setHarvestFilenamePrefix(String prefix); long getForceMaxBytesPerDomain(); boolean isConfigurationSetsObjectLimit(); boolean isConfigurationSetsByteLimit(); long getMinCountObjects(); long getMaxCountObjects(); long getTotalCountObjects(); String getHarvestAudience(); void setHarvestAudience(String theAudience); List<String> getSortedSeedList();  }