@Test public void testExpectedNumberOfObjects100ObjectsHarvested() { DomainConfiguration dc = createDefaultDomainConfiguration(); addHistoryObject(dc, 100L, 1000L, StopReason.SIZE_LIMIT); assertEquals("The expected object size should be 10 because 100 objects were harvested", 250050L, dc.getExpectedNumberOfObjects(-1L, 5000000L)); }
public long getExpectedNumberOfObjects(long objectLimit, long byteLimit) { long prevresultfactor = Settings.getLong(HarvesterSettings.ERRORFACTOR_PERMITTED_PREVRESULT); HarvestInfo best = DomainHistory.getBestHarvestInfoExpectation(configName, this.domainhistory); log.trace("Getting expectation, using domain info '{}' for configuration '{}'", best, cfgToString(this)); long expectedObjectSize = getExpectedBytesPerObject(best); long maximum; if (objectLimit != Constants.HERITRIX_MAXOBJECTS_INFINITY || byteLimit != Constants.HERITRIX_MAXBYTES_INFINITY) { maximum = minObjectsBytesLimit(objectLimit, byteLimit, expectedObjectSize); } else if (maxObjects != Constants.HERITRIX_MAXOBJECTS_INFINITY || maxBytes != Constants.HERITRIX_MAXBYTES_INFINITY) { maximum = minObjectsBytesLimit(maxObjects, maxBytes, expectedObjectSize); } else { maximum = Settings.getLong(HarvesterSettings.MAX_DOMAIN_SIZE); } log.trace("Initial maximum: {}", maximum); long minimum; if (best != null) { minimum = best.getCountObjectRetrieved(); } else { minimum = NumberUtils.minInf(Constants.HERITRIX_MAXOBJECTS_INFINITY, maxObjects); } log.trace("Initial minimum: {}", minimum); long expectation; if (best != null) { if (best.getStopReason() == StopReason.DOWNLOAD_COMPLETE && maximum != -1) { expectation = minimum + ((maximum - minimum) / prevresultfactor); } else { expectation = minimum + ((maximum - minimum) / 2); } } else { expectation = NumberUtils.minInf(Settings.getLong(HarvesterSettings.MAX_DOMAIN_SIZE), maxObjects); } log.trace("Initial expectation: {}", expectation); if ((maxObjects > Constants.HERITRIX_MAXOBJECTS_INFINITY && maximum > maxObjects) || (maxBytes > Constants.HERITRIX_MAXBYTES_INFINITY && maximum > maxBytes / expectedObjectSize)) { log.trace("Using domain limits for {}", cfgToString(this)); maximum = minObjectsBytesLimit(maxObjects, maxBytes, expectedObjectSize); log.trace("New maximum: {}", maximum); } expectation = Math.min(expectation, maximum); log.trace("Expected number of objects for configuration '{}' is {}", cfgToString(this), expectation); return expectation; }
DomainConfiguration implements Named { public long getExpectedNumberOfObjects(long objectLimit, long byteLimit) { long prevresultfactor = Settings.getLong(HarvesterSettings.ERRORFACTOR_PERMITTED_PREVRESULT); HarvestInfo best = DomainHistory.getBestHarvestInfoExpectation(configName, this.domainhistory); log.trace("Getting expectation, using domain info '{}' for configuration '{}'", best, cfgToString(this)); long expectedObjectSize = getExpectedBytesPerObject(best); long maximum; if (objectLimit != Constants.HERITRIX_MAXOBJECTS_INFINITY || byteLimit != Constants.HERITRIX_MAXBYTES_INFINITY) { maximum = minObjectsBytesLimit(objectLimit, byteLimit, expectedObjectSize); } else if (maxObjects != Constants.HERITRIX_MAXOBJECTS_INFINITY || maxBytes != Constants.HERITRIX_MAXBYTES_INFINITY) { maximum = minObjectsBytesLimit(maxObjects, maxBytes, expectedObjectSize); } else { maximum = Settings.getLong(HarvesterSettings.MAX_DOMAIN_SIZE); } log.trace("Initial maximum: {}", maximum); long minimum; if (best != null) { minimum = best.getCountObjectRetrieved(); } else { minimum = NumberUtils.minInf(Constants.HERITRIX_MAXOBJECTS_INFINITY, maxObjects); } log.trace("Initial minimum: {}", minimum); long expectation; if (best != null) { if (best.getStopReason() == StopReason.DOWNLOAD_COMPLETE && maximum != -1) { expectation = minimum + ((maximum - minimum) / prevresultfactor); } else { expectation = minimum + ((maximum - minimum) / 2); } } else { expectation = NumberUtils.minInf(Settings.getLong(HarvesterSettings.MAX_DOMAIN_SIZE), maxObjects); } log.trace("Initial expectation: {}", expectation); if ((maxObjects > Constants.HERITRIX_MAXOBJECTS_INFINITY && maximum > maxObjects) || (maxBytes > Constants.HERITRIX_MAXBYTES_INFINITY && maximum > maxBytes / expectedObjectSize)) { log.trace("Using domain limits for {}", cfgToString(this)); maximum = minObjectsBytesLimit(maxObjects, maxBytes, expectedObjectSize); log.trace("New maximum: {}", maximum); } expectation = Math.min(expectation, maximum); log.trace("Expected number of objects for configuration '{}' is {}", cfgToString(this), expectation); return expectation; } }
DomainConfiguration implements Named { public long getExpectedNumberOfObjects(long objectLimit, long byteLimit) { long prevresultfactor = Settings.getLong(HarvesterSettings.ERRORFACTOR_PERMITTED_PREVRESULT); HarvestInfo best = DomainHistory.getBestHarvestInfoExpectation(configName, this.domainhistory); log.trace("Getting expectation, using domain info '{}' for configuration '{}'", best, cfgToString(this)); long expectedObjectSize = getExpectedBytesPerObject(best); long maximum; if (objectLimit != Constants.HERITRIX_MAXOBJECTS_INFINITY || byteLimit != Constants.HERITRIX_MAXBYTES_INFINITY) { maximum = minObjectsBytesLimit(objectLimit, byteLimit, expectedObjectSize); } else if (maxObjects != Constants.HERITRIX_MAXOBJECTS_INFINITY || maxBytes != Constants.HERITRIX_MAXBYTES_INFINITY) { maximum = minObjectsBytesLimit(maxObjects, maxBytes, expectedObjectSize); } else { maximum = Settings.getLong(HarvesterSettings.MAX_DOMAIN_SIZE); } log.trace("Initial maximum: {}", maximum); long minimum; if (best != null) { minimum = best.getCountObjectRetrieved(); } else { minimum = NumberUtils.minInf(Constants.HERITRIX_MAXOBJECTS_INFINITY, maxObjects); } log.trace("Initial minimum: {}", minimum); long expectation; if (best != null) { if (best.getStopReason() == StopReason.DOWNLOAD_COMPLETE && maximum != -1) { expectation = minimum + ((maximum - minimum) / prevresultfactor); } else { expectation = minimum + ((maximum - minimum) / 2); } } else { expectation = NumberUtils.minInf(Settings.getLong(HarvesterSettings.MAX_DOMAIN_SIZE), maxObjects); } log.trace("Initial expectation: {}", expectation); if ((maxObjects > Constants.HERITRIX_MAXOBJECTS_INFINITY && maximum > maxObjects) || (maxBytes > Constants.HERITRIX_MAXBYTES_INFINITY && maximum > maxBytes / expectedObjectSize)) { log.trace("Using domain limits for {}", cfgToString(this)); maximum = minObjectsBytesLimit(maxObjects, maxBytes, expectedObjectSize); log.trace("New maximum: {}", maximum); } expectation = Math.min(expectation, maximum); log.trace("Expected number of objects for configuration '{}' is {}", cfgToString(this), expectation); return expectation; } DomainConfiguration(String theConfigName, Domain domain, List<SeedList> seedlists, List<Password> passwords);  DomainConfiguration(String theConfigName, String domainName, DomainHistory history,
            List<String> crawlertraps, List<SeedList> seedlists, List<Password> passwords); }
DomainConfiguration implements Named { public long getExpectedNumberOfObjects(long objectLimit, long byteLimit) { long prevresultfactor = Settings.getLong(HarvesterSettings.ERRORFACTOR_PERMITTED_PREVRESULT); HarvestInfo best = DomainHistory.getBestHarvestInfoExpectation(configName, this.domainhistory); log.trace("Getting expectation, using domain info '{}' for configuration '{}'", best, cfgToString(this)); long expectedObjectSize = getExpectedBytesPerObject(best); long maximum; if (objectLimit != Constants.HERITRIX_MAXOBJECTS_INFINITY || byteLimit != Constants.HERITRIX_MAXBYTES_INFINITY) { maximum = minObjectsBytesLimit(objectLimit, byteLimit, expectedObjectSize); } else if (maxObjects != Constants.HERITRIX_MAXOBJECTS_INFINITY || maxBytes != Constants.HERITRIX_MAXBYTES_INFINITY) { maximum = minObjectsBytesLimit(maxObjects, maxBytes, expectedObjectSize); } else { maximum = Settings.getLong(HarvesterSettings.MAX_DOMAIN_SIZE); } log.trace("Initial maximum: {}", maximum); long minimum; if (best != null) { minimum = best.getCountObjectRetrieved(); } else { minimum = NumberUtils.minInf(Constants.HERITRIX_MAXOBJECTS_INFINITY, maxObjects); } log.trace("Initial minimum: {}", minimum); long expectation; if (best != null) { if (best.getStopReason() == StopReason.DOWNLOAD_COMPLETE && maximum != -1) { expectation = minimum + ((maximum - minimum) / prevresultfactor); } else { expectation = minimum + ((maximum - minimum) / 2); } } else { expectation = NumberUtils.minInf(Settings.getLong(HarvesterSettings.MAX_DOMAIN_SIZE), maxObjects); } log.trace("Initial expectation: {}", expectation); if ((maxObjects > Constants.HERITRIX_MAXOBJECTS_INFINITY && maximum > maxObjects) || (maxBytes > Constants.HERITRIX_MAXBYTES_INFINITY && maximum > maxBytes / expectedObjectSize)) { log.trace("Using domain limits for {}", cfgToString(this)); maximum = minObjectsBytesLimit(maxObjects, maxBytes, expectedObjectSize); log.trace("New maximum: {}", maximum); } expectation = Math.min(expectation, maximum); log.trace("Expected number of objects for configuration '{}' is {}", cfgToString(this), expectation); return expectation; } DomainConfiguration(String theConfigName, Domain domain, List<SeedList> seedlists, List<Password> passwords);  DomainConfiguration(String theConfigName, String domainName, DomainHistory history,
            List<String> crawlertraps, List<SeedList> seedlists, List<Password> passwords); static String cfgToString(DomainConfiguration cfg); void setOrderXmlName(String ordername); void setMaxObjects(long max); void setMaxRequestRate(int maxrate); void setMaxBytes(long maxBytes); String getName(); String getComments(); String getOrderXmlName(); long getMaxObjects(); int getMaxRequestRate(); long getMaxBytes(); String getDomainName(); Iterator<SeedList> getSeedLists(); void addSeedList(Domain domain, SeedList seedlist); void setSeedLists(Domain domain, List<SeedList> newSeedlists); Iterator<Password> getPasswords(); void addPassword(Domain domain, Password password); long getExpectedNumberOfObjects(long objectLimit, long byteLimit); long minObjectsBytesLimit(long objectLimit, long byteLimit, long expectedObjectSize); void setComments(String comments); void removePassword(String passwordName); boolean usesPassword(String passwordName); void setPasswords(Domain domain, List<Password> newPasswords); Long getID(); String toString(); void setCrawlertraps(List<String> someCrawlertraps); List<String> getCrawlertraps(); DomainHistory getDomainhistory(); void setDomainhistory(DomainHistory newDomainhistory); void setName(String configName); List<AttributeAndType> getAttributesAndTypes(); void setAttributesAndTypes(List<AttributeAndType> attributesAndTypes); }
DomainConfiguration implements Named { public long getExpectedNumberOfObjects(long objectLimit, long byteLimit) { long prevresultfactor = Settings.getLong(HarvesterSettings.ERRORFACTOR_PERMITTED_PREVRESULT); HarvestInfo best = DomainHistory.getBestHarvestInfoExpectation(configName, this.domainhistory); log.trace("Getting expectation, using domain info '{}' for configuration '{}'", best, cfgToString(this)); long expectedObjectSize = getExpectedBytesPerObject(best); long maximum; if (objectLimit != Constants.HERITRIX_MAXOBJECTS_INFINITY || byteLimit != Constants.HERITRIX_MAXBYTES_INFINITY) { maximum = minObjectsBytesLimit(objectLimit, byteLimit, expectedObjectSize); } else if (maxObjects != Constants.HERITRIX_MAXOBJECTS_INFINITY || maxBytes != Constants.HERITRIX_MAXBYTES_INFINITY) { maximum = minObjectsBytesLimit(maxObjects, maxBytes, expectedObjectSize); } else { maximum = Settings.getLong(HarvesterSettings.MAX_DOMAIN_SIZE); } log.trace("Initial maximum: {}", maximum); long minimum; if (best != null) { minimum = best.getCountObjectRetrieved(); } else { minimum = NumberUtils.minInf(Constants.HERITRIX_MAXOBJECTS_INFINITY, maxObjects); } log.trace("Initial minimum: {}", minimum); long expectation; if (best != null) { if (best.getStopReason() == StopReason.DOWNLOAD_COMPLETE && maximum != -1) { expectation = minimum + ((maximum - minimum) / prevresultfactor); } else { expectation = minimum + ((maximum - minimum) / 2); } } else { expectation = NumberUtils.minInf(Settings.getLong(HarvesterSettings.MAX_DOMAIN_SIZE), maxObjects); } log.trace("Initial expectation: {}", expectation); if ((maxObjects > Constants.HERITRIX_MAXOBJECTS_INFINITY && maximum > maxObjects) || (maxBytes > Constants.HERITRIX_MAXBYTES_INFINITY && maximum > maxBytes / expectedObjectSize)) { log.trace("Using domain limits for {}", cfgToString(this)); maximum = minObjectsBytesLimit(maxObjects, maxBytes, expectedObjectSize); log.trace("New maximum: {}", maximum); } expectation = Math.min(expectation, maximum); log.trace("Expected number of objects for configuration '{}' is {}", cfgToString(this), expectation); return expectation; } DomainConfiguration(String theConfigName, Domain domain, List<SeedList> seedlists, List<Password> passwords);  DomainConfiguration(String theConfigName, String domainName, DomainHistory history,
            List<String> crawlertraps, List<SeedList> seedlists, List<Password> passwords); static String cfgToString(DomainConfiguration cfg); void setOrderXmlName(String ordername); void setMaxObjects(long max); void setMaxRequestRate(int maxrate); void setMaxBytes(long maxBytes); String getName(); String getComments(); String getOrderXmlName(); long getMaxObjects(); int getMaxRequestRate(); long getMaxBytes(); String getDomainName(); Iterator<SeedList> getSeedLists(); void addSeedList(Domain domain, SeedList seedlist); void setSeedLists(Domain domain, List<SeedList> newSeedlists); Iterator<Password> getPasswords(); void addPassword(Domain domain, Password password); long getExpectedNumberOfObjects(long objectLimit, long byteLimit); long minObjectsBytesLimit(long objectLimit, long byteLimit, long expectedObjectSize); void setComments(String comments); void removePassword(String passwordName); boolean usesPassword(String passwordName); void setPasswords(Domain domain, List<Password> newPasswords); Long getID(); String toString(); void setCrawlertraps(List<String> someCrawlertraps); List<String> getCrawlertraps(); DomainHistory getDomainhistory(); void setDomainhistory(DomainHistory newDomainhistory); void setName(String configName); List<AttributeAndType> getAttributesAndTypes(); void setAttributesAndTypes(List<AttributeAndType> attributesAndTypes);  }