@Test public void testApplyAnnotations() throws URIException, URISyntaxException { innerTestApplyAnnotations("http: innerTestApplyAnnotations("http: innerTestApplyAnnotations("http: innerTestApplyAnnotations("http: }
public void applyAnnotations(URI uri, SolrInputDocument solr) throws URISyntaxException, URIException { LOG.debug("Updating collections for " + solr.getField(SolrFields.SOLR_URL)); Set<String> crawl_dates = new HashSet<String>(); crawl_dates.add((String) solr.getField(SolrFields.CRAWL_DATE) .getValue()); if (solr.getField(SolrFields.CRAWL_DATES) != null) { for (Object d : solr.getField(SolrFields.CRAWL_DATES).getValues()) { @SuppressWarnings("unchecked") HashMap<String, String> dhm = (HashMap<String, String>) d; crawl_dates.addAll(dhm.values()); } } String normd = uri.toString(); LOG.debug("Comparing with " + normd); if (this.annotations.getCollections().containsKey("resource")) { if (this.annotations.getCollections().get("resource").keySet() .contains(normd)) { LOG.debug("Applying resource-level annotations..."); updateCollections(this.annotations.getCollections() .get("resource").get(normd), solr, crawl_dates); } } if (this.annotations.getCollections().containsKey("root")) { if (this.annotations.getCollections().get("root").keySet() .contains(normd)) { LOG.debug("Applying root-level annotations..."); updateCollections(this.annotations.getCollections().get("root") .get(normd), solr, crawl_dates); } } if (this.annotations.getCollections().containsKey("subdomains")) { String host; String domain = uri.getHost().replaceAll("^www\\.", ""); HashMap<String, UriCollection> subdomains = this.annotations .getCollections().get("subdomains"); for (String key : subdomains.keySet()) { LOG.debug("Applying subdomain annotations for: " + key); host = URI.create(key).getHost(); if (host == null) { host = key; } if (host.equals(domain) || host.endsWith("." + domain)) { updateCollections(subdomains.get(key), solr, crawl_dates); } } } if (this.annotations.getCollections() .containsKey("source_file_matches")) { Pattern pattern; Matcher matcher; String sourceFile = (String) solr.getField(SolrFields.SOURCE_FILE) .getValue(); HashMap<String, UriCollection> sourceFileMatches = this.annotations .getCollections().get("source_file_matches"); for (String key : sourceFileMatches.keySet()) { LOG.debug( "Applying source_file_matches annotations for: " + key); pattern = Pattern.compile(key); matcher = pattern.matcher(sourceFile); while (matcher.find()) { updateCollections(sourceFileMatches.get(key), solr, crawl_dates); } } } if (this.openAccessSurts != null) { LOG.debug("Attempting to apply " + this.openAccessSurts.size() + " OA Surts to " + uri); String surt = SurtPrefixSet .getCandidateSurt( UsableURIFactory.getInstance(uri.toString())); if (this.openAccessSurts.containsPrefixOf(surt)) { setUpdateField(solr, SolrFields.ACCESS_TERMS, "OA"); } else { setUpdateField(solr, SolrFields.ACCESS_TERMS, "RRO"); } } }
Annotator { public void applyAnnotations(URI uri, SolrInputDocument solr) throws URISyntaxException, URIException { LOG.debug("Updating collections for " + solr.getField(SolrFields.SOLR_URL)); Set<String> crawl_dates = new HashSet<String>(); crawl_dates.add((String) solr.getField(SolrFields.CRAWL_DATE) .getValue()); if (solr.getField(SolrFields.CRAWL_DATES) != null) { for (Object d : solr.getField(SolrFields.CRAWL_DATES).getValues()) { @SuppressWarnings("unchecked") HashMap<String, String> dhm = (HashMap<String, String>) d; crawl_dates.addAll(dhm.values()); } } String normd = uri.toString(); LOG.debug("Comparing with " + normd); if (this.annotations.getCollections().containsKey("resource")) { if (this.annotations.getCollections().get("resource").keySet() .contains(normd)) { LOG.debug("Applying resource-level annotations..."); updateCollections(this.annotations.getCollections() .get("resource").get(normd), solr, crawl_dates); } } if (this.annotations.getCollections().containsKey("root")) { if (this.annotations.getCollections().get("root").keySet() .contains(normd)) { LOG.debug("Applying root-level annotations..."); updateCollections(this.annotations.getCollections().get("root") .get(normd), solr, crawl_dates); } } if (this.annotations.getCollections().containsKey("subdomains")) { String host; String domain = uri.getHost().replaceAll("^www\\.", ""); HashMap<String, UriCollection> subdomains = this.annotations .getCollections().get("subdomains"); for (String key : subdomains.keySet()) { LOG.debug("Applying subdomain annotations for: " + key); host = URI.create(key).getHost(); if (host == null) { host = key; } if (host.equals(domain) || host.endsWith("." + domain)) { updateCollections(subdomains.get(key), solr, crawl_dates); } } } if (this.annotations.getCollections() .containsKey("source_file_matches")) { Pattern pattern; Matcher matcher; String sourceFile = (String) solr.getField(SolrFields.SOURCE_FILE) .getValue(); HashMap<String, UriCollection> sourceFileMatches = this.annotations .getCollections().get("source_file_matches"); for (String key : sourceFileMatches.keySet()) { LOG.debug( "Applying source_file_matches annotations for: " + key); pattern = Pattern.compile(key); matcher = pattern.matcher(sourceFile); while (matcher.find()) { updateCollections(sourceFileMatches.get(key), solr, crawl_dates); } } } if (this.openAccessSurts != null) { LOG.debug("Attempting to apply " + this.openAccessSurts.size() + " OA Surts to " + uri); String surt = SurtPrefixSet .getCandidateSurt( UsableURIFactory.getInstance(uri.toString())); if (this.openAccessSurts.containsPrefixOf(surt)) { setUpdateField(solr, SolrFields.ACCESS_TERMS, "OA"); } else { setUpdateField(solr, SolrFields.ACCESS_TERMS, "RRO"); } } } }
Annotator { public void applyAnnotations(URI uri, SolrInputDocument solr) throws URISyntaxException, URIException { LOG.debug("Updating collections for " + solr.getField(SolrFields.SOLR_URL)); Set<String> crawl_dates = new HashSet<String>(); crawl_dates.add((String) solr.getField(SolrFields.CRAWL_DATE) .getValue()); if (solr.getField(SolrFields.CRAWL_DATES) != null) { for (Object d : solr.getField(SolrFields.CRAWL_DATES).getValues()) { @SuppressWarnings("unchecked") HashMap<String, String> dhm = (HashMap<String, String>) d; crawl_dates.addAll(dhm.values()); } } String normd = uri.toString(); LOG.debug("Comparing with " + normd); if (this.annotations.getCollections().containsKey("resource")) { if (this.annotations.getCollections().get("resource").keySet() .contains(normd)) { LOG.debug("Applying resource-level annotations..."); updateCollections(this.annotations.getCollections() .get("resource").get(normd), solr, crawl_dates); } } if (this.annotations.getCollections().containsKey("root")) { if (this.annotations.getCollections().get("root").keySet() .contains(normd)) { LOG.debug("Applying root-level annotations..."); updateCollections(this.annotations.getCollections().get("root") .get(normd), solr, crawl_dates); } } if (this.annotations.getCollections().containsKey("subdomains")) { String host; String domain = uri.getHost().replaceAll("^www\\.", ""); HashMap<String, UriCollection> subdomains = this.annotations .getCollections().get("subdomains"); for (String key : subdomains.keySet()) { LOG.debug("Applying subdomain annotations for: " + key); host = URI.create(key).getHost(); if (host == null) { host = key; } if (host.equals(domain) || host.endsWith("." + domain)) { updateCollections(subdomains.get(key), solr, crawl_dates); } } } if (this.annotations.getCollections() .containsKey("source_file_matches")) { Pattern pattern; Matcher matcher; String sourceFile = (String) solr.getField(SolrFields.SOURCE_FILE) .getValue(); HashMap<String, UriCollection> sourceFileMatches = this.annotations .getCollections().get("source_file_matches"); for (String key : sourceFileMatches.keySet()) { LOG.debug( "Applying source_file_matches annotations for: " + key); pattern = Pattern.compile(key); matcher = pattern.matcher(sourceFile); while (matcher.find()) { updateCollections(sourceFileMatches.get(key), solr, crawl_dates); } } } if (this.openAccessSurts != null) { LOG.debug("Attempting to apply " + this.openAccessSurts.size() + " OA Surts to " + uri); String surt = SurtPrefixSet .getCandidateSurt( UsableURIFactory.getInstance(uri.toString())); if (this.openAccessSurts.containsPrefixOf(surt)) { setUpdateField(solr, SolrFields.ACCESS_TERMS, "OA"); } else { setUpdateField(solr, SolrFields.ACCESS_TERMS, "RRO"); } } } Annotator(Annotations annotations, SurtPrefixSet oaSurts); }
Annotator { public void applyAnnotations(URI uri, SolrInputDocument solr) throws URISyntaxException, URIException { LOG.debug("Updating collections for " + solr.getField(SolrFields.SOLR_URL)); Set<String> crawl_dates = new HashSet<String>(); crawl_dates.add((String) solr.getField(SolrFields.CRAWL_DATE) .getValue()); if (solr.getField(SolrFields.CRAWL_DATES) != null) { for (Object d : solr.getField(SolrFields.CRAWL_DATES).getValues()) { @SuppressWarnings("unchecked") HashMap<String, String> dhm = (HashMap<String, String>) d; crawl_dates.addAll(dhm.values()); } } String normd = uri.toString(); LOG.debug("Comparing with " + normd); if (this.annotations.getCollections().containsKey("resource")) { if (this.annotations.getCollections().get("resource").keySet() .contains(normd)) { LOG.debug("Applying resource-level annotations..."); updateCollections(this.annotations.getCollections() .get("resource").get(normd), solr, crawl_dates); } } if (this.annotations.getCollections().containsKey("root")) { if (this.annotations.getCollections().get("root").keySet() .contains(normd)) { LOG.debug("Applying root-level annotations..."); updateCollections(this.annotations.getCollections().get("root") .get(normd), solr, crawl_dates); } } if (this.annotations.getCollections().containsKey("subdomains")) { String host; String domain = uri.getHost().replaceAll("^www\\.", ""); HashMap<String, UriCollection> subdomains = this.annotations .getCollections().get("subdomains"); for (String key : subdomains.keySet()) { LOG.debug("Applying subdomain annotations for: " + key); host = URI.create(key).getHost(); if (host == null) { host = key; } if (host.equals(domain) || host.endsWith("." + domain)) { updateCollections(subdomains.get(key), solr, crawl_dates); } } } if (this.annotations.getCollections() .containsKey("source_file_matches")) { Pattern pattern; Matcher matcher; String sourceFile = (String) solr.getField(SolrFields.SOURCE_FILE) .getValue(); HashMap<String, UriCollection> sourceFileMatches = this.annotations .getCollections().get("source_file_matches"); for (String key : sourceFileMatches.keySet()) { LOG.debug( "Applying source_file_matches annotations for: " + key); pattern = Pattern.compile(key); matcher = pattern.matcher(sourceFile); while (matcher.find()) { updateCollections(sourceFileMatches.get(key), solr, crawl_dates); } } } if (this.openAccessSurts != null) { LOG.debug("Attempting to apply " + this.openAccessSurts.size() + " OA Surts to " + uri); String surt = SurtPrefixSet .getCandidateSurt( UsableURIFactory.getInstance(uri.toString())); if (this.openAccessSurts.containsPrefixOf(surt)) { setUpdateField(solr, SolrFields.ACCESS_TERMS, "OA"); } else { setUpdateField(solr, SolrFields.ACCESS_TERMS, "RRO"); } } } Annotator(Annotations annotations, SurtPrefixSet oaSurts); static Annotator annotationsFromAct(); static SurtPrefixSet loadSurtPrefix(String surtPrefixFile); void applyAnnotations(URI uri, SolrInputDocument solr); static void main(String[] args); }
Annotator { public void applyAnnotations(URI uri, SolrInputDocument solr) throws URISyntaxException, URIException { LOG.debug("Updating collections for " + solr.getField(SolrFields.SOLR_URL)); Set<String> crawl_dates = new HashSet<String>(); crawl_dates.add((String) solr.getField(SolrFields.CRAWL_DATE) .getValue()); if (solr.getField(SolrFields.CRAWL_DATES) != null) { for (Object d : solr.getField(SolrFields.CRAWL_DATES).getValues()) { @SuppressWarnings("unchecked") HashMap<String, String> dhm = (HashMap<String, String>) d; crawl_dates.addAll(dhm.values()); } } String normd = uri.toString(); LOG.debug("Comparing with " + normd); if (this.annotations.getCollections().containsKey("resource")) { if (this.annotations.getCollections().get("resource").keySet() .contains(normd)) { LOG.debug("Applying resource-level annotations..."); updateCollections(this.annotations.getCollections() .get("resource").get(normd), solr, crawl_dates); } } if (this.annotations.getCollections().containsKey("root")) { if (this.annotations.getCollections().get("root").keySet() .contains(normd)) { LOG.debug("Applying root-level annotations..."); updateCollections(this.annotations.getCollections().get("root") .get(normd), solr, crawl_dates); } } if (this.annotations.getCollections().containsKey("subdomains")) { String host; String domain = uri.getHost().replaceAll("^www\\.", ""); HashMap<String, UriCollection> subdomains = this.annotations .getCollections().get("subdomains"); for (String key : subdomains.keySet()) { LOG.debug("Applying subdomain annotations for: " + key); host = URI.create(key).getHost(); if (host == null) { host = key; } if (host.equals(domain) || host.endsWith("." + domain)) { updateCollections(subdomains.get(key), solr, crawl_dates); } } } if (this.annotations.getCollections() .containsKey("source_file_matches")) { Pattern pattern; Matcher matcher; String sourceFile = (String) solr.getField(SolrFields.SOURCE_FILE) .getValue(); HashMap<String, UriCollection> sourceFileMatches = this.annotations .getCollections().get("source_file_matches"); for (String key : sourceFileMatches.keySet()) { LOG.debug( "Applying source_file_matches annotations for: " + key); pattern = Pattern.compile(key); matcher = pattern.matcher(sourceFile); while (matcher.find()) { updateCollections(sourceFileMatches.get(key), solr, crawl_dates); } } } if (this.openAccessSurts != null) { LOG.debug("Attempting to apply " + this.openAccessSurts.size() + " OA Surts to " + uri); String surt = SurtPrefixSet .getCandidateSurt( UsableURIFactory.getInstance(uri.toString())); if (this.openAccessSurts.containsPrefixOf(surt)) { setUpdateField(solr, SolrFields.ACCESS_TERMS, "OA"); } else { setUpdateField(solr, SolrFields.ACCESS_TERMS, "RRO"); } } } Annotator(Annotations annotations, SurtPrefixSet oaSurts); static Annotator annotationsFromAct(); static SurtPrefixSet loadSurtPrefix(String surtPrefixFile); void applyAnnotations(URI uri, SolrInputDocument solr); static void main(String[] args);  }