@Test public void testParseURL() throws NoSuchAlgorithmException, URIException { WARCIndexer windex = new WARCIndexer(); String[] testUrls = { "http: "http: "http: "http: for (String urlString : testUrls) { SolrRecord solr = SolrRecordFactory.createFactory(null).createRecord(); URI test1 = windex.parseURL(solr, urlString); assertEquals(1, solr.getField(SolrFields.PUBLIC_SUFFIX).getValueCount()); } }
protected URI parseURL(SolrRecord solr, String fullUrl) throws URIException { UsableURI url = UsableURIFactory.getInstance(fullUrl); solr.setField(SolrFields.SOLR_URL_PATH, url.getPath()); if (url.getPath().equals("/") || url.getPath().equals("") || url.getPath().matches("/index\\.[a-z]+$")) { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_SLASHPAGE); } else if (url.getPath().equalsIgnoreCase("/robots.txt")) { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_ROBOTS_TXT); } else { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_NORMAL); } String host = url.getHost(); if (CANONICALISE_HOST) host = Normalisation.canonicaliseHost(host); solr.setField(SolrFields.SOLR_HOST, host); solr.removeField(SolrFields.SOLR_HOST_SURT); ImmutableList<String> levels = LinkExtractor.allLevels(host); if (levels != null) { for (String level : levels) { solr.addField(SolrFields.SOLR_HOST_SURT, SURT.toSURT(level)); } } final String domain = LinkExtractor.extractPrivateSuffixFromHost(host); solr.setField(SolrFields.DOMAIN, domain); solr.setField(SolrFields.PUBLIC_SUFFIX, LinkExtractor.extractPublicSuffixFromHost(host)); org.apache.commons.httpclient.URI tempUri = new org.apache.commons.httpclient .URI(url.getEscapedURI(), false); return URI.create(tempUri.getEscapedURI()); }
WARCIndexer { protected URI parseURL(SolrRecord solr, String fullUrl) throws URIException { UsableURI url = UsableURIFactory.getInstance(fullUrl); solr.setField(SolrFields.SOLR_URL_PATH, url.getPath()); if (url.getPath().equals("/") || url.getPath().equals("") || url.getPath().matches("/index\\.[a-z]+$")) { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_SLASHPAGE); } else if (url.getPath().equalsIgnoreCase("/robots.txt")) { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_ROBOTS_TXT); } else { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_NORMAL); } String host = url.getHost(); if (CANONICALISE_HOST) host = Normalisation.canonicaliseHost(host); solr.setField(SolrFields.SOLR_HOST, host); solr.removeField(SolrFields.SOLR_HOST_SURT); ImmutableList<String> levels = LinkExtractor.allLevels(host); if (levels != null) { for (String level : levels) { solr.addField(SolrFields.SOLR_HOST_SURT, SURT.toSURT(level)); } } final String domain = LinkExtractor.extractPrivateSuffixFromHost(host); solr.setField(SolrFields.DOMAIN, domain); solr.setField(SolrFields.PUBLIC_SUFFIX, LinkExtractor.extractPublicSuffixFromHost(host)); org.apache.commons.httpclient.URI tempUri = new org.apache.commons.httpclient .URI(url.getEscapedURI(), false); return URI.create(tempUri.getEscapedURI()); } }
WARCIndexer { protected URI parseURL(SolrRecord solr, String fullUrl) throws URIException { UsableURI url = UsableURIFactory.getInstance(fullUrl); solr.setField(SolrFields.SOLR_URL_PATH, url.getPath()); if (url.getPath().equals("/") || url.getPath().equals("") || url.getPath().matches("/index\\.[a-z]+$")) { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_SLASHPAGE); } else if (url.getPath().equalsIgnoreCase("/robots.txt")) { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_ROBOTS_TXT); } else { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_NORMAL); } String host = url.getHost(); if (CANONICALISE_HOST) host = Normalisation.canonicaliseHost(host); solr.setField(SolrFields.SOLR_HOST, host); solr.removeField(SolrFields.SOLR_HOST_SURT); ImmutableList<String> levels = LinkExtractor.allLevels(host); if (levels != null) { for (String level : levels) { solr.addField(SolrFields.SOLR_HOST_SURT, SURT.toSURT(level)); } } final String domain = LinkExtractor.extractPrivateSuffixFromHost(host); solr.setField(SolrFields.DOMAIN, domain); solr.setField(SolrFields.PUBLIC_SUFFIX, LinkExtractor.extractPublicSuffixFromHost(host)); org.apache.commons.httpclient.URI tempUri = new org.apache.commons.httpclient .URI(url.getEscapedURI(), false); return URI.create(tempUri.getEscapedURI()); } WARCIndexer();  WARCIndexer( Config conf ); }
WARCIndexer { protected URI parseURL(SolrRecord solr, String fullUrl) throws URIException { UsableURI url = UsableURIFactory.getInstance(fullUrl); solr.setField(SolrFields.SOLR_URL_PATH, url.getPath()); if (url.getPath().equals("/") || url.getPath().equals("") || url.getPath().matches("/index\\.[a-z]+$")) { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_SLASHPAGE); } else if (url.getPath().equalsIgnoreCase("/robots.txt")) { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_ROBOTS_TXT); } else { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_NORMAL); } String host = url.getHost(); if (CANONICALISE_HOST) host = Normalisation.canonicaliseHost(host); solr.setField(SolrFields.SOLR_HOST, host); solr.removeField(SolrFields.SOLR_HOST_SURT); ImmutableList<String> levels = LinkExtractor.allLevels(host); if (levels != null) { for (String level : levels) { solr.addField(SolrFields.SOLR_HOST_SURT, SURT.toSURT(level)); } } final String domain = LinkExtractor.extractPrivateSuffixFromHost(host); solr.setField(SolrFields.DOMAIN, domain); solr.setField(SolrFields.PUBLIC_SUFFIX, LinkExtractor.extractPublicSuffixFromHost(host)); org.apache.commons.httpclient.URI tempUri = new org.apache.commons.httpclient .URI(url.getEscapedURI(), false); return URI.create(tempUri.getEscapedURI()); } WARCIndexer();  WARCIndexer( Config conf ); void setAnnotations(Annotations ann, SurtPrefixSet openAccessSurts); boolean isCheckSolrForDuplicates(); void setCheckSolrForDuplicates(boolean checkSolrForDuplicates); SolrRecord extract( String archiveName, ArchiveRecord record ); SolrRecord extract( String archiveName, ArchiveRecord record, boolean isTextIncluded ); static Date getWaybackDate( String timestamp ); static String extractYear( String timestamp ); }
WARCIndexer { protected URI parseURL(SolrRecord solr, String fullUrl) throws URIException { UsableURI url = UsableURIFactory.getInstance(fullUrl); solr.setField(SolrFields.SOLR_URL_PATH, url.getPath()); if (url.getPath().equals("/") || url.getPath().equals("") || url.getPath().matches("/index\\.[a-z]+$")) { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_SLASHPAGE); } else if (url.getPath().equalsIgnoreCase("/robots.txt")) { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_ROBOTS_TXT); } else { solr.setField(SolrFields.SOLR_URL_TYPE, SolrFields.SOLR_URL_TYPE_NORMAL); } String host = url.getHost(); if (CANONICALISE_HOST) host = Normalisation.canonicaliseHost(host); solr.setField(SolrFields.SOLR_HOST, host); solr.removeField(SolrFields.SOLR_HOST_SURT); ImmutableList<String> levels = LinkExtractor.allLevels(host); if (levels != null) { for (String level : levels) { solr.addField(SolrFields.SOLR_HOST_SURT, SURT.toSURT(level)); } } final String domain = LinkExtractor.extractPrivateSuffixFromHost(host); solr.setField(SolrFields.DOMAIN, domain); solr.setField(SolrFields.PUBLIC_SUFFIX, LinkExtractor.extractPublicSuffixFromHost(host)); org.apache.commons.httpclient.URI tempUri = new org.apache.commons.httpclient .URI(url.getEscapedURI(), false); return URI.create(tempUri.getEscapedURI()); } WARCIndexer();  WARCIndexer( Config conf ); void setAnnotations(Annotations ann, SurtPrefixSet openAccessSurts); boolean isCheckSolrForDuplicates(); void setCheckSolrForDuplicates(boolean checkSolrForDuplicates); SolrRecord extract( String archiveName, ArchiveRecord record ); SolrRecord extract( String archiveName, ArchiveRecord record, boolean isTextIncluded ); static Date getWaybackDate( String timestamp ); static String extractYear( String timestamp ); static final boolean CANONICALISE_HOST; static SimpleDateFormat formatter; }