@Test public void testTruncatedTime() throws NoSuchAlgorithmException, IOException { final String WARC = "truncated_datetime.warc"; final String RECORD_ID = "201908150102/+BV/tmv/tASHANg2c3/2MA=="; WARCIndexer windex = new WARCIndexer(ConfigFactory.load()); String inputFile = this.getClass().getClassLoader().getResource(WARC).getPath(); ArchiveReader reader = ArchiveReaderFactory.get(inputFile); for (ArchiveRecord rec : reader) { SolrRecord doc = windex.extract("", rec); if (doc != null && doc.getField(SolrFields.ID).getValue().equals(RECORD_ID)) { String crawl = doc.getField(SolrFields.CRAWL_DATE).getValue().toString().replaceAll("[^0-9]", ""); String wayback = doc.getField(SolrFields.WAYBACK_DATE).getValue().toString(); assertEquals("crawlDate and waybackDate should designate the same instance", crawl, wayback); break; } } }
public SolrRecord extract( String archiveName, ArchiveRecord record ) throws IOException { return this.extract( archiveName, record, this.extractText ); }
WARCIndexer { public SolrRecord extract( String archiveName, ArchiveRecord record ) throws IOException { return this.extract( archiveName, record, this.extractText ); } }
WARCIndexer { public SolrRecord extract( String archiveName, ArchiveRecord record ) throws IOException { return this.extract( archiveName, record, this.extractText ); } WARCIndexer();  WARCIndexer( Config conf ); }
WARCIndexer { public SolrRecord extract( String archiveName, ArchiveRecord record ) throws IOException { return this.extract( archiveName, record, this.extractText ); } WARCIndexer();  WARCIndexer( Config conf ); void setAnnotations(Annotations ann, SurtPrefixSet openAccessSurts); boolean isCheckSolrForDuplicates(); void setCheckSolrForDuplicates(boolean checkSolrForDuplicates); SolrRecord extract( String archiveName, ArchiveRecord record ); SolrRecord extract( String archiveName, ArchiveRecord record, boolean isTextIncluded ); static Date getWaybackDate( String timestamp ); static String extractYear( String timestamp ); }
WARCIndexer { public SolrRecord extract( String archiveName, ArchiveRecord record ) throws IOException { return this.extract( archiveName, record, this.extractText ); } WARCIndexer();  WARCIndexer( Config conf ); void setAnnotations(Annotations ann, SurtPrefixSet openAccessSurts); boolean isCheckSolrForDuplicates(); void setCheckSolrForDuplicates(boolean checkSolrForDuplicates); SolrRecord extract( String archiveName, ArchiveRecord record ); SolrRecord extract( String archiveName, ArchiveRecord record, boolean isTextIncluded ); static Date getWaybackDate( String timestamp ); static String extractYear( String timestamp ); static final boolean CANONICALISE_HOST; static SimpleDateFormat formatter; }