WARCRecord { public Header getHeader() { return header; } WARCRecord(DataInput in);  WARCRecord(WARCRecord other); boolean isContentApplicationHttpResponse(); String getHTTPHeaders(); Header getHeader(); byte[] getContent(); void write(DataOutput out); @Override String toString(); void setContent(String newContent); static String extractHTTPHeaderContentType(String httpHeaderText); static String extractHTTPHeaderCharset(String httpHeaderText); static final String WARC_VERSION; }
@Test public void testParseWARCInfo() throws IOException { DataInputStream stream = new DataInputStream(new ByteArrayInputStream( ("WARC/1.0\r\n" + "WARC-Type: warcinfo\r\n" + "WARC-Date: 2014-03-18T17:47:38Z\r\n" + "WARC-Record-ID: <urn:uuid:d9bbb325-c09f-473c-8600-1c9dbd4ec443>\r\n" + "Content-Length: 371\r\n" + "Content-Type: application/warc-fields\r\n" + "WARC-Filename: CC-MAIN-20140313024455-00000-ip-10-183-142-35.ec2.internal.warc.gz\r\n" + "\r\n" + "robots: classic\r\n" + "hostname: ip-10-183-142-35.ec2.internal\r\n" + "software: Nutch 1.6 (CC)/CC WarcExport 1.0\r\n" + "isPartOf: CC-MAIN-2014-10\r\n" + "operator: CommonCrawl Admin\r\n" + "description: Wide crawl of the web with URLs provided by Blekko for March 2014\r\n" + "publisher: CommonCrawl\r\n" + "format: WARC File Format 1.0\r\n" + "conformsTo: http: + "\r\n" + "\r\n" + "\r\n").getBytes("UTF-8"))); WARCRecord record = new WARCRecord(stream); assertEquals(371, record.getHeader().getContentLength()); assertEquals("warcinfo", record.getHeader().getRecordType()); assertEquals("2014-03-18T17:47:38Z", record.getHeader().getDateString()); assertEquals("<urn:uuid:d9bbb325-c09f-473c-8600-1c9dbd4ec443>", record.getHeader().getRecordID()); assertEquals("application/warc-fields", record.getHeader().getContentType()); assertNull(record.getHeader().getTargetURI()); }