TokenizedDocumentSerializer extends DocumentSerializer { @Override public byte[] toBytes(Document doc) throws IOException { ByteArrayOutputStream headerArray = new ByteArrayOutputStream(); DataOutputStream output = new DataOutputStream(headerArray); output.writeLong(doc.identifier); byte[] bytes = ByteUtil.fromString(doc.name); output.writeInt(bytes.length); output.write(bytes); output.close(); ByteArrayOutputStream metadataArray = SerializerCommon.writeMetadata(doc); ByteArrayOutputStream textArray = SerializerCommon.writeText(doc); ByteArrayOutputStream termsArray = SerializerCommon.writeTerms(doc); ByteArrayOutputStream docArray = new ByteArrayOutputStream(); output = new DataOutputStream(new SnappyOutputStream(docArray)); output.write(headerArray.toByteArray()); output.writeInt(metadataArray.size()); output.writeInt(textArray.size()); output.writeInt(termsArray.size()); output.write(metadataArray.toByteArray()); output.write(textArray.toByteArray()); output.write(termsArray.toByteArray()); output.close(); return docArray.toByteArray(); } TokenizedDocumentSerializer(Parameters opts); @Override byte[] toBytes(Document doc); @Override Document fromStream(DataInputStream stream, Document.DocumentComponents selection);  }
@Test public void testSerializeDocument() throws IOException { TagTokenizer tagTokenizer = new TagTokenizer(); DocumentSerializer tds = new TokenizedDocumentSerializer(Parameters.create()); Document doc = new Document(); doc.metadata.put("meta-key", "value"); doc.metadata.put("null-meta-key", null); doc.name = "doc-name"; doc.text = "doc text goes here\nand <tag>continues. This is weird</tag>"; tagTokenizer.tokenize(doc); byte[] docBytes = tds.toBytes(doc); assertNotNull(docBytes); Document doc2 = tds.fromBytes(docBytes, Document.DocumentComponents.All); assertEquals(doc.name, doc2.name); assertEquals(doc.text, doc2.text); assertNotNull(doc2.metadata); assertEquals(doc.metadata.get("meta-key"), doc2.metadata.get("meta-key")); assertTrue(doc.metadata.containsKey("null-meta-key")); assertNull(doc.metadata.get("null-meta-key")); assertEquals(doc.terms, doc2.terms); assertNull(doc2.tags); Document doc3 = tds.fromBytes(docBytes, new Document.DocumentComponents(false, true, true)); assertEquals(doc.name, doc3.name); assertNull(doc3.text); assertNotNull(doc3.metadata); assertNotNull(doc3.metadata.get("meta-key")); assertEquals("value", doc3.metadata.get("meta-key")); assertNull(doc3.metadata.get("null-meta-key")); assertEquals(doc.terms, doc2.terms); assertEquals(null, doc2.tags); }