TXTParser extends AbstractEncodingDetectorParser { public void parse( InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { try (AutoDetectReader reader = new AutoDetectReader( new CloseShieldInputStream(stream), metadata, getEncodingDetector(context))) { String incomingMime = metadata.get(Metadata.CONTENT_TYPE); MediaType mediaType = MediaType.TEXT_PLAIN; if (incomingMime != null) { MediaType tmpMediaType = MediaType.parse(incomingMime); if (tmpMediaType != null) { mediaType = tmpMediaType; } } Charset charset = reader.getCharset(); MediaType type = new MediaType(mediaType, charset); metadata.set(Metadata.CONTENT_TYPE, type.toString()); metadata.set(Metadata.CONTENT_ENCODING, charset.name()); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); xhtml.startElement("p"); char[] buffer = new char[4096]; int n = reader.read(buffer); while (n != -1) { xhtml.characters(buffer, 0, n); n = reader.read(buffer); } xhtml.endElement("p"); xhtml.endDocument(); } } TXTParser();  TXTParser(EncodingDetector encodingDetector); Set<MediaType> getSupportedTypes(ParseContext context); void parse(
            InputStream stream, ContentHandler handler,
            Metadata metadata, ParseContext context);  }
@Test public void testEnglishText() throws Exception { String text = "Hello, World! This is simple UTF-8 text content written" + " in English to test autodetection of both the character" + " encoding and the language of the input stream."; Metadata metadata = new Metadata(); StringWriter writer = new StringWriter(); parser.parse( new ByteArrayInputStream(text.getBytes(ISO_8859_1)), new WriteOutContentHandler(writer), metadata, new ParseContext()); String content = writer.toString(); assertEquals("text/plain; charset=ISO-8859-1", metadata.get(Metadata.CONTENT_TYPE)); assertNull(metadata.get(Metadata.CONTENT_LANGUAGE)); assertNull(metadata.get(TikaCoreProperties.LANGUAGE)); TikaTest.assertContains("Hello", content); TikaTest.assertContains("World", content); TikaTest.assertContains("autodetection", content); TikaTest.assertContains("stream", content); }
@Test public void testUTF8Text() throws Exception { String text = "I\u00F1t\u00EBrn\u00E2ti\u00F4n\u00E0liz\u00E6ti\u00F8n"; ContentHandler handler = new BodyContentHandler(); Metadata metadata = new Metadata(); parser.parse( new ByteArrayInputStream(text.getBytes(UTF_8)), handler, metadata, new ParseContext()); assertEquals("text/plain; charset=UTF-8", metadata.get(Metadata.CONTENT_TYPE)); assertEquals("UTF-8", metadata.get(Metadata.CONTENT_ENCODING)); TikaTest.assertContains(text, handler.toString()); }
@Test public void testEmptyText() throws Exception { ContentHandler handler = new BodyContentHandler(); Metadata metadata = new Metadata(); parser.parse( new ByteArrayInputStream(new byte[0]), handler, metadata, new ParseContext()); assertEquals("text/plain; charset=UTF-8", metadata.get(Metadata.CONTENT_TYPE)); assertEquals("\n", handler.toString()); }
@Test public void testLatinDetectionHeuristics() throws Exception { String windows = "test\r\n"; String unix = "test\n"; String euro = "test \u20ac\n"; Metadata metadata; metadata = new Metadata(); parser.parse( new ByteArrayInputStream(windows.getBytes("ISO-8859-15")), new DefaultHandler(), metadata, new ParseContext()); assertEquals( "text/plain; charset=windows-1252", metadata.get(Metadata.CONTENT_TYPE)); metadata = new Metadata(); parser.parse( new ByteArrayInputStream(unix.getBytes("ISO-8859-15")), new DefaultHandler(), metadata, new ParseContext()); assertEquals( "text/plain; charset=ISO-8859-1", metadata.get(Metadata.CONTENT_TYPE)); metadata = new Metadata(); parser.parse( new ByteArrayInputStream(euro.getBytes("ISO-8859-15")), new DefaultHandler(), metadata, new ParseContext()); assertEquals( "text/plain; charset=ISO-8859-15", metadata.get(Metadata.CONTENT_TYPE)); }
@Test public void testUseIncomingCharsetAsHint() throws Exception { final String test2 = "the name is \u00e1ndre"; Metadata metadata = new Metadata(); parser.parse( new ByteArrayInputStream(test2.getBytes(ISO_8859_1)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("text/plain; charset=ISO-8859-1", metadata.get(Metadata.CONTENT_TYPE)); assertEquals("ISO-8859-1", metadata.get(Metadata.CONTENT_ENCODING)); metadata.set(Metadata.CONTENT_TYPE, "text/plain; charset=ISO-8859-15"); parser.parse( new ByteArrayInputStream(test2.getBytes(ISO_8859_1)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("text/plain; charset=ISO-8859-15", metadata.get(Metadata.CONTENT_TYPE)); assertEquals("ISO-8859-15", metadata.get(Metadata.CONTENT_ENCODING)); }
@Test public void testUsingCharsetInContentTypeHeader() throws Exception { final String test2 = "the name is \u00e1ndre"; Metadata metadata = new Metadata(); parser.parse( new ByteArrayInputStream(test2.getBytes(ISO_8859_1)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("text/plain; charset=ISO-8859-1", metadata.get(Metadata.CONTENT_TYPE)); assertEquals("ISO-8859-1", metadata.get(Metadata.CONTENT_ENCODING)); metadata = new Metadata(); metadata.set(Metadata.CONTENT_TYPE, "text/html; charset=ISO-8859-15"); parser.parse( new ByteArrayInputStream(test2.getBytes(ISO_8859_1)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("text/html; charset=ISO-8859-15", metadata.get(Metadata.CONTENT_TYPE)); assertEquals("ISO-8859-15", metadata.get(Metadata.CONTENT_ENCODING)); }
@Test public void testRetainIncomingLanguage() throws Exception { final String test = "Simple Content"; Metadata metadata = new Metadata(); metadata.set(TikaCoreProperties.LANGUAGE, "en"); parser.parse( new ByteArrayInputStream(test.getBytes(UTF_8)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("en", metadata.get(TikaCoreProperties.LANGUAGE)); }
@Test public void testCP866() throws Exception { Metadata metadata = new Metadata(); StringWriter writer = new StringWriter(); parser.parse( TXTParserTest.class.getResourceAsStream("/test-documents/russian.cp866.txt"), new WriteOutContentHandler(writer), metadata, new ParseContext()); assertEquals("text/plain; charset=IBM866", metadata.get(Metadata.CONTENT_TYPE)); }
@Test public void testEBCDIC_CP500() throws Exception { Metadata metadata = new Metadata(); StringWriter writer = new StringWriter(); parser.parse( TXTParserTest.class.getResourceAsStream("/test-documents/english.cp500.txt"), new WriteOutContentHandler(writer), metadata, new ParseContext()); assertEquals("text/plain; charset=IBM500", metadata.get(Metadata.CONTENT_TYPE)); metadata = new Metadata(); writer = new StringWriter(); parser.parse( new ByteArrayInputStream("<html><body>hello world</body></html>".getBytes(ISO_8859_1)), new WriteOutContentHandler(writer), metadata, new ParseContext()); assertEquals("text/plain; charset=ISO-8859-1", metadata.get(Metadata.CONTENT_TYPE)); }
@Test public void testCharsetDetectionWithShortSnipet() throws Exception { final String text = "Hello, World!"; Metadata metadata = new Metadata(); parser.parse( new ByteArrayInputStream(text.getBytes(UTF_8)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("text/plain; charset=ISO-8859-1", metadata.get(Metadata.CONTENT_TYPE)); metadata.set(Metadata.CONTENT_TYPE, "application/binary; charset=UTF-8"); parser.parse( new ByteArrayInputStream(text.getBytes(UTF_8)), new BodyContentHandler(), metadata, new ParseContext()); assertEquals("application/binary; charset=UTF-8", metadata.get(Metadata.CONTENT_TYPE)); }