@Test public void testLatinDetectionHeuristics() throws Exception { String windows = "test\r\n"; String unix = "test\n"; String euro = "test \u20ac\n"; Metadata metadata; metadata = new Metadata(); parser.parse( new ByteArrayInputStream(windows.getBytes("ISO-8859-15")), new DefaultHandler(), metadata, new ParseContext()); assertEquals( "text/plain; charset=windows-1252", metadata.get(Metadata.CONTENT_TYPE)); metadata = new Metadata(); parser.parse( new ByteArrayInputStream(unix.getBytes("ISO-8859-15")), new DefaultHandler(), metadata, new ParseContext()); assertEquals( "text/plain; charset=ISO-8859-1", metadata.get(Metadata.CONTENT_TYPE)); metadata = new Metadata(); parser.parse( new ByteArrayInputStream(euro.getBytes("ISO-8859-15")), new DefaultHandler(), metadata, new ParseContext()); assertEquals( "text/plain; charset=ISO-8859-15", metadata.get(Metadata.CONTENT_TYPE)); }
public void parse( InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { try (AutoDetectReader reader = new AutoDetectReader( new CloseShieldInputStream(stream), metadata, getEncodingDetector(context))) { String incomingMime = metadata.get(Metadata.CONTENT_TYPE); MediaType mediaType = MediaType.TEXT_PLAIN; if (incomingMime != null) { MediaType tmpMediaType = MediaType.parse(incomingMime); if (tmpMediaType != null) { mediaType = tmpMediaType; } } Charset charset = reader.getCharset(); MediaType type = new MediaType(mediaType, charset); metadata.set(Metadata.CONTENT_TYPE, type.toString()); metadata.set(Metadata.CONTENT_ENCODING, charset.name()); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); xhtml.startElement("p"); char[] buffer = new char[4096]; int n = reader.read(buffer); while (n != -1) { xhtml.characters(buffer, 0, n); n = reader.read(buffer); } xhtml.endElement("p"); xhtml.endDocument(); } }
TXTParser extends AbstractEncodingDetectorParser { public void parse( InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { try (AutoDetectReader reader = new AutoDetectReader( new CloseShieldInputStream(stream), metadata, getEncodingDetector(context))) { String incomingMime = metadata.get(Metadata.CONTENT_TYPE); MediaType mediaType = MediaType.TEXT_PLAIN; if (incomingMime != null) { MediaType tmpMediaType = MediaType.parse(incomingMime); if (tmpMediaType != null) { mediaType = tmpMediaType; } } Charset charset = reader.getCharset(); MediaType type = new MediaType(mediaType, charset); metadata.set(Metadata.CONTENT_TYPE, type.toString()); metadata.set(Metadata.CONTENT_ENCODING, charset.name()); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); xhtml.startElement("p"); char[] buffer = new char[4096]; int n = reader.read(buffer); while (n != -1) { xhtml.characters(buffer, 0, n); n = reader.read(buffer); } xhtml.endElement("p"); xhtml.endDocument(); } } }
TXTParser extends AbstractEncodingDetectorParser { public void parse( InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { try (AutoDetectReader reader = new AutoDetectReader( new CloseShieldInputStream(stream), metadata, getEncodingDetector(context))) { String incomingMime = metadata.get(Metadata.CONTENT_TYPE); MediaType mediaType = MediaType.TEXT_PLAIN; if (incomingMime != null) { MediaType tmpMediaType = MediaType.parse(incomingMime); if (tmpMediaType != null) { mediaType = tmpMediaType; } } Charset charset = reader.getCharset(); MediaType type = new MediaType(mediaType, charset); metadata.set(Metadata.CONTENT_TYPE, type.toString()); metadata.set(Metadata.CONTENT_ENCODING, charset.name()); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); xhtml.startElement("p"); char[] buffer = new char[4096]; int n = reader.read(buffer); while (n != -1) { xhtml.characters(buffer, 0, n); n = reader.read(buffer); } xhtml.endElement("p"); xhtml.endDocument(); } } TXTParser();  TXTParser(EncodingDetector encodingDetector); }
TXTParser extends AbstractEncodingDetectorParser { public void parse( InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { try (AutoDetectReader reader = new AutoDetectReader( new CloseShieldInputStream(stream), metadata, getEncodingDetector(context))) { String incomingMime = metadata.get(Metadata.CONTENT_TYPE); MediaType mediaType = MediaType.TEXT_PLAIN; if (incomingMime != null) { MediaType tmpMediaType = MediaType.parse(incomingMime); if (tmpMediaType != null) { mediaType = tmpMediaType; } } Charset charset = reader.getCharset(); MediaType type = new MediaType(mediaType, charset); metadata.set(Metadata.CONTENT_TYPE, type.toString()); metadata.set(Metadata.CONTENT_ENCODING, charset.name()); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); xhtml.startElement("p"); char[] buffer = new char[4096]; int n = reader.read(buffer); while (n != -1) { xhtml.characters(buffer, 0, n); n = reader.read(buffer); } xhtml.endElement("p"); xhtml.endDocument(); } } TXTParser();  TXTParser(EncodingDetector encodingDetector); Set<MediaType> getSupportedTypes(ParseContext context); void parse(
            InputStream stream, ContentHandler handler,
            Metadata metadata, ParseContext context); }
TXTParser extends AbstractEncodingDetectorParser { public void parse( InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { try (AutoDetectReader reader = new AutoDetectReader( new CloseShieldInputStream(stream), metadata, getEncodingDetector(context))) { String incomingMime = metadata.get(Metadata.CONTENT_TYPE); MediaType mediaType = MediaType.TEXT_PLAIN; if (incomingMime != null) { MediaType tmpMediaType = MediaType.parse(incomingMime); if (tmpMediaType != null) { mediaType = tmpMediaType; } } Charset charset = reader.getCharset(); MediaType type = new MediaType(mediaType, charset); metadata.set(Metadata.CONTENT_TYPE, type.toString()); metadata.set(Metadata.CONTENT_ENCODING, charset.name()); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); xhtml.startDocument(); xhtml.startElement("p"); char[] buffer = new char[4096]; int n = reader.read(buffer); while (n != -1) { xhtml.characters(buffer, 0, n); n = reader.read(buffer); } xhtml.endElement("p"); xhtml.endDocument(); } } TXTParser();  TXTParser(EncodingDetector encodingDetector); Set<MediaType> getSupportedTypes(ParseContext context); void parse(
            InputStream stream, ContentHandler handler,
            Metadata metadata, ParseContext context);  }