RFC822Parser extends AbstractParser { public void parse(InputStream stream, ContentHandler handler, Metadata metadata, ParseContext context) throws IOException, SAXException, TikaException { MimeConfig config = new MimeConfig.Builder() .setMaxLineLen(100000) .setMaxHeaderLen(100000) .build(); config = context.get(MimeConfig.class, config); Detector localDetector = context.get(Detector.class); if (localDetector == null) { if (detector == null) { EmbeddedDocumentUtil embeddedDocumentUtil = new EmbeddedDocumentUtil(context); detector = embeddedDocumentUtil.getDetector(); } localDetector = detector; } MimeStreamParser parser = new MimeStreamParser(config, null, new DefaultBodyDescriptorBuilder()); XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); MailContentHandler mch = new MailContentHandler( xhtml, localDetector, metadata, context, config.isStrictParsing(), extractAllAlternatives); parser.setContentHandler(mch); parser.setContentDecoding(true); xhtml.startDocument(); TikaInputStream tstream = TikaInputStream.get(stream); try { parser.parse(tstream); } catch (IOException e) { tstream.throwIfCauseOf(e); throw new TikaException("Failed to parse an email message", e); } catch (MimeException e) { Throwable cause = e.getCause(); if (cause instanceof TikaException) { throw (TikaException) cause; } else if (cause instanceof SAXException) { throw (SAXException) cause; } else { throw new TikaException("Failed to parse an email message", e); } } xhtml.endDocument(); }  Set<MediaType> getSupportedTypes(ParseContext context); void parse(InputStream stream, ContentHandler handler,
                      Metadata metadata, ParseContext context); void setExtractAllAlternatives(boolean extractAllAlternatives);  }
@Test public void testSimple() throws Exception { Metadata metadata = new Metadata(); InputStream stream = getStream("test-documents/testRFC822"); ContentHandler handler = mock(DefaultHandler.class); try { EXTRACT_ALL_ALTERNATIVES_PARSER.parse(stream, handler, metadata, new ParseContext()); verify(handler).startDocument(); verify(handler).startElement(eq(XHTMLContentHandler.XHTML), eq("p"), eq("p"), any(Attributes.class)); verify(handler).endElement(XHTMLContentHandler.XHTML, "p", "p"); verify(handler, never()).startElement(eq(XHTMLContentHandler.XHTML), eq("div"), eq("div"), any(Attributes.class)); verify(handler, never()).endElement(XHTMLContentHandler.XHTML, "div", "div"); verify(handler).endDocument(); assertEquals("Julien Nioche (JIRA) <jira@apache.org>", metadata.get(TikaCoreProperties.CREATOR)); assertEquals("[jira] Commented: (TIKA-461) RFC822 messages not parsed", metadata.get(TikaCoreProperties.TITLE)); assertEquals("[jira] Commented: (TIKA-461) RFC822 messages not parsed", metadata.get(TikaCoreProperties.SUBJECT)); } catch (Exception e) { fail("Exception thrown: " + e.getMessage()); } }
@Test public void testMultipart() { Metadata metadata = new Metadata(); InputStream stream = getStream("test-documents/testRFC822-multipart"); ContentHandler handler = mock(XHTMLContentHandler.class); ParseContext context = new ParseContext(); context.set(Parser.class, EXTRACT_ALL_ALTERNATIVES_PARSER); try { EXTRACT_ALL_ALTERNATIVES_PARSER.parse(stream, handler, metadata, context); verify(handler).startDocument(); int bodyExpectedTimes = 4, multipackExpectedTimes = 5; if (!TesseractOCRParserTest.canRun()) { verify(handler, times(bodyExpectedTimes)).startElement(eq(XHTMLContentHandler.XHTML), eq("div"), eq("div"), any(Attributes.class)); verify(handler, times(bodyExpectedTimes)).endElement(XHTMLContentHandler.XHTML, "div", "div"); } verify(handler, times(multipackExpectedTimes)).startElement(eq(XHTMLContentHandler.XHTML), eq("p"), eq("p"), any(Attributes.class)); verify(handler, times(multipackExpectedTimes)).endElement(XHTMLContentHandler.XHTML, "p", "p"); verify(handler).endDocument(); } catch (Exception e) { fail("Exception thrown: " + e.getMessage()); } metadata = new Metadata(); stream = getStream("test-documents/testRFC822-multipart"); handler = new BodyContentHandler(); try { EXTRACT_ALL_ALTERNATIVES_PARSER.parse(stream, handler, metadata, context); String bodyText = handler.toString(); assertTrue(bodyText.contains("body 1")); assertTrue(bodyText.contains("body 2")); assertFalse(bodyText.contains("R0lGODlhNgE8AMQAA")); } catch (Exception e) { fail("Exception thrown: " + e.getMessage()); } }
@Test public void testQuotedPrintable() { Metadata metadata = new Metadata(); InputStream stream = getStream("test-documents/testRFC822_quoted"); ContentHandler handler = new BodyContentHandler(); ParseContext context = new ParseContext(); try { EXTRACT_ALL_ALTERNATIVES_PARSER.parse(stream, handler, metadata, context); String bodyText = handler.toString(); assertTrue(bodyText.contains("D\u00FCsseldorf has non-ascii.")); assertTrue(bodyText.contains("Lines can be split like this.")); assertTrue(bodyText.contains("Spaces at the end of a line \r\nmust be encoded.\r\n")); assertFalse(bodyText.contains("=")); } catch (Exception e) { fail("Exception thrown: " + e.getMessage()); } }
@Test public void testBase64() throws Exception { Metadata metadata = new Metadata(); InputStream stream = getStream("test-documents/testRFC822_base64"); ContentHandler handler = new BodyContentHandler(); ParseContext context = new ParseContext(); context.set(Parser.class, EXTRACT_ALL_ALTERNATIVES_PARSER); metadata.set(Metadata.CONTENT_TYPE, "message/rfc822"); try { EXTRACT_ALL_ALTERNATIVES_PARSER.parse(stream, handler, metadata, context); assertContains("Here is some text, with international characters, voil\u00E0!", handler.toString()); } catch (Exception e) { fail("Exception thrown: " + e.getMessage()); } }
@Test public void testI18NHeaders() { Metadata metadata = new Metadata(); InputStream stream = getStream("test-documents/testRFC822_i18nheaders"); ContentHandler handler = mock(DefaultHandler.class); try { EXTRACT_ALL_ALTERNATIVES_PARSER.parse(stream, handler, metadata, new ParseContext()); assertEquals("Keld J\u00F8rn Simonsen <keld@dkuug.dk>", metadata.get(TikaCoreProperties.CREATOR)); assertEquals("If you can read this you understand the example.", metadata.get(TikaCoreProperties.TITLE)); assertEquals("If you can read this you understand the example.", metadata.get(TikaCoreProperties.SUBJECT)); } catch (Exception e) { fail("Exception thrown: " + e.getMessage()); } }
@Test public void testUnusualFromAddress() throws Exception { Metadata metadata = new Metadata(); InputStream stream = getStream("test-documents/testRFC822_oddfrom"); ContentHandler handler = mock(DefaultHandler.class); EXTRACT_ALL_ALTERNATIVES_PARSER.parse(stream, handler, metadata, new ParseContext()); assertEquals("Saved by Windows Internet Explorer 7", metadata.get(TikaCoreProperties.CREATOR)); assertEquals("Air Permit Programs | Air & Radiation | US EPA", metadata.get(TikaCoreProperties.TITLE)); assertEquals("Air Permit Programs | Air & Radiation | US EPA", metadata.get(TikaCoreProperties.SUBJECT)); }
@Test public void testLongHeader() throws Exception { StringBuilder inputBuilder = new StringBuilder(); for (int i = 0; i < 2000; ++i) { inputBuilder.append( "really really really really really really long name "); } String name = inputBuilder.toString(); byte[] data = ("Status: 520\r\nFrom: " + name + "\r\n\r\n").getBytes(US_ASCII); ContentHandler handler = new DefaultHandler(); Metadata metadata = new Metadata(); ParseContext context = new ParseContext(); try { EXTRACT_ALL_ALTERNATIVES_PARSER.parse( new ByteArrayInputStream(data), handler, metadata, context); fail(); } catch (TikaException expected) { } MimeConfig config = new MimeConfig.Builder().setMaxHeaderLen(-1).setMaxLineLen(-1).build(); context.set(MimeConfig.class, config); EXTRACT_ALL_ALTERNATIVES_PARSER.parse( new ByteArrayInputStream(data), handler, metadata, context); assertEquals(name.trim(), metadata.get(TikaCoreProperties.CREATOR)); }
@Test public void testSomeMissingHeaders() throws Exception { Metadata metadata = new Metadata(); InputStream stream = getStream("test-documents/testRFC822-limitedheaders"); ContentHandler handler = new BodyContentHandler(); ParseContext context = new ParseContext(); context.set(Parser.class, EXTRACT_ALL_ALTERNATIVES_PARSER); EXTRACT_ALL_ALTERNATIVES_PARSER.parse(stream, handler, metadata, context); assertEquals(true, metadata.isMultiValued(TikaCoreProperties.CREATOR)); assertEquals("xyz", metadata.getValues(TikaCoreProperties.CREATOR)[0]); assertEquals("abc", metadata.getValues(TikaCoreProperties.CREATOR)[1]); assertEquals(true, metadata.isMultiValued(Metadata.MESSAGE_FROM)); assertEquals("xyz", metadata.getValues(Metadata.MESSAGE_FROM)[0]); assertEquals("abc", metadata.getValues(Metadata.MESSAGE_FROM)[1]); assertEquals(true, metadata.isMultiValued(Metadata.MESSAGE_TO)); assertEquals("abc", metadata.getValues(Metadata.MESSAGE_TO)[0]); assertEquals("def", metadata.getValues(Metadata.MESSAGE_TO)[1]); assertEquals("abcd", metadata.get(TikaCoreProperties.TITLE)); assertEquals("abcd", metadata.get(TikaCoreProperties.SUBJECT)); assertContains("bar biz bat", handler.toString()); }
@Test public void testTrickyDates() throws Exception { DateFormat df = new SimpleDateFormat("yyyy-MM-dd", new DateFormatSymbols(Locale.US)); Date date1980 = df.parse("1980-01-01"); for (String dateString : new String[] { "Mon, 29 Jan 96 14:02 GMT", "7/20/95 1:12pm", "08/14/2000 12:48 AM", "06/24/2008, Tuesday, 11 AM", "11/14/08", "12/02/1996", "96/12/02", }) { Date parsedDate = getDate(dateString); if (parsedDate != null) { assertTrue("date must be after 1980:"+dateString, parsedDate.getTime() > date1980.getTime()); } } }
@Test public void testMultipleSubjects() throws Exception { String s = "From: Shawn Jones [chiroshawn@yahoo.com]\n" + "Subject: 2006N-3502\n" + "Subject: I Urge You to Require Notice of Mercury"; Parser p = new RFC822Parser(); Metadata m = new Metadata(); p.parse(TikaInputStream.get(s.getBytes(StandardCharsets.UTF_8)), new DefaultHandler(), m, new ParseContext()); assertEquals("I Urge You to Require Notice of Mercury", m.get(TikaCoreProperties.TITLE)); }