TikaEvalMetadataFilter implements MetadataFilter { @Override public void filter(Metadata metadata) throws TikaException { String content = metadata.get(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT); if (StringUtils.isAllBlank(content)) { return; } calcStats(content, metadata); }  @Override void filter(Metadata metadata); static String TIKA_EVAL_NS; static Property NUM_TOKENS; static Property NUM_UNIQUE_TOKENS; static Property NUM_ALPHA_TOKENS; static Property NUM_UNIQUE_ALPHA_TOKENS; static Property LANGUAGE; static Property LANGUAGE_CONFIDENCE; static Property OUT_OF_VOCABULARY; }
@Test public void testBasic() throws Exception { for (MetadataFilter filter : new MetadataFilter[]{ new TikaEvalMetadataFilter(), new DefaultMetadataFilter() }) { Metadata metadata = new Metadata(); String content = "the quick brown fox, Zothro 1234 1235, jumped over the lazy dog"; metadata.set(AbstractRecursiveParserWrapperHandler.TIKA_CONTENT, content); filter.filter(metadata); assertEquals("eng", metadata.get(TikaEvalMetadataFilter.LANGUAGE)); assertEquals(12, (int) metadata.getInt(TikaEvalMetadataFilter.NUM_TOKENS)); assertEquals(11, (int) metadata.getInt(TikaEvalMetadataFilter.NUM_UNIQUE_TOKENS)); assertEquals(10, (int) metadata.getInt(TikaEvalMetadataFilter.NUM_ALPHA_TOKENS)); assertEquals(9, (int) metadata.getInt(TikaEvalMetadataFilter.NUM_UNIQUE_ALPHA_TOKENS)); assertEquals(0.0999, Double.parseDouble(metadata.get(TikaEvalMetadataFilter.OUT_OF_VOCABULARY)), 0.1); assertEquals("eng", metadata.get(TikaEvalMetadataFilter.LANGUAGE)); assertEquals(0.0196, Double.parseDouble(metadata.get(TikaEvalMetadataFilter.LANGUAGE_CONFIDENCE)), 0.1); } }