@Test public void testIndexWithNewline() throws IOException { String[] sentence = "He belongs to Apache \n Software Foundation .".split(" "); NameContextGenerator CG = new DefaultNameContextGenerator( (AdaptiveFeatureGenerator[]) null); NameSample nameSample = new NameSample(sentence, new Span[] { new Span(3, 7) }, false); ObjectStream<Event> eventStream = new NameFinderEventStream( ObjectStreamUtils.createObjectStream(nameSample), "org", CG, null); DataIndexer indexer = new TwoPassDataIndexer(); indexer.init(new TrainingParameters(Collections.emptyMap()), null); indexer.index(eventStream); Assert.assertEquals(5, indexer.getContexts().length); }
@Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with TwoPass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String,Integer> predicateIndex = new HashMap<>(); File tmp = File.createTempFile("events", null); tmp.deleteOnExit(); int numEvents; BigInteger writeHash; HashSumEventStream writeEventStream = new HashSumEventStream(eventStream); try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) { numEvents = computeEventCounts(writeEventStream, dos, predicateIndex, cutoff); } writeHash = writeEventStream.calculateHashSum(); display("done. " + numEvents + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare; BigInteger readHash = null; try (HashSumEventStream readStream = new HashSumEventStream(new EventStream(tmp))) { eventsToCompare = index(readStream, predicateIndex); readHash = readStream.calculateHashSum(); } tmp.delete(); if (readHash.compareTo(writeHash) != 0) throw new IOException("Event hash for writing and reading events did not match."); display("done.\n"); if (sort) { display("Sorting and merging events... "); } else { display("Collecting events... "); } sortAndMerge(eventsToCompare,sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); }
TwoPassDataIndexer extends AbstractDataIndexer { @Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with TwoPass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String,Integer> predicateIndex = new HashMap<>(); File tmp = File.createTempFile("events", null); tmp.deleteOnExit(); int numEvents; BigInteger writeHash; HashSumEventStream writeEventStream = new HashSumEventStream(eventStream); try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) { numEvents = computeEventCounts(writeEventStream, dos, predicateIndex, cutoff); } writeHash = writeEventStream.calculateHashSum(); display("done. " + numEvents + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare; BigInteger readHash = null; try (HashSumEventStream readStream = new HashSumEventStream(new EventStream(tmp))) { eventsToCompare = index(readStream, predicateIndex); readHash = readStream.calculateHashSum(); } tmp.delete(); if (readHash.compareTo(writeHash) != 0) throw new IOException("Event hash for writing and reading events did not match."); display("done.\n"); if (sort) { display("Sorting and merging events... "); } else { display("Collecting events... "); } sortAndMerge(eventsToCompare,sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); } }
TwoPassDataIndexer extends AbstractDataIndexer { @Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with TwoPass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String,Integer> predicateIndex = new HashMap<>(); File tmp = File.createTempFile("events", null); tmp.deleteOnExit(); int numEvents; BigInteger writeHash; HashSumEventStream writeEventStream = new HashSumEventStream(eventStream); try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) { numEvents = computeEventCounts(writeEventStream, dos, predicateIndex, cutoff); } writeHash = writeEventStream.calculateHashSum(); display("done. " + numEvents + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare; BigInteger readHash = null; try (HashSumEventStream readStream = new HashSumEventStream(new EventStream(tmp))) { eventsToCompare = index(readStream, predicateIndex); readHash = readStream.calculateHashSum(); } tmp.delete(); if (readHash.compareTo(writeHash) != 0) throw new IOException("Event hash for writing and reading events did not match."); display("done.\n"); if (sort) { display("Sorting and merging events... "); } else { display("Collecting events... "); } sortAndMerge(eventsToCompare,sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); } TwoPassDataIndexer(); }
TwoPassDataIndexer extends AbstractDataIndexer { @Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with TwoPass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String,Integer> predicateIndex = new HashMap<>(); File tmp = File.createTempFile("events", null); tmp.deleteOnExit(); int numEvents; BigInteger writeHash; HashSumEventStream writeEventStream = new HashSumEventStream(eventStream); try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) { numEvents = computeEventCounts(writeEventStream, dos, predicateIndex, cutoff); } writeHash = writeEventStream.calculateHashSum(); display("done. " + numEvents + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare; BigInteger readHash = null; try (HashSumEventStream readStream = new HashSumEventStream(new EventStream(tmp))) { eventsToCompare = index(readStream, predicateIndex); readHash = readStream.calculateHashSum(); } tmp.delete(); if (readHash.compareTo(writeHash) != 0) throw new IOException("Event hash for writing and reading events did not match."); display("done.\n"); if (sort) { display("Sorting and merging events... "); } else { display("Collecting events... "); } sortAndMerge(eventsToCompare,sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); } TwoPassDataIndexer(); @Override void index(ObjectStream<Event> eventStream); }
TwoPassDataIndexer extends AbstractDataIndexer { @Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with TwoPass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String,Integer> predicateIndex = new HashMap<>(); File tmp = File.createTempFile("events", null); tmp.deleteOnExit(); int numEvents; BigInteger writeHash; HashSumEventStream writeEventStream = new HashSumEventStream(eventStream); try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) { numEvents = computeEventCounts(writeEventStream, dos, predicateIndex, cutoff); } writeHash = writeEventStream.calculateHashSum(); display("done. " + numEvents + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare; BigInteger readHash = null; try (HashSumEventStream readStream = new HashSumEventStream(new EventStream(tmp))) { eventsToCompare = index(readStream, predicateIndex); readHash = readStream.calculateHashSum(); } tmp.delete(); if (readHash.compareTo(writeHash) != 0) throw new IOException("Event hash for writing and reading events did not match."); display("done.\n"); if (sort) { display("Sorting and merging events... "); } else { display("Collecting events... "); } sortAndMerge(eventsToCompare,sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); } TwoPassDataIndexer(); @Override void index(ObjectStream<Event> eventStream);  }