@Test public void testIndex() throws IOException { ObjectStream<Event> eventStream = new SimpleEventStreamBuilder() .add("other/w=he n1w=belongs n2w=to po=other pow=other,He powf=other,ic ppo=other") .add("other/w=belongs p1w=he n1w=to n2w=apache po=other pow=other,belongs powf=other,lc ppo=other") .add("other/w=to p1w=belongs p2w=he n1w=apache n2w=software po=other pow=other,to" + " powf=other,lc ppo=other") .add("org-start/w=apache p1w=to p2w=belongs n1w=software n2w=foundation po=other pow=other,Apache" + " powf=other,ic ppo=other") .add("org-cont/w=software p1w=apache p2w=to n1w=foundation n2w=. po=org-start" + " pow=org-start,Software powf=org-start,ic ppo=other") .add("org-cont/w=foundation p1w=software p2w=apache n1w=. po=org-cont pow=org-cont,Foundation" + " powf=org-cont,ic ppo=org-start") .add("other/w=. p1w=foundation p2w=software po=org-cont pow=org-cont,. powf=org-cont,other" + " ppo=org-cont") .build(); DataIndexer indexer = new TwoPassDataIndexer(); indexer.init(new TrainingParameters(Collections.emptyMap()), null); indexer.index(eventStream); Assert.assertEquals(3, indexer.getContexts().length); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[0]); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[1]); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[2]); Assert.assertNull(indexer.getValues()); Assert.assertEquals(5, indexer.getNumEvents()); Assert.assertArrayEquals(new int[]{0, 1, 2}, indexer.getOutcomeList()); Assert.assertArrayEquals(new int[]{3, 1, 1}, indexer.getNumTimesEventsSeen()); Assert.assertArrayEquals(new String[]{"ppo=other"}, indexer.getPredLabels()); Assert.assertArrayEquals(new String[]{"other", "org-start", "org-cont"}, indexer.getOutcomeLabels()); Assert.assertArrayEquals(new int[]{5}, indexer.getPredCounts()); }
@Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with TwoPass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String,Integer> predicateIndex = new HashMap<>(); File tmp = File.createTempFile("events", null); tmp.deleteOnExit(); int numEvents; BigInteger writeHash; HashSumEventStream writeEventStream = new HashSumEventStream(eventStream); try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) { numEvents = computeEventCounts(writeEventStream, dos, predicateIndex, cutoff); } writeHash = writeEventStream.calculateHashSum(); display("done. " + numEvents + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare; BigInteger readHash = null; try (HashSumEventStream readStream = new HashSumEventStream(new EventStream(tmp))) { eventsToCompare = index(readStream, predicateIndex); readHash = readStream.calculateHashSum(); } tmp.delete(); if (readHash.compareTo(writeHash) != 0) throw new IOException("Event hash for writing and reading events did not match."); display("done.\n"); if (sort) { display("Sorting and merging events... "); } else { display("Collecting events... "); } sortAndMerge(eventsToCompare,sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); }
TwoPassDataIndexer extends AbstractDataIndexer { @Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with TwoPass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String,Integer> predicateIndex = new HashMap<>(); File tmp = File.createTempFile("events", null); tmp.deleteOnExit(); int numEvents; BigInteger writeHash; HashSumEventStream writeEventStream = new HashSumEventStream(eventStream); try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) { numEvents = computeEventCounts(writeEventStream, dos, predicateIndex, cutoff); } writeHash = writeEventStream.calculateHashSum(); display("done. " + numEvents + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare; BigInteger readHash = null; try (HashSumEventStream readStream = new HashSumEventStream(new EventStream(tmp))) { eventsToCompare = index(readStream, predicateIndex); readHash = readStream.calculateHashSum(); } tmp.delete(); if (readHash.compareTo(writeHash) != 0) throw new IOException("Event hash for writing and reading events did not match."); display("done.\n"); if (sort) { display("Sorting and merging events... "); } else { display("Collecting events... "); } sortAndMerge(eventsToCompare,sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); } }
TwoPassDataIndexer extends AbstractDataIndexer { @Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with TwoPass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String,Integer> predicateIndex = new HashMap<>(); File tmp = File.createTempFile("events", null); tmp.deleteOnExit(); int numEvents; BigInteger writeHash; HashSumEventStream writeEventStream = new HashSumEventStream(eventStream); try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) { numEvents = computeEventCounts(writeEventStream, dos, predicateIndex, cutoff); } writeHash = writeEventStream.calculateHashSum(); display("done. " + numEvents + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare; BigInteger readHash = null; try (HashSumEventStream readStream = new HashSumEventStream(new EventStream(tmp))) { eventsToCompare = index(readStream, predicateIndex); readHash = readStream.calculateHashSum(); } tmp.delete(); if (readHash.compareTo(writeHash) != 0) throw new IOException("Event hash for writing and reading events did not match."); display("done.\n"); if (sort) { display("Sorting and merging events... "); } else { display("Collecting events... "); } sortAndMerge(eventsToCompare,sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); } TwoPassDataIndexer(); }
TwoPassDataIndexer extends AbstractDataIndexer { @Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with TwoPass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String,Integer> predicateIndex = new HashMap<>(); File tmp = File.createTempFile("events", null); tmp.deleteOnExit(); int numEvents; BigInteger writeHash; HashSumEventStream writeEventStream = new HashSumEventStream(eventStream); try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) { numEvents = computeEventCounts(writeEventStream, dos, predicateIndex, cutoff); } writeHash = writeEventStream.calculateHashSum(); display("done. " + numEvents + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare; BigInteger readHash = null; try (HashSumEventStream readStream = new HashSumEventStream(new EventStream(tmp))) { eventsToCompare = index(readStream, predicateIndex); readHash = readStream.calculateHashSum(); } tmp.delete(); if (readHash.compareTo(writeHash) != 0) throw new IOException("Event hash for writing and reading events did not match."); display("done.\n"); if (sort) { display("Sorting and merging events... "); } else { display("Collecting events... "); } sortAndMerge(eventsToCompare,sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); } TwoPassDataIndexer(); @Override void index(ObjectStream<Event> eventStream); }
TwoPassDataIndexer extends AbstractDataIndexer { @Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with TwoPass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String,Integer> predicateIndex = new HashMap<>(); File tmp = File.createTempFile("events", null); tmp.deleteOnExit(); int numEvents; BigInteger writeHash; HashSumEventStream writeEventStream = new HashSumEventStream(eventStream); try (DataOutputStream dos = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) { numEvents = computeEventCounts(writeEventStream, dos, predicateIndex, cutoff); } writeHash = writeEventStream.calculateHashSum(); display("done. " + numEvents + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare; BigInteger readHash = null; try (HashSumEventStream readStream = new HashSumEventStream(new EventStream(tmp))) { eventsToCompare = index(readStream, predicateIndex); readHash = readStream.calculateHashSum(); } tmp.delete(); if (readHash.compareTo(writeHash) != 0) throw new IOException("Event hash for writing and reading events did not match."); display("done.\n"); if (sort) { display("Sorting and merging events... "); } else { display("Collecting events... "); } sortAndMerge(eventsToCompare,sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); } TwoPassDataIndexer(); @Override void index(ObjectStream<Event> eventStream);  }