@Test public void testIndex() throws IOException { ObjectStream<Event> eventStream = new SimpleEventStreamBuilder() .add("other/w=he n1w=belongs n2w=to po=other pow=other,He powf=other,ic ppo=other") .add("other/w=belongs p1w=he n1w=to n2w=apache po=other pow=other,belongs powf=other,lc ppo=other") .add("other/w=to p1w=belongs p2w=he n1w=apache n2w=software po=other pow=other,to" + " powf=other,lc ppo=other") .add("org-start/w=apache p1w=to p2w=belongs n1w=software n2w=foundation po=other pow=other,Apache" + " powf=other,ic ppo=other") .add("org-cont/w=software p1w=apache p2w=to n1w=foundation n2w=. po=org-start" + " pow=org-start,Software powf=org-start,ic ppo=other") .add("org-cont/w=foundation p1w=software p2w=apache n1w=. po=org-cont pow=org-cont,Foundation" + " powf=org-cont,ic ppo=org-start") .add("other/w=. p1w=foundation p2w=software po=org-cont pow=org-cont,. powf=org-cont,other" + " ppo=org-cont") .build(); DataIndexer indexer = new OnePassDataIndexer(); indexer.init(new TrainingParameters(Collections.emptyMap()), null); indexer.index(eventStream); Assert.assertEquals(3, indexer.getContexts().length); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[0]); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[1]); Assert.assertArrayEquals(new int[]{0}, indexer.getContexts()[2]); Assert.assertNull(indexer.getValues()); Assert.assertEquals(5, indexer.getNumEvents()); Assert.assertArrayEquals(new int[]{0, 1, 2}, indexer.getOutcomeList()); Assert.assertArrayEquals(new int[]{3, 1, 1}, indexer.getNumTimesEventsSeen()); Assert.assertArrayEquals(new String[]{"ppo=other"}, indexer.getPredLabels()); Assert.assertArrayEquals(new String[]{"other", "org-start", "org-cont"}, indexer.getOutcomeLabels()); Assert.assertArrayEquals(new int[]{5}, indexer.getPredCounts()); }
@Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with OnePass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String, Integer> predicateIndex = new HashMap<>(); List<Event> events = computeEventCounts(eventStream, predicateIndex, cutoff); display("done. " + events.size() + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare = index(ObjectStreamUtils.createObjectStream(events), predicateIndex); display("done.\n"); display("Sorting and merging events... "); sortAndMerge(eventsToCompare, sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); }
OnePassDataIndexer extends AbstractDataIndexer { @Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with OnePass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String, Integer> predicateIndex = new HashMap<>(); List<Event> events = computeEventCounts(eventStream, predicateIndex, cutoff); display("done. " + events.size() + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare = index(ObjectStreamUtils.createObjectStream(events), predicateIndex); display("done.\n"); display("Sorting and merging events... "); sortAndMerge(eventsToCompare, sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); } }
OnePassDataIndexer extends AbstractDataIndexer { @Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with OnePass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String, Integer> predicateIndex = new HashMap<>(); List<Event> events = computeEventCounts(eventStream, predicateIndex, cutoff); display("done. " + events.size() + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare = index(ObjectStreamUtils.createObjectStream(events), predicateIndex); display("done.\n"); display("Sorting and merging events... "); sortAndMerge(eventsToCompare, sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); } OnePassDataIndexer(); }
OnePassDataIndexer extends AbstractDataIndexer { @Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with OnePass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String, Integer> predicateIndex = new HashMap<>(); List<Event> events = computeEventCounts(eventStream, predicateIndex, cutoff); display("done. " + events.size() + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare = index(ObjectStreamUtils.createObjectStream(events), predicateIndex); display("done.\n"); display("Sorting and merging events... "); sortAndMerge(eventsToCompare, sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); } OnePassDataIndexer(); @Override void index(ObjectStream<Event> eventStream); }
OnePassDataIndexer extends AbstractDataIndexer { @Override public void index(ObjectStream<Event> eventStream) throws IOException { int cutoff = trainingParameters.getIntParameter(CUTOFF_PARAM, CUTOFF_DEFAULT); boolean sort = trainingParameters.getBooleanParameter(SORT_PARAM, SORT_DEFAULT); long start = System.currentTimeMillis(); display("Indexing events with OnePass using cutoff of " + cutoff + "\n\n"); display("\tComputing event counts... "); Map<String, Integer> predicateIndex = new HashMap<>(); List<Event> events = computeEventCounts(eventStream, predicateIndex, cutoff); display("done. " + events.size() + " events\n"); display("\tIndexing... "); List<ComparableEvent> eventsToCompare = index(ObjectStreamUtils.createObjectStream(events), predicateIndex); display("done.\n"); display("Sorting and merging events... "); sortAndMerge(eventsToCompare, sort); display(String.format("Done indexing in %.2f s.\n", (System.currentTimeMillis() - start) / 1000d)); } OnePassDataIndexer(); @Override void index(ObjectStream<Event> eventStream);  }