ContextualOutlierDetector { public Map<Context, List<OutlierClassificationResult>> searchContextualOutliers(List<ContextualDatum> data) throws Exception { Stopwatch sw = Stopwatch.createUnstarted(); log.debug("Find global context outliers on data num tuples: {} , MBs {} ", data.size()); sw.start(); HashSet<ContextualDatum> sample = randomSampling(data, 100); globalContext = new Context(sample, densityPruning, dependencyPruning, alpha); contextualOutlierDetection(data, globalContext); sw.stop(); long globalOutlierDetecionTime = sw.elapsed(TimeUnit.MILLISECONDS); sw.reset(); log.debug("Done global context outlier remaining data size {} : (duration: {}ms)", data.size(), globalOutlierDetecionTime); List<LatticeNode> preLatticeNodes = new ArrayList<LatticeNode>(); List<LatticeNode> curLatticeNodes = new ArrayList<LatticeNode>(); for (int level = 1; level <= totalContextualDimensions; level++) { if (level > maxPredicates) break; log.debug("Build {}-dimensional contexts on all attributes", level); sw.start(); if (level == 1) { curLatticeNodes = buildOneDimensionalLatticeNodes(data); } else { curLatticeNodes = levelUpLattice(preLatticeNodes, data); } sw.stop(); long latticeNodesBuildTimeCurLevel = sw.elapsed(TimeUnit.MILLISECONDS); sw.reset(); log.debug("Done building {}-dimensional contexts on all attributes (duration: {}ms)", level, latticeNodesBuildTimeCurLevel); log.debug("Memory Usage: {}", MemoryUtil.checkMemoryUsage()); if (curLatticeNodes.size() == 0) { log.debug("No more dense contexts, thus no need to level up anymore"); break; } log.debug("Find {}-dimensional contextual outliers", level); sw.start(); int numDenseContextsCurLevel = 0; for (LatticeNode node : curLatticeNodes) { for (Context context : node.getDenseContexts()) { contextualOutlierDetection(data, context); numDenseContextsCurLevel++; } } sw.stop(); long contextualOutlierDetectionTimeCurLevel = sw.elapsed(TimeUnit.MILLISECONDS); sw.reset(); log.debug("Done Find {}-dimensional contextual outliers (duration: {}ms)", level, contextualOutlierDetectionTimeCurLevel); log.debug( "Done Find {}-dimensional contextual outliers, there are {} dense contexts(average duration per context: {}ms)", level, numDenseContextsCurLevel, (numDenseContextsCurLevel == 0) ? 0 : contextualOutlierDetectionTimeCurLevel / numDenseContextsCurLevel); log.debug("Done Find {}-dimensional contextual outliers, densityPruning2: {}, " + "numOutlierDetectionRunsWithoutTrainingWithoutScoring: {}, " + "numOutlierDetectionRunsWithoutTrainingWithScoring: {}, " + "numOutlierDetectionRunsWithTrainingWithScoring: {}", level, densityPruning2, numOutlierDetectionRunsWithoutTrainingWithoutScoring, numOutlierDetectionRunsWithoutTrainingWithScoring, numOutlierDetectionRunsWithTrainingWithScoring); log.debug("----------------------------------------------------------"); if (level >= 2) { for (LatticeNode node : preLatticeNodes) { for (Context context : node.getDenseContexts()) { context2BitSet.remove(context); } } } preLatticeNodes = curLatticeNodes; } return context2Outliers; } ContextualOutlierDetector(MacroBaseConf conf); Map<Context, List<OutlierClassificationResult>> searchContextualOutliers(List<ContextualDatum> data); Map<Context, List<OutlierClassificationResult>> searchContextGivenOutliers(List<ContextualDatum> data); Map<Context, List<OutlierClassificationResult>> searchContextGivenOutliers(List<ContextualDatum> data, List<ContextualDatum> inputOutliers); List<Datum> contextualOutlierDetection(List<ContextualDatum> data, Context context);  }
@Test public void testContextualDiscreteAttribute() throws Exception { MacroBaseConf conf = new MacroBaseConf(); conf.set(MacroBaseConf.METRICS, Arrays.asList("A1")); conf.set(MacroBaseConf.TRANSFORM_TYPE, "MAD"); List<String> contextualDiscreteAttributes = new ArrayList<String>(); contextualDiscreteAttributes.add("C1_Discrete"); List<String> contextualDoubleAttributes = new ArrayList<String>(); List<String> attributes = new ArrayList<>(); attributes.addAll(contextualDiscreteAttributes); attributes.addAll(contextualDoubleAttributes); conf.set(MacroBaseConf.ATTRIBUTES, attributes); conf.set(ContextualConf.CONTEXTUAL_DISCRETE_ATTRIBUTES, contextualDiscreteAttributes); conf.set(ContextualConf.CONTEXTUAL_DOUBLE_ATTRIBUTES, contextualDoubleAttributes); conf.set(ContextualConf.CONTEXTUAL_DENSECONTEXTTAU, 0.4); conf.set(ContextualConf.CONTEXTUAL_NUMINTERVALS, 10); conf.set(MacroBaseConf.OUTLIER_STATIC_THRESHOLD, 3.0); conf.getEncoder().recordAttributeName(1, "A1"); conf.getEncoder().recordAttributeName(2, "C1_Discrete"); ContextualOutlierDetector contextualDetector = new ContextualOutlierDetector(conf); List<Datum> data = new ArrayList<>(); for (int i = 0; i < 100; ++i) { double[] sample = new double[1]; sample[0] = i; Integer[] c1 = new Integer[1]; if (i < 5) { c1[0] = 1; } else if (i >= 5 && i < 50) { c1[0] = 2; } else { c1[0] = 1; } data.add(makeDatum(conf, new ArrayList<>(), new ArrayRealVector(sample), new ArrayList<>(Arrays.asList(c1)), new ArrayRealVector())); } ContextualTransformer transformer = new ContextualTransformer(conf); transformer.consume(data); List<ContextualDatum> cdata = transformer.getStream().drain(); Map<Context, List<OutlierClassificationResult>> context2Outliers = contextualDetector.searchContextualOutliers(cdata); TestCase.assertEquals(context2Outliers.size(), 1); for (Context context : context2Outliers.keySet()) { List<Interval> intervals = context.getIntervals(); TestCase.assertEquals(intervals.size(), 1); assertEquals(intervals.get(0).getColumnName(), "C1_Discrete"); TestCase.assertEquals(intervals.get(0) instanceof IntervalDiscrete, true); assertEquals("1", conf.getEncoder().getAttribute(((IntervalDiscrete) intervals.get(0)).getValue()).getValue()); } }
@Test public void testContextualDoubleAttribute() throws Exception { MacroBaseConf conf = new MacroBaseConf(); conf.set(MacroBaseConf.METRICS, Arrays.asList("A1")); conf.set(MacroBaseConf.TRANSFORM_TYPE, "MAD"); conf.set(MacroBaseConf.OUTLIER_STATIC_THRESHOLD, 3.0); List<String> contextualDiscreteAttributes = new ArrayList<String>(); List<String> contextualDoubleAttributes = new ArrayList<String>(); contextualDoubleAttributes.add("C1_Double"); List<String> attributes = new ArrayList<>(); attributes.addAll(contextualDiscreteAttributes); attributes.addAll(contextualDoubleAttributes); conf.set(MacroBaseConf.ATTRIBUTES, attributes); conf.set(ContextualConf.CONTEXTUAL_DISCRETE_ATTRIBUTES, contextualDiscreteAttributes); conf.set(ContextualConf.CONTEXTUAL_DOUBLE_ATTRIBUTES, contextualDoubleAttributes); conf.set(ContextualConf.CONTEXTUAL_DENSECONTEXTTAU, 0.4); conf.set(ContextualConf.CONTEXTUAL_NUMINTERVALS, 10); ContextualOutlierDetector contextualDetector = new ContextualOutlierDetector(conf); List<Datum> data = new ArrayList<>(); for (int i = 0; i < 100; ++i) { double[] sample = new double[1]; sample[0] = i; double[] c1 = new double[1]; if (i < 5) { c1[0] = 1; } else if (i >= 5 && i < 50) { c1[0] = 100; } else { c1[0] = 1; } data.add(makeDatum(conf, new ArrayList<>(), new ArrayRealVector(sample), new ArrayList<Integer>(), new ArrayRealVector(c1))); } ContextualTransformer transformer = new ContextualTransformer(conf); transformer.consume(data); List<ContextualDatum> cdata = transformer.getStream().drain(); Map<Context, List<OutlierClassificationResult>> context2Outliers = contextualDetector.searchContextualOutliers(cdata); TestCase.assertEquals(context2Outliers.size(), 1); for (Context context : context2Outliers.keySet()) { List<Interval> intervals = context.getIntervals(); TestCase.assertEquals(intervals.size(), 1); assertEquals(intervals.get(0).getColumnName(), "C1_Double"); TestCase.assertEquals(intervals.get(0) instanceof IntervalDouble, true); assertEquals(((IntervalDouble) intervals.get(0)).getMin(), 1.0); assertEquals(((IntervalDouble) intervals.get(0)).getMax(), 10.9); } }