@Test public void filterProbability_ngrams() { UnigramThresholdFilter target = new UnigramThresholdFilter(frequencyMap, probabilityParameterSet); Parsing parsing = new Parsing(tokens, 14); String [] expected = new String [] {"Software Developer", "engineer", "quality", "assurance"}; TreeSet<Parsing> parsings = new TreeSet<Parsing>(); parsings.add(parsing); target.filterParsings(parsings, new ArrayList<TreeMap<String,Object>>()); String [] actual = parsings.first().toStringArray(); for(int i = 0; i < actual.length; ++i) { Assert.assertTrue(expected[i].compareTo(actual[i])== 0); } }
public void filterParsings(TreeSet<Parsing> parsings, List<TreeMap<String,Object>> output){ if(returnScoringOutput) { TreeMap<String, Object> frequencyInformation = new TreeMap<String, Object>(); frequencyInformation.put("total_unigrams", frequencyMap.descriptionTermFrequencyMap.get(NGramField.content)); frequencyInformation.put("single_occurrence_probability", 1 / (double)frequencyMap.totalDescriptionTermFrequencyMap.get(NGramField.content)); frequencyInformation.put("probability_threshold", probabilityThreshold); frequencyInformation.put("max_frequency_below_p_threshold", Math.floor(probabilityThreshold*frequencyMap.totalDescriptionTermFrequencyMap.get(NGramField.content))); frequencyInformation.put("frequency_threshold", frequencyThreshold); frequencyInformation.put("threshold_type", thresholdType); output.add(frequencyInformation); } int parsingCount = 0; for(Iterator<Parsing> iter = parsings.descendingIterator(); iter.hasNext() && parsingCount < parameterSet.resultLimit;) { filterParsing(iter.next()); parsingCount++; } }
UnigramThresholdFilter implements ITermFilter { public void filterParsings(TreeSet<Parsing> parsings, List<TreeMap<String,Object>> output){ if(returnScoringOutput) { TreeMap<String, Object> frequencyInformation = new TreeMap<String, Object>(); frequencyInformation.put("total_unigrams", frequencyMap.descriptionTermFrequencyMap.get(NGramField.content)); frequencyInformation.put("single_occurrence_probability", 1 / (double)frequencyMap.totalDescriptionTermFrequencyMap.get(NGramField.content)); frequencyInformation.put("probability_threshold", probabilityThreshold); frequencyInformation.put("max_frequency_below_p_threshold", Math.floor(probabilityThreshold*frequencyMap.totalDescriptionTermFrequencyMap.get(NGramField.content))); frequencyInformation.put("frequency_threshold", frequencyThreshold); frequencyInformation.put("threshold_type", thresholdType); output.add(frequencyInformation); } int parsingCount = 0; for(Iterator<Parsing> iter = parsings.descendingIterator(); iter.hasNext() && parsingCount < parameterSet.resultLimit;) { filterParsing(iter.next()); parsingCount++; } } }
UnigramThresholdFilter implements ITermFilter { public void filterParsings(TreeSet<Parsing> parsings, List<TreeMap<String,Object>> output){ if(returnScoringOutput) { TreeMap<String, Object> frequencyInformation = new TreeMap<String, Object>(); frequencyInformation.put("total_unigrams", frequencyMap.descriptionTermFrequencyMap.get(NGramField.content)); frequencyInformation.put("single_occurrence_probability", 1 / (double)frequencyMap.totalDescriptionTermFrequencyMap.get(NGramField.content)); frequencyInformation.put("probability_threshold", probabilityThreshold); frequencyInformation.put("max_frequency_below_p_threshold", Math.floor(probabilityThreshold*frequencyMap.totalDescriptionTermFrequencyMap.get(NGramField.content))); frequencyInformation.put("frequency_threshold", frequencyThreshold); frequencyInformation.put("threshold_type", thresholdType); output.add(frequencyInformation); } int parsingCount = 0; for(Iterator<Parsing> iter = parsings.descendingIterator(); iter.hasNext() && parsingCount < parameterSet.resultLimit;) { filterParsing(iter.next()); parsingCount++; } } UnigramThresholdFilter(FrequencyMap frequencyMap,
                                  ParseParameterSet parameterSet); }
UnigramThresholdFilter implements ITermFilter { public void filterParsings(TreeSet<Parsing> parsings, List<TreeMap<String,Object>> output){ if(returnScoringOutput) { TreeMap<String, Object> frequencyInformation = new TreeMap<String, Object>(); frequencyInformation.put("total_unigrams", frequencyMap.descriptionTermFrequencyMap.get(NGramField.content)); frequencyInformation.put("single_occurrence_probability", 1 / (double)frequencyMap.totalDescriptionTermFrequencyMap.get(NGramField.content)); frequencyInformation.put("probability_threshold", probabilityThreshold); frequencyInformation.put("max_frequency_below_p_threshold", Math.floor(probabilityThreshold*frequencyMap.totalDescriptionTermFrequencyMap.get(NGramField.content))); frequencyInformation.put("frequency_threshold", frequencyThreshold); frequencyInformation.put("threshold_type", thresholdType); output.add(frequencyInformation); } int parsingCount = 0; for(Iterator<Parsing> iter = parsings.descendingIterator(); iter.hasNext() && parsingCount < parameterSet.resultLimit;) { filterParsing(iter.next()); parsingCount++; } } UnigramThresholdFilter(FrequencyMap frequencyMap,
                                  ParseParameterSet parameterSet); void filterParsings(TreeSet<Parsing> parsings, List<TreeMap<String,Object>> output); }
UnigramThresholdFilter implements ITermFilter { public void filterParsings(TreeSet<Parsing> parsings, List<TreeMap<String,Object>> output){ if(returnScoringOutput) { TreeMap<String, Object> frequencyInformation = new TreeMap<String, Object>(); frequencyInformation.put("total_unigrams", frequencyMap.descriptionTermFrequencyMap.get(NGramField.content)); frequencyInformation.put("single_occurrence_probability", 1 / (double)frequencyMap.totalDescriptionTermFrequencyMap.get(NGramField.content)); frequencyInformation.put("probability_threshold", probabilityThreshold); frequencyInformation.put("max_frequency_below_p_threshold", Math.floor(probabilityThreshold*frequencyMap.totalDescriptionTermFrequencyMap.get(NGramField.content))); frequencyInformation.put("frequency_threshold", frequencyThreshold); frequencyInformation.put("threshold_type", thresholdType); output.add(frequencyInformation); } int parsingCount = 0; for(Iterator<Parsing> iter = parsings.descendingIterator(); iter.hasNext() && parsingCount < parameterSet.resultLimit;) { filterParsing(iter.next()); parsingCount++; } } UnigramThresholdFilter(FrequencyMap frequencyMap,
                                  ParseParameterSet parameterSet); void filterParsings(TreeSet<Parsing> parsings, List<TreeMap<String,Object>> output);  }