@Test public void testShinglerSimilarity() { logger.info("shinglerSimilarity"); String text1 = "This book has been written against a background of both reckless optimism and reckless despair. It holds that Progress and Doom are two sides of the same medal; that both are articles of superstition, not of faith. It was written out of the conviction that it should be possible to discover the hidden mechanics by which all traditional elements of our political and spiritual world were dissolved into a conglomeration where everything seems to have lost specific value, and has become unrecognizable for human comprehension, unusable for human purpose. Hannah Arendt, The Origins of Totalitarianism (New York: Harcourt Brace Jovanovich, Inc., 1973 ed.), p.vii, Preface to the First Edition."; String text2 = "The first edition of The Origins of Totalitarianism was written in 1950. Soon after the Second World War, this was a time of both reckless optimism and reckless despair. During this time, Dr. Arendt argues, the traditional elements of the political and spiritual world were dissolved into a conglomeration where everything seems to have lost specific value. In particular, the separation between the State and Society seems to have been destroyed. In this book, she seeks to disclose the hidden mechanics by which this transformation occurred."; int w = 3; double expResult = 0.15151515151515152; double result = TextSimilarity.shinglerSimilarity(text1, text2, w); assertEquals(expResult, result, Constants.DOUBLE_ACCURACY_HIGH); }
public static double shinglerSimilarity(String text1, String text2, int w) { preprocessDocument(text1); preprocessDocument(text2); NgramsExtractor.Parameters parameters = new NgramsExtractor.Parameters(); parameters.setMaxCombinations(w); parameters.setMaxDistanceBetweenKwds(0); parameters.setExaminationWindowLength(w); NgramsExtractor ngrams = new NgramsExtractor(parameters); Map<String, Double> keywords1 = ngrams.extract(text1); Map<String, Double> keywords2 = ngrams.extract(text2); filterKeywordCombinations(keywords1, w); filterKeywordCombinations(keywords2, w); double totalKeywords=0.0; double commonKeywords=0.0; Set<String> union = new HashSet<>(keywords1.keySet()); union.addAll(keywords2.keySet()); totalKeywords+=union.size(); Set<String> intersect = new HashSet<>(keywords1.keySet()); intersect.retainAll(keywords2.keySet()); commonKeywords+=intersect.size(); double resemblance=commonKeywords/totalKeywords; return resemblance; }
TextSimilarity { public static double shinglerSimilarity(String text1, String text2, int w) { preprocessDocument(text1); preprocessDocument(text2); NgramsExtractor.Parameters parameters = new NgramsExtractor.Parameters(); parameters.setMaxCombinations(w); parameters.setMaxDistanceBetweenKwds(0); parameters.setExaminationWindowLength(w); NgramsExtractor ngrams = new NgramsExtractor(parameters); Map<String, Double> keywords1 = ngrams.extract(text1); Map<String, Double> keywords2 = ngrams.extract(text2); filterKeywordCombinations(keywords1, w); filterKeywordCombinations(keywords2, w); double totalKeywords=0.0; double commonKeywords=0.0; Set<String> union = new HashSet<>(keywords1.keySet()); union.addAll(keywords2.keySet()); totalKeywords+=union.size(); Set<String> intersect = new HashSet<>(keywords1.keySet()); intersect.retainAll(keywords2.keySet()); commonKeywords+=intersect.size(); double resemblance=commonKeywords/totalKeywords; return resemblance; } }
TextSimilarity { public static double shinglerSimilarity(String text1, String text2, int w) { preprocessDocument(text1); preprocessDocument(text2); NgramsExtractor.Parameters parameters = new NgramsExtractor.Parameters(); parameters.setMaxCombinations(w); parameters.setMaxDistanceBetweenKwds(0); parameters.setExaminationWindowLength(w); NgramsExtractor ngrams = new NgramsExtractor(parameters); Map<String, Double> keywords1 = ngrams.extract(text1); Map<String, Double> keywords2 = ngrams.extract(text2); filterKeywordCombinations(keywords1, w); filterKeywordCombinations(keywords2, w); double totalKeywords=0.0; double commonKeywords=0.0; Set<String> union = new HashSet<>(keywords1.keySet()); union.addAll(keywords2.keySet()); totalKeywords+=union.size(); Set<String> intersect = new HashSet<>(keywords1.keySet()); intersect.retainAll(keywords2.keySet()); commonKeywords+=intersect.size(); double resemblance=commonKeywords/totalKeywords; return resemblance; }  }
TextSimilarity { public static double shinglerSimilarity(String text1, String text2, int w) { preprocessDocument(text1); preprocessDocument(text2); NgramsExtractor.Parameters parameters = new NgramsExtractor.Parameters(); parameters.setMaxCombinations(w); parameters.setMaxDistanceBetweenKwds(0); parameters.setExaminationWindowLength(w); NgramsExtractor ngrams = new NgramsExtractor(parameters); Map<String, Double> keywords1 = ngrams.extract(text1); Map<String, Double> keywords2 = ngrams.extract(text2); filterKeywordCombinations(keywords1, w); filterKeywordCombinations(keywords2, w); double totalKeywords=0.0; double commonKeywords=0.0; Set<String> union = new HashSet<>(keywords1.keySet()); union.addAll(keywords2.keySet()); totalKeywords+=union.size(); Set<String> intersect = new HashSet<>(keywords1.keySet()); intersect.retainAll(keywords2.keySet()); commonKeywords+=intersect.size(); double resemblance=commonKeywords/totalKeywords; return resemblance; }  static double oliverSimilarity(String text1, String text2); static double shinglerSimilarity(String text1, String text2, int w); }
TextSimilarity { public static double shinglerSimilarity(String text1, String text2, int w) { preprocessDocument(text1); preprocessDocument(text2); NgramsExtractor.Parameters parameters = new NgramsExtractor.Parameters(); parameters.setMaxCombinations(w); parameters.setMaxDistanceBetweenKwds(0); parameters.setExaminationWindowLength(w); NgramsExtractor ngrams = new NgramsExtractor(parameters); Map<String, Double> keywords1 = ngrams.extract(text1); Map<String, Double> keywords2 = ngrams.extract(text2); filterKeywordCombinations(keywords1, w); filterKeywordCombinations(keywords2, w); double totalKeywords=0.0; double commonKeywords=0.0; Set<String> union = new HashSet<>(keywords1.keySet()); union.addAll(keywords2.keySet()); totalKeywords+=union.size(); Set<String> intersect = new HashSet<>(keywords1.keySet()); intersect.retainAll(keywords2.keySet()); commonKeywords+=intersect.size(); double resemblance=commonKeywords/totalKeywords; return resemblance; }  static double oliverSimilarity(String text1, String text2); static double shinglerSimilarity(String text1, String text2, int w);  }