@Test public void wordOverlapTest() { Sentence s1 = DocumentFactory.fromText("This text is about word overlapping.").getSentence(0); Sentence s2 = DocumentFactory.fromText("This text is not about word overlapping.").getSentence(0); assertEquals(36, WordHelpers.getSpanOverlapLength(s1, s2)); assertEquals( 4, WordHelpers.getSpanOverlapLength(s1.getToken(0), s2.getToken(0))); assertEquals( 4, WordHelpers.getSpanOverlapLength(s2.getToken(0), s1.getToken(0))); assertEquals( 4, WordHelpers.getSpanOverlapLength(s1.getToken(1), s2.getToken(1))); assertEquals( 3, WordHelpers.getSpanOverlapLength(s1.getToken(3), s2.getToken(3))); assertEquals( 0, WordHelpers.getSpanOverlapLength(s1.getToken(1), s2.getToken(2))); assertEquals( 4, WordHelpers.getSpanOverlapLength(s1.getToken(4), s2.getToken(4))); assertEquals( 4, WordHelpers.getSpanOverlapLength(s1.getToken(5), s2.getToken(5))); assertEquals( 6, WordHelpers.getSpanOverlapLength(s1.getToken(5), s2.getToken(6))); assertEquals( 1, WordHelpers.getSpanOverlapLength(s1.getToken(6), s2.getToken(6))); assertEquals( 0, WordHelpers.getSpanOverlapLength(s1.getToken(0), s2.getToken(6))); assertEquals( 0, WordHelpers.getSpanOverlapLength(s1.getToken(6), s2.getToken(0))); }
public static int getSpanOverlapLength(Span a, Span b) { int begin = Math.max(a.getBegin(), b.getBegin()); int end = Math.min(a.getEnd(), b.getEnd()); if(begin < end) return end - begin; else return 0; }
WordHelpers { public static int getSpanOverlapLength(Span a, Span b) { int begin = Math.max(a.getBegin(), b.getBegin()); int end = Math.min(a.getEnd(), b.getEnd()); if(begin < end) return end - begin; else return 0; } }
WordHelpers { public static int getSpanOverlapLength(Span a, Span b) { int begin = Math.max(a.getBegin(), b.getBegin()); int end = Math.min(a.getEnd(), b.getEnd()); if(begin < end) return end - begin; else return 0; } WordHelpers(Language lang); }
WordHelpers { public static int getSpanOverlapLength(Span a, Span b) { int begin = Math.max(a.getBegin(), b.getBegin()); int end = Math.min(a.getEnd(), b.getEnd()); if(begin < end) return end - begin; else return 0; } WordHelpers(Language lang); static Language getLanguage(String language); List<String> getStopWords(); boolean isStopWord(String word, TokenPreProcess pre); boolean isStopWord(String word); static String wordsToText(Iterable<Token> tokens); static String tokensToText(Iterable<Token> tokens, int beginOffset); static double cosineSim(INDArray arr1, INDArray arr2); static String vecToString(INDArray vec); static INDArray stringToVec(String str); static String replaceAccents(String str); static String replaceUmlauts(String str); static String replaceDashes(String str, String rep); static String replacePunctuation(String str, String rep); static String replaceNumbers(String str, String rep); static String replaceSpaces(String str, String rep); static String[] splitSpaces(String str); static int getSpanOverlapLength(Span a, Span b); }
WordHelpers { public static int getSpanOverlapLength(Span a, Span b) { int begin = Math.max(a.getBegin(), b.getBegin()); int end = Math.min(a.getEnd(), b.getEnd()); if(begin < end) return end - begin; else return 0; } WordHelpers(Language lang); static Language getLanguage(String language); List<String> getStopWords(); boolean isStopWord(String word, TokenPreProcess pre); boolean isStopWord(String word); static String wordsToText(Iterable<Token> tokens); static String tokensToText(Iterable<Token> tokens, int beginOffset); static double cosineSim(INDArray arr1, INDArray arr2); static String vecToString(INDArray vec); static INDArray stringToVec(String str); static String replaceAccents(String str); static String replaceUmlauts(String str); static String replaceDashes(String str, String rep); static String replacePunctuation(String str, String rep); static String replaceNumbers(String str, String rep); static String replaceSpaces(String str, String rep); static String[] splitSpaces(String str); static int getSpanOverlapLength(Span a, Span b); static HashSet<String> skipSpaceBefore; static HashSet<String> skipSpaceAfter; static final Pattern dashPattern; static final Pattern punctPattern; static final Pattern spacePattern; static final Pattern numericPattern; static final Pattern bracketsPattern; static HashSet<String> abbreviationsEN; static HashSet<String> abbreviationsDE; }