@Test public void testGetWordsInStringWithBigrams() throws Exception { String[] actual = FeatureExtractor.getWordsInStringWithBigrams("hello what is your name", false); String[] expected = {"hello", "your_name", "is", "hello_what", "what", "name", "your", "is_your", "what_is"}; assertArrayEquals(expected, actual); String[] actualEmpty = FeatureExtractor.getWordsInStringWithBigrams("", false); String[] expectedEmpty = {""}; assertArrayEquals(expectedEmpty, actualEmpty); }
static public String[] getWordsInStringWithBigrams(String inputText, boolean useStemming) { String text = inputText; text = text.toLowerCase(); String regexp = "(^|\\s)rt\\s|@\\S+|http\\S+|www\\.\\S+|[-.,;:_+?&=\"*~¨^´`<>\\[\\]{}()\\\\/|%€¤$£@!§½…]"; text = text.replaceAll(regexp, ""); String[] words = text.split("\\s+"); if (useStemming) { for (int i = 0; i < words.length; i++) { words[i] = naiveStemming(words[i]); } } HashSet<String> bigrams = new HashSet<String>(); for (int index = 0; index < words.length; index ++) { words[index] = words[index].replaceAll(SINGLE_QUOTE_PATTERN_AT_EXTREME,""); } for (int i = 0; i < words.length - 1; i++) { String w1 = words[i]; if (isStopword(w1)) { continue; } String w2 = ""; int j = i + 1; while (j < words.length && isStopword(w2 = words[j])) { j++; } if (!isStopword(w2)) { bigrams.add(w1 + "_" + w2); } } bigrams.addAll(Arrays.asList(words)); if (bigrams.isEmpty()) { return new String[0]; } else { return bigrams.toArray(new String[bigrams.size()]); } }
FeatureExtractor extends PipelineProcess { static public String[] getWordsInStringWithBigrams(String inputText, boolean useStemming) { String text = inputText; text = text.toLowerCase(); String regexp = "(^|\\s)rt\\s|@\\S+|http\\S+|www\\.\\S+|[-.,;:_+?&=\"*~¨^´`<>\\[\\]{}()\\\\/|%€¤$£@!§½…]"; text = text.replaceAll(regexp, ""); String[] words = text.split("\\s+"); if (useStemming) { for (int i = 0; i < words.length; i++) { words[i] = naiveStemming(words[i]); } } HashSet<String> bigrams = new HashSet<String>(); for (int index = 0; index < words.length; index ++) { words[index] = words[index].replaceAll(SINGLE_QUOTE_PATTERN_AT_EXTREME,""); } for (int i = 0; i < words.length - 1; i++) { String w1 = words[i]; if (isStopword(w1)) { continue; } String w2 = ""; int j = i + 1; while (j < words.length && isStopword(w2 = words[j])) { j++; } if (!isStopword(w2)) { bigrams.add(w1 + "_" + w2); } } bigrams.addAll(Arrays.asList(words)); if (bigrams.isEmpty()) { return new String[0]; } else { return bigrams.toArray(new String[bigrams.size()]); } } }
FeatureExtractor extends PipelineProcess { static public String[] getWordsInStringWithBigrams(String inputText, boolean useStemming) { String text = inputText; text = text.toLowerCase(); String regexp = "(^|\\s)rt\\s|@\\S+|http\\S+|www\\.\\S+|[-.,;:_+?&=\"*~¨^´`<>\\[\\]{}()\\\\/|%€¤$£@!§½…]"; text = text.replaceAll(regexp, ""); String[] words = text.split("\\s+"); if (useStemming) { for (int i = 0; i < words.length; i++) { words[i] = naiveStemming(words[i]); } } HashSet<String> bigrams = new HashSet<String>(); for (int index = 0; index < words.length; index ++) { words[index] = words[index].replaceAll(SINGLE_QUOTE_PATTERN_AT_EXTREME,""); } for (int i = 0; i < words.length - 1; i++) { String w1 = words[i]; if (isStopword(w1)) { continue; } String w2 = ""; int j = i + 1; while (j < words.length && isStopword(w2 = words[j])) { j++; } if (!isStopword(w2)) { bigrams.add(w1 + "_" + w2); } } bigrams.addAll(Arrays.asList(words)); if (bigrams.isEmpty()) { return new String[0]; } else { return bigrams.toArray(new String[bigrams.size()]); } }  }
FeatureExtractor extends PipelineProcess { static public String[] getWordsInStringWithBigrams(String inputText, boolean useStemming) { String text = inputText; text = text.toLowerCase(); String regexp = "(^|\\s)rt\\s|@\\S+|http\\S+|www\\.\\S+|[-.,;:_+?&=\"*~¨^´`<>\\[\\]{}()\\\\/|%€¤$£@!§½…]"; text = text.replaceAll(regexp, ""); String[] words = text.split("\\s+"); if (useStemming) { for (int i = 0; i < words.length; i++) { words[i] = naiveStemming(words[i]); } } HashSet<String> bigrams = new HashSet<String>(); for (int index = 0; index < words.length; index ++) { words[index] = words[index].replaceAll(SINGLE_QUOTE_PATTERN_AT_EXTREME,""); } for (int i = 0; i < words.length - 1; i++) { String w1 = words[i]; if (isStopword(w1)) { continue; } String w2 = ""; int j = i + 1; while (j < words.length && isStopword(w2 = words[j])) { j++; } if (!isStopword(w2)) { bigrams.add(w1 + "_" + w2); } } bigrams.addAll(Arrays.asList(words)); if (bigrams.isEmpty()) { return new String[0]; } else { return bigrams.toArray(new String[bigrams.size()]); } }  static String[] getWordsInStringWithBigrams(String inputText,
			boolean useStemming); static String naiveStemming(String str); }
FeatureExtractor extends PipelineProcess { static public String[] getWordsInStringWithBigrams(String inputText, boolean useStemming) { String text = inputText; text = text.toLowerCase(); String regexp = "(^|\\s)rt\\s|@\\S+|http\\S+|www\\.\\S+|[-.,;:_+?&=\"*~¨^´`<>\\[\\]{}()\\\\/|%€¤$£@!§½…]"; text = text.replaceAll(regexp, ""); String[] words = text.split("\\s+"); if (useStemming) { for (int i = 0; i < words.length; i++) { words[i] = naiveStemming(words[i]); } } HashSet<String> bigrams = new HashSet<String>(); for (int index = 0; index < words.length; index ++) { words[index] = words[index].replaceAll(SINGLE_QUOTE_PATTERN_AT_EXTREME,""); } for (int i = 0; i < words.length - 1; i++) { String w1 = words[i]; if (isStopword(w1)) { continue; } String w2 = ""; int j = i + 1; while (j < words.length && isStopword(w2 = words[j])) { j++; } if (!isStopword(w2)) { bigrams.add(w1 + "_" + w2); } } bigrams.addAll(Arrays.asList(words)); if (bigrams.isEmpty()) { return new String[0]; } else { return bigrams.toArray(new String[bigrams.size()]); } }  static String[] getWordsInStringWithBigrams(String inputText,
			boolean useStemming); static String naiveStemming(String str);  }