@Test public void testExtract() throws IOException { System.out.println("n-gram extraction"); String text = new String(Files.readAllBytes(smile.util.Paths.getTestData("text/turing.txt"))); PorterStemmer stemmer = new PorterStemmer(); SimpleTokenizer tokenizer = new SimpleTokenizer(); ArrayList<String[]> sentences = new ArrayList<>(); for (String paragraph : SimpleParagraphSplitter.getInstance().split(text)) { for (String s : SimpleSentenceSplitter.getInstance().split(paragraph)) { String[] sentence = tokenizer.split(s); for (int i = 0; i < sentence.length; i++) { sentence[i] = stemmer.stripPluralParticiple(sentence[i]).toLowerCase(); } sentences.add(sentence); } } NGram[][] result = NGram.of(sentences, 4, 4); assertEquals(5, result.length); for (NGram[] ngrams : result) { for (NGram ngram : ngrams) { System.out.println(ngram); } System.out.println(); } assertEquals(0, result[0].length); assertEquals(331, result[1].length); assertEquals(16, result[2].length); assertEquals(7, result[3].length); assertEquals(0, result[4].length); }
public static NGram[][] of(Collection<String[]> sentences, int maxNGramSize, int minFrequency) { ArrayList<Set<NGram>> features = new ArrayList<>(maxNGramSize + 1); Set<NGram> feature = new HashSet<>(); features.add(feature); EnglishPunctuations punctuations = EnglishPunctuations.getInstance(); for (int n = 1; n <= maxNGramSize; n++) { Map<smile.nlp.NGram, MutableInt> candidates = new HashMap<>(); for (String[] sentence : sentences) { for (int i = 0; i <= sentence.length - n; i++) { smile.nlp.NGram ngram = new smile.nlp.NGram(Arrays.copyOfRange(sentence, i, i+n)); boolean add = false; if (n == 1) { add = true; } else { NGram initialGram = new NGram(Arrays.copyOfRange(sentence, i, i+n-1), 0); NGram finalGram = new NGram(Arrays.copyOfRange(sentence, i+1, i+n), 0); if (feature.contains(initialGram) && feature.contains(finalGram)) { add = true; } } if (add) { MutableInt count = candidates.get(ngram); if (count == null) { candidates.put(ngram, new MutableInt(1)); } else { count.increment(); } } } } feature = new HashSet<>(); features.add(feature); for (Map.Entry<smile.nlp.NGram, MutableInt> entry : candidates.entrySet()) { MutableInt count = entry.getValue(); if (count.value >= minFrequency) { smile.nlp.NGram ngram = entry.getKey(); if (ngram.words.length == 1 && punctuations.contains(ngram.words[0])) { continue; } feature.add(new NGram(ngram.words, count.value)); } } } EnglishStopWords stopWords = EnglishStopWords.DEFAULT; return features.stream().map(ngrams -> { NGram[] collocations = ngrams.stream().filter(ngram -> { boolean stopWord = true; String[] words = ngram.words; if (!stopWords.contains(words[0]) && !stopWords.contains(words[words.length - 1])) { for (String word : words) { if (!stopWords.contains(word)) { stopWord = false; break; } } } return !stopWord; }).toArray(NGram[]::new); Arrays.sort(collocations, Collections.reverseOrder()); return collocations; }).toArray(NGram[][]::new); }
NGram extends smile.nlp.NGram implements Comparable<NGram> { public static NGram[][] of(Collection<String[]> sentences, int maxNGramSize, int minFrequency) { ArrayList<Set<NGram>> features = new ArrayList<>(maxNGramSize + 1); Set<NGram> feature = new HashSet<>(); features.add(feature); EnglishPunctuations punctuations = EnglishPunctuations.getInstance(); for (int n = 1; n <= maxNGramSize; n++) { Map<smile.nlp.NGram, MutableInt> candidates = new HashMap<>(); for (String[] sentence : sentences) { for (int i = 0; i <= sentence.length - n; i++) { smile.nlp.NGram ngram = new smile.nlp.NGram(Arrays.copyOfRange(sentence, i, i+n)); boolean add = false; if (n == 1) { add = true; } else { NGram initialGram = new NGram(Arrays.copyOfRange(sentence, i, i+n-1), 0); NGram finalGram = new NGram(Arrays.copyOfRange(sentence, i+1, i+n), 0); if (feature.contains(initialGram) && feature.contains(finalGram)) { add = true; } } if (add) { MutableInt count = candidates.get(ngram); if (count == null) { candidates.put(ngram, new MutableInt(1)); } else { count.increment(); } } } } feature = new HashSet<>(); features.add(feature); for (Map.Entry<smile.nlp.NGram, MutableInt> entry : candidates.entrySet()) { MutableInt count = entry.getValue(); if (count.value >= minFrequency) { smile.nlp.NGram ngram = entry.getKey(); if (ngram.words.length == 1 && punctuations.contains(ngram.words[0])) { continue; } feature.add(new NGram(ngram.words, count.value)); } } } EnglishStopWords stopWords = EnglishStopWords.DEFAULT; return features.stream().map(ngrams -> { NGram[] collocations = ngrams.stream().filter(ngram -> { boolean stopWord = true; String[] words = ngram.words; if (!stopWords.contains(words[0]) && !stopWords.contains(words[words.length - 1])) { for (String word : words) { if (!stopWords.contains(word)) { stopWord = false; break; } } } return !stopWord; }).toArray(NGram[]::new); Arrays.sort(collocations, Collections.reverseOrder()); return collocations; }).toArray(NGram[][]::new); } }
NGram extends smile.nlp.NGram implements Comparable<NGram> { public static NGram[][] of(Collection<String[]> sentences, int maxNGramSize, int minFrequency) { ArrayList<Set<NGram>> features = new ArrayList<>(maxNGramSize + 1); Set<NGram> feature = new HashSet<>(); features.add(feature); EnglishPunctuations punctuations = EnglishPunctuations.getInstance(); for (int n = 1; n <= maxNGramSize; n++) { Map<smile.nlp.NGram, MutableInt> candidates = new HashMap<>(); for (String[] sentence : sentences) { for (int i = 0; i <= sentence.length - n; i++) { smile.nlp.NGram ngram = new smile.nlp.NGram(Arrays.copyOfRange(sentence, i, i+n)); boolean add = false; if (n == 1) { add = true; } else { NGram initialGram = new NGram(Arrays.copyOfRange(sentence, i, i+n-1), 0); NGram finalGram = new NGram(Arrays.copyOfRange(sentence, i+1, i+n), 0); if (feature.contains(initialGram) && feature.contains(finalGram)) { add = true; } } if (add) { MutableInt count = candidates.get(ngram); if (count == null) { candidates.put(ngram, new MutableInt(1)); } else { count.increment(); } } } } feature = new HashSet<>(); features.add(feature); for (Map.Entry<smile.nlp.NGram, MutableInt> entry : candidates.entrySet()) { MutableInt count = entry.getValue(); if (count.value >= minFrequency) { smile.nlp.NGram ngram = entry.getKey(); if (ngram.words.length == 1 && punctuations.contains(ngram.words[0])) { continue; } feature.add(new NGram(ngram.words, count.value)); } } } EnglishStopWords stopWords = EnglishStopWords.DEFAULT; return features.stream().map(ngrams -> { NGram[] collocations = ngrams.stream().filter(ngram -> { boolean stopWord = true; String[] words = ngram.words; if (!stopWords.contains(words[0]) && !stopWords.contains(words[words.length - 1])) { for (String word : words) { if (!stopWords.contains(word)) { stopWord = false; break; } } } return !stopWord; }).toArray(NGram[]::new); Arrays.sort(collocations, Collections.reverseOrder()); return collocations; }).toArray(NGram[][]::new); } NGram(String[] words, int count); }
NGram extends smile.nlp.NGram implements Comparable<NGram> { public static NGram[][] of(Collection<String[]> sentences, int maxNGramSize, int minFrequency) { ArrayList<Set<NGram>> features = new ArrayList<>(maxNGramSize + 1); Set<NGram> feature = new HashSet<>(); features.add(feature); EnglishPunctuations punctuations = EnglishPunctuations.getInstance(); for (int n = 1; n <= maxNGramSize; n++) { Map<smile.nlp.NGram, MutableInt> candidates = new HashMap<>(); for (String[] sentence : sentences) { for (int i = 0; i <= sentence.length - n; i++) { smile.nlp.NGram ngram = new smile.nlp.NGram(Arrays.copyOfRange(sentence, i, i+n)); boolean add = false; if (n == 1) { add = true; } else { NGram initialGram = new NGram(Arrays.copyOfRange(sentence, i, i+n-1), 0); NGram finalGram = new NGram(Arrays.copyOfRange(sentence, i+1, i+n), 0); if (feature.contains(initialGram) && feature.contains(finalGram)) { add = true; } } if (add) { MutableInt count = candidates.get(ngram); if (count == null) { candidates.put(ngram, new MutableInt(1)); } else { count.increment(); } } } } feature = new HashSet<>(); features.add(feature); for (Map.Entry<smile.nlp.NGram, MutableInt> entry : candidates.entrySet()) { MutableInt count = entry.getValue(); if (count.value >= minFrequency) { smile.nlp.NGram ngram = entry.getKey(); if (ngram.words.length == 1 && punctuations.contains(ngram.words[0])) { continue; } feature.add(new NGram(ngram.words, count.value)); } } } EnglishStopWords stopWords = EnglishStopWords.DEFAULT; return features.stream().map(ngrams -> { NGram[] collocations = ngrams.stream().filter(ngram -> { boolean stopWord = true; String[] words = ngram.words; if (!stopWords.contains(words[0]) && !stopWords.contains(words[words.length - 1])) { for (String word : words) { if (!stopWords.contains(word)) { stopWord = false; break; } } } return !stopWord; }).toArray(NGram[]::new); Arrays.sort(collocations, Collections.reverseOrder()); return collocations; }).toArray(NGram[][]::new); } NGram(String[] words, int count); @Override String toString(); @Override int compareTo(NGram o); static NGram[][] of(Collection<String[]> sentences, int maxNGramSize, int minFrequency); }
NGram extends smile.nlp.NGram implements Comparable<NGram> { public static NGram[][] of(Collection<String[]> sentences, int maxNGramSize, int minFrequency) { ArrayList<Set<NGram>> features = new ArrayList<>(maxNGramSize + 1); Set<NGram> feature = new HashSet<>(); features.add(feature); EnglishPunctuations punctuations = EnglishPunctuations.getInstance(); for (int n = 1; n <= maxNGramSize; n++) { Map<smile.nlp.NGram, MutableInt> candidates = new HashMap<>(); for (String[] sentence : sentences) { for (int i = 0; i <= sentence.length - n; i++) { smile.nlp.NGram ngram = new smile.nlp.NGram(Arrays.copyOfRange(sentence, i, i+n)); boolean add = false; if (n == 1) { add = true; } else { NGram initialGram = new NGram(Arrays.copyOfRange(sentence, i, i+n-1), 0); NGram finalGram = new NGram(Arrays.copyOfRange(sentence, i+1, i+n), 0); if (feature.contains(initialGram) && feature.contains(finalGram)) { add = true; } } if (add) { MutableInt count = candidates.get(ngram); if (count == null) { candidates.put(ngram, new MutableInt(1)); } else { count.increment(); } } } } feature = new HashSet<>(); features.add(feature); for (Map.Entry<smile.nlp.NGram, MutableInt> entry : candidates.entrySet()) { MutableInt count = entry.getValue(); if (count.value >= minFrequency) { smile.nlp.NGram ngram = entry.getKey(); if (ngram.words.length == 1 && punctuations.contains(ngram.words[0])) { continue; } feature.add(new NGram(ngram.words, count.value)); } } } EnglishStopWords stopWords = EnglishStopWords.DEFAULT; return features.stream().map(ngrams -> { NGram[] collocations = ngrams.stream().filter(ngram -> { boolean stopWord = true; String[] words = ngram.words; if (!stopWords.contains(words[0]) && !stopWords.contains(words[words.length - 1])) { for (String word : words) { if (!stopWords.contains(word)) { stopWord = false; break; } } } return !stopWord; }).toArray(NGram[]::new); Arrays.sort(collocations, Collections.reverseOrder()); return collocations; }).toArray(NGram[][]::new); } NGram(String[] words, int count); @Override String toString(); @Override int compareTo(NGram o); static NGram[][] of(Collection<String[]> sentences, int maxNGramSize, int minFrequency); final int count; }