@Test void testDocument() throws RedPenException { WordFrequencyValidator validator = (WordFrequencyValidator) ValidatorFactory.getInstance("WordFrequency"); Document document = buildDocument(); List<ValidationError> errors = new ArrayList<>(); validator.setErrorList(errors); validator.validate(document); assertEquals(1, errors.size()); }
@Override public void validate(Document document) { lastSentence = null; for (int i = 0; i < document.size(); i++) { for (Paragraph para : document.getSection(i).getParagraphs()) { for (Sentence sentence : para.getSentences()) { processSentence(sentence); lastSentence = sentence; } } } if (wordCount >= getInt("min_word_count")) { Map<String, Double> documentWordFrequencies = new HashMap<>(); documentWordOccurances.forEach((word, count) -> { documentWordFrequencies.put(word, 100.0 * (double) count / (double) wordCount); }); DecimalFormat df = new DecimalFormat("0.00"); Map<String, Double> documentDeviations = new HashMap<>(); double stddev = initDeviations(documentWordFrequencies, documentDeviations); documentDeviations.forEach((word, deviation) -> { Double referenceDeviation = referenceWordDeviations.get(word); if (referenceDeviation != null) { double devRatio = deviation / stddev; double docPercentage = documentWordFrequencies.get(word); double referencePercentage = referenceWordFrequencies.get(word); if ((devRatio > 1) && (docPercentage > referencePercentage * getFloat("deviation_factor"))) { addLocalizedError( "WordUsedTooFrequently", lastSentence, word, df.format(docPercentage), df.format(referencePercentage)); } } }); } }
WordFrequencyValidator extends SpellingDictionaryValidator { @Override public void validate(Document document) { lastSentence = null; for (int i = 0; i < document.size(); i++) { for (Paragraph para : document.getSection(i).getParagraphs()) { for (Sentence sentence : para.getSentences()) { processSentence(sentence); lastSentence = sentence; } } } if (wordCount >= getInt("min_word_count")) { Map<String, Double> documentWordFrequencies = new HashMap<>(); documentWordOccurances.forEach((word, count) -> { documentWordFrequencies.put(word, 100.0 * (double) count / (double) wordCount); }); DecimalFormat df = new DecimalFormat("0.00"); Map<String, Double> documentDeviations = new HashMap<>(); double stddev = initDeviations(documentWordFrequencies, documentDeviations); documentDeviations.forEach((word, deviation) -> { Double referenceDeviation = referenceWordDeviations.get(word); if (referenceDeviation != null) { double devRatio = deviation / stddev; double docPercentage = documentWordFrequencies.get(word); double referencePercentage = referenceWordFrequencies.get(word); if ((devRatio > 1) && (docPercentage > referencePercentage * getFloat("deviation_factor"))) { addLocalizedError( "WordUsedTooFrequently", lastSentence, word, df.format(docPercentage), df.format(referencePercentage)); } } }); } } }
WordFrequencyValidator extends SpellingDictionaryValidator { @Override public void validate(Document document) { lastSentence = null; for (int i = 0; i < document.size(); i++) { for (Paragraph para : document.getSection(i).getParagraphs()) { for (Sentence sentence : para.getSentences()) { processSentence(sentence); lastSentence = sentence; } } } if (wordCount >= getInt("min_word_count")) { Map<String, Double> documentWordFrequencies = new HashMap<>(); documentWordOccurances.forEach((word, count) -> { documentWordFrequencies.put(word, 100.0 * (double) count / (double) wordCount); }); DecimalFormat df = new DecimalFormat("0.00"); Map<String, Double> documentDeviations = new HashMap<>(); double stddev = initDeviations(documentWordFrequencies, documentDeviations); documentDeviations.forEach((word, deviation) -> { Double referenceDeviation = referenceWordDeviations.get(word); if (referenceDeviation != null) { double devRatio = deviation / stddev; double docPercentage = documentWordFrequencies.get(word); double referencePercentage = referenceWordFrequencies.get(word); if ((devRatio > 1) && (docPercentage > referencePercentage * getFloat("deviation_factor"))) { addLocalizedError( "WordUsedTooFrequently", lastSentence, word, df.format(docPercentage), df.format(referencePercentage)); } } }); } } WordFrequencyValidator(); }
WordFrequencyValidator extends SpellingDictionaryValidator { @Override public void validate(Document document) { lastSentence = null; for (int i = 0; i < document.size(); i++) { for (Paragraph para : document.getSection(i).getParagraphs()) { for (Sentence sentence : para.getSentences()) { processSentence(sentence); lastSentence = sentence; } } } if (wordCount >= getInt("min_word_count")) { Map<String, Double> documentWordFrequencies = new HashMap<>(); documentWordOccurances.forEach((word, count) -> { documentWordFrequencies.put(word, 100.0 * (double) count / (double) wordCount); }); DecimalFormat df = new DecimalFormat("0.00"); Map<String, Double> documentDeviations = new HashMap<>(); double stddev = initDeviations(documentWordFrequencies, documentDeviations); documentDeviations.forEach((word, deviation) -> { Double referenceDeviation = referenceWordDeviations.get(word); if (referenceDeviation != null) { double devRatio = deviation / stddev; double docPercentage = documentWordFrequencies.get(word); double referencePercentage = referenceWordFrequencies.get(word); if ((devRatio > 1) && (docPercentage > referencePercentage * getFloat("deviation_factor"))) { addLocalizedError( "WordUsedTooFrequently", lastSentence, word, df.format(docPercentage), df.format(referencePercentage)); } } }); } } WordFrequencyValidator(); @Override void validate(Document document); }
WordFrequencyValidator extends SpellingDictionaryValidator { @Override public void validate(Document document) { lastSentence = null; for (int i = 0; i < document.size(); i++) { for (Paragraph para : document.getSection(i).getParagraphs()) { for (Sentence sentence : para.getSentences()) { processSentence(sentence); lastSentence = sentence; } } } if (wordCount >= getInt("min_word_count")) { Map<String, Double> documentWordFrequencies = new HashMap<>(); documentWordOccurances.forEach((word, count) -> { documentWordFrequencies.put(word, 100.0 * (double) count / (double) wordCount); }); DecimalFormat df = new DecimalFormat("0.00"); Map<String, Double> documentDeviations = new HashMap<>(); double stddev = initDeviations(documentWordFrequencies, documentDeviations); documentDeviations.forEach((word, deviation) -> { Double referenceDeviation = referenceWordDeviations.get(word); if (referenceDeviation != null) { double devRatio = deviation / stddev; double docPercentage = documentWordFrequencies.get(word); double referencePercentage = referenceWordFrequencies.get(word); if ((devRatio > 1) && (docPercentage > referencePercentage * getFloat("deviation_factor"))) { addLocalizedError( "WordUsedTooFrequently", lastSentence, word, df.format(docPercentage), df.format(referencePercentage)); } } }); } } WordFrequencyValidator(); @Override void validate(Document document);  }