@Test void testNotDetectSingleJoshi() throws Exception { List<Document> documents = new ArrayList<>(); documents.add(Document.builder(new NeologdJapaneseTokenizer()) .addSection(1) .addParagraph() .addSentence(new Sentence("私は彼が好き。", 1)) .build()); Configuration config = Configuration.builder("ja") .addValidatorConfig(new ValidatorConfiguration("DoubledJoshi")) .build(); RedPen redPen = new RedPen(config); Map<Document, List<ValidationError>> errors = redPen.validate(documents); assertEquals(0, errors.get(documents.get(0)).size()); }
@Override public void validate(Sentence sentence) { Map<String, List<TokenElement>> counts = new HashMap<>(); Map<TokenElement, Integer> positions = new HashMap<>(); Set<String> skipList = getSet("list"); int id = 0; for (TokenElement tokenElement : sentence.getTokens()) { if (tokenElement.getTags().get(0).equals("助詞") && !skipList.contains(tokenElement.getSurface())) { if (!counts.containsKey(tokenElement.getSurface())) { counts.put(tokenElement.getSurface(), new LinkedList<>()); } counts.get(tokenElement.getSurface()).add(tokenElement); positions.put(tokenElement, id); id++; } } for (String joshi : counts.keySet()) { if (counts.get(joshi).size() < 2) { continue; } TokenElement prev = null; for (TokenElement token : counts.get(joshi)) { int prevPosition = getPosition(prev, positions); int currentPosition = getPosition(token, positions); prev = token; if (prevPosition == -1 || currentPosition == -1) { continue; } if ((currentPosition - prevPosition) <= getInt("min_interval")) { addLocalizedError(sentence, joshi); } } } }
DoubledJoshiValidator extends DictionaryValidator { @Override public void validate(Sentence sentence) { Map<String, List<TokenElement>> counts = new HashMap<>(); Map<TokenElement, Integer> positions = new HashMap<>(); Set<String> skipList = getSet("list"); int id = 0; for (TokenElement tokenElement : sentence.getTokens()) { if (tokenElement.getTags().get(0).equals("助詞") && !skipList.contains(tokenElement.getSurface())) { if (!counts.containsKey(tokenElement.getSurface())) { counts.put(tokenElement.getSurface(), new LinkedList<>()); } counts.get(tokenElement.getSurface()).add(tokenElement); positions.put(tokenElement, id); id++; } } for (String joshi : counts.keySet()) { if (counts.get(joshi).size() < 2) { continue; } TokenElement prev = null; for (TokenElement token : counts.get(joshi)) { int prevPosition = getPosition(prev, positions); int currentPosition = getPosition(token, positions); prev = token; if (prevPosition == -1 || currentPosition == -1) { continue; } if ((currentPosition - prevPosition) <= getInt("min_interval")) { addLocalizedError(sentence, joshi); } } } } }
DoubledJoshiValidator extends DictionaryValidator { @Override public void validate(Sentence sentence) { Map<String, List<TokenElement>> counts = new HashMap<>(); Map<TokenElement, Integer> positions = new HashMap<>(); Set<String> skipList = getSet("list"); int id = 0; for (TokenElement tokenElement : sentence.getTokens()) { if (tokenElement.getTags().get(0).equals("助詞") && !skipList.contains(tokenElement.getSurface())) { if (!counts.containsKey(tokenElement.getSurface())) { counts.put(tokenElement.getSurface(), new LinkedList<>()); } counts.get(tokenElement.getSurface()).add(tokenElement); positions.put(tokenElement, id); id++; } } for (String joshi : counts.keySet()) { if (counts.get(joshi).size() < 2) { continue; } TokenElement prev = null; for (TokenElement token : counts.get(joshi)) { int prevPosition = getPosition(prev, positions); int currentPosition = getPosition(token, positions); prev = token; if (prevPosition == -1 || currentPosition == -1) { continue; } if ((currentPosition - prevPosition) <= getInt("min_interval")) { addLocalizedError(sentence, joshi); } } } } DoubledJoshiValidator(); }
DoubledJoshiValidator extends DictionaryValidator { @Override public void validate(Sentence sentence) { Map<String, List<TokenElement>> counts = new HashMap<>(); Map<TokenElement, Integer> positions = new HashMap<>(); Set<String> skipList = getSet("list"); int id = 0; for (TokenElement tokenElement : sentence.getTokens()) { if (tokenElement.getTags().get(0).equals("助詞") && !skipList.contains(tokenElement.getSurface())) { if (!counts.containsKey(tokenElement.getSurface())) { counts.put(tokenElement.getSurface(), new LinkedList<>()); } counts.get(tokenElement.getSurface()).add(tokenElement); positions.put(tokenElement, id); id++; } } for (String joshi : counts.keySet()) { if (counts.get(joshi).size() < 2) { continue; } TokenElement prev = null; for (TokenElement token : counts.get(joshi)) { int prevPosition = getPosition(prev, positions); int currentPosition = getPosition(token, positions); prev = token; if (prevPosition == -1 || currentPosition == -1) { continue; } if ((currentPosition - prevPosition) <= getInt("min_interval")) { addLocalizedError(sentence, joshi); } } } } DoubledJoshiValidator(); @Override void validate(Sentence sentence); @Override List<String> getSupportedLanguages(); }
DoubledJoshiValidator extends DictionaryValidator { @Override public void validate(Sentence sentence) { Map<String, List<TokenElement>> counts = new HashMap<>(); Map<TokenElement, Integer> positions = new HashMap<>(); Set<String> skipList = getSet("list"); int id = 0; for (TokenElement tokenElement : sentence.getTokens()) { if (tokenElement.getTags().get(0).equals("助詞") && !skipList.contains(tokenElement.getSurface())) { if (!counts.containsKey(tokenElement.getSurface())) { counts.put(tokenElement.getSurface(), new LinkedList<>()); } counts.get(tokenElement.getSurface()).add(tokenElement); positions.put(tokenElement, id); id++; } } for (String joshi : counts.keySet()) { if (counts.get(joshi).size() < 2) { continue; } TokenElement prev = null; for (TokenElement token : counts.get(joshi)) { int prevPosition = getPosition(prev, positions); int currentPosition = getPosition(token, positions); prev = token; if (prevPosition == -1 || currentPosition == -1) { continue; } if ((currentPosition - prevPosition) <= getInt("min_interval")) { addLocalizedError(sentence, joshi); } } } } DoubledJoshiValidator(); @Override void validate(Sentence sentence); @Override List<String> getSupportedLanguages();  }