DictionaryDetokenizer implements Detokenizer { public DetokenizationOperation[] detokenize(String[] tokens) { DetokenizationOperation operations[] = new DetokenizationOperation[tokens.length]; Set<String> matchingTokens = new HashSet<String>(); for (int i = 0; i < tokens.length; i++) { DetokenizationDictionary.Operation dictOperation = dict.getOperation(tokens[i]); if (dictOperation == null) { operations[i] = Detokenizer.DetokenizationOperation.NO_OPERATION; } else if (DetokenizationDictionary.Operation.MOVE_LEFT.equals(dictOperation)) { operations[i] = Detokenizer.DetokenizationOperation.MERGE_TO_LEFT; } else if (DetokenizationDictionary.Operation.MOVE_RIGHT.equals(dictOperation)) { operations[i] = Detokenizer.DetokenizationOperation.MERGE_TO_RIGHT; } else if (DetokenizationDictionary.Operation.MOVE_BOTH.equals(dictOperation)) { operations[i] = Detokenizer.DetokenizationOperation.MERGE_BOTH; } else if (DetokenizationDictionary.Operation.RIGHT_LEFT_MATCHING.equals(dictOperation)) { if (matchingTokens.contains(tokens[i])) { operations[i] = Detokenizer.DetokenizationOperation.MERGE_TO_LEFT; matchingTokens.remove(tokens[i]); } else { operations[i] = Detokenizer.DetokenizationOperation.MERGE_TO_RIGHT; matchingTokens.add(tokens[i]); } } else { throw new IllegalStateException("Unknown operation: " + dictOperation); } } return operations; } DictionaryDetokenizer(DetokenizationDictionary dict); DetokenizationOperation[] detokenize(String[] tokens); String detokenize(String tokens[], String splitMarker);  }
@Test public void testDetokenizer() { String tokens[] = new String[]{".", "!", "(", ")", "\"", "-"}; Operation operations[] = new Operation[]{ Operation.MOVE_LEFT, Operation.MOVE_LEFT, Operation.MOVE_RIGHT, Operation.MOVE_LEFT, Operation.RIGHT_LEFT_MATCHING, Operation.MOVE_BOTH }; DetokenizationDictionary dict = new DetokenizationDictionary(tokens, operations); Detokenizer detokenizer = new DictionaryDetokenizer(dict); DetokenizationOperation detokenizeOperations[] = detokenizer.detokenize(new String[]{"Simple", "test", ".", "co", "-", "worker"}); assertEquals(DetokenizationOperation.NO_OPERATION, detokenizeOperations[0]); assertEquals(DetokenizationOperation.NO_OPERATION, detokenizeOperations[1]); assertEquals(DetokenizationOperation.MERGE_TO_LEFT, detokenizeOperations[2]); assertEquals(DetokenizationOperation.NO_OPERATION, detokenizeOperations[3]); assertEquals(DetokenizationOperation.MERGE_BOTH, detokenizeOperations[4]); assertEquals(DetokenizationOperation.NO_OPERATION, detokenizeOperations[5]); }
@Test public void testDetokenizeToString() throws IOException { Detokenizer detokenizer = createLatinDetokenizer(); String tokens[] = new String[]{"A", "test", ",", "(", "string", ")", "."}; String sentence = detokenizer.detokenize(tokens, null); assertEquals("A test, (string).", sentence); }
@Test public void testDetokenizeToString2() throws IOException { Detokenizer detokenizer = createLatinDetokenizer(); String tokens[] = new String[]{"A", "co", "-", "worker", "helped", "."}; String sentence = detokenizer.detokenize(tokens, null); assertEquals("A co-worker helped.", sentence); }