@Test public void testCommentCombinations() { List<String> content = new ArrayList<>(); List<String> delimiters = new ArrayList<>(); String input = "hello world "; CommentTokenizer tokenizer = new CommentTokenizer(input); tokenizer.tokenize(content, delimiters); assertEquals(4, content.size()); assertEquals(4, delimiters.size()); assertEquals("", delimiters.get(1)); assertEquals("", delimiters.get(3)); assertEquals("hello world ", content.get(0)); assertEquals(" assertEquals("", content.get(2)); assertEquals(" bar baz ", content.get(3)); }
public void tokenize(List<String> content, List<String> delimiters) { assert (content != null && content.size() == 0); assert (delimiters != null && delimiters.size() == 0); TokenizerMode mode = TokenizerMode.NEUTRAL; if (length == 0) { delimiters.add(""); content.add(""); return; } while (offset < length) { switch (mode) { case NEUTRAL: { updatePositions(); int smallest = min(oneLineComment, multiLineComment, parameter, doubleQuote, singleQuote); if (smallest == length) { delimiters.add(""); content.add(input.substring(offset)); offset = length; } else if (smallest == doubleQuote) { delimiters.add("\""); content.add(input.substring(offset, doubleQuote)); offset = doubleQuote + 1; mode = TokenizerMode.BEGIN_DOUBLE_QUOTE; } else if (smallest == singleQuote) { delimiters.add("\'"); content.add(input.substring(offset, singleQuote)); offset = singleQuote + 1; mode = TokenizerMode.BEGIN_SINGLE_QUOTE; } else if (smallest == oneLineComment) { delimiters.add(" content.add(input.substring(offset, oneLineComment)); offset = oneLineComment + 2; mode = TokenizerMode.BEGIN_SINGLELINE_COMMENT; } else if (smallest == multiLineComment) { delimiters.add("", offset), length); content.add(input.substring(offset, endComment)); if (endComment == length) { delimiters.add(""); } else { delimiters.add("*/"); } offset = endComment + 2; mode = TokenizerMode.NEUTRAL; break; } case BEGIN_PARAMETER: { int endParameter = convertMissing(input.indexOf("]%", offset), length); content.add(input.substring(offset, endParameter)); if (endParameter == length) { delimiters.add(""); } else { delimiters.add("]%"); } offset = endParameter + 2; mode = TokenizerMode.NEUTRAL; break; } } } }
CommentTokenizer { public void tokenize(List<String> content, List<String> delimiters) { assert (content != null && content.size() == 0); assert (delimiters != null && delimiters.size() == 0); TokenizerMode mode = TokenizerMode.NEUTRAL; if (length == 0) { delimiters.add(""); content.add(""); return; } while (offset < length) { switch (mode) { case NEUTRAL: { updatePositions(); int smallest = min(oneLineComment, multiLineComment, parameter, doubleQuote, singleQuote); if (smallest == length) { delimiters.add(""); content.add(input.substring(offset)); offset = length; } else if (smallest == doubleQuote) { delimiters.add("\""); content.add(input.substring(offset, doubleQuote)); offset = doubleQuote + 1; mode = TokenizerMode.BEGIN_DOUBLE_QUOTE; } else if (smallest == singleQuote) { delimiters.add("\'"); content.add(input.substring(offset, singleQuote)); offset = singleQuote + 1; mode = TokenizerMode.BEGIN_SINGLE_QUOTE; } else if (smallest == oneLineComment) { delimiters.add(" content.add(input.substring(offset, oneLineComment)); offset = oneLineComment + 2; mode = TokenizerMode.BEGIN_SINGLELINE_COMMENT; } else if (smallest == multiLineComment) { delimiters.add("", offset), length); content.add(input.substring(offset, endComment)); if (endComment == length) { delimiters.add(""); } else { delimiters.add("*/"); } offset = endComment + 2; mode = TokenizerMode.NEUTRAL; break; } case BEGIN_PARAMETER: { int endParameter = convertMissing(input.indexOf("]%", offset), length); content.add(input.substring(offset, endParameter)); if (endParameter == length) { delimiters.add(""); } else { delimiters.add("]%"); } offset = endParameter + 2; mode = TokenizerMode.NEUTRAL; break; } } } } }
CommentTokenizer { public void tokenize(List<String> content, List<String> delimiters) { assert (content != null && content.size() == 0); assert (delimiters != null && delimiters.size() == 0); TokenizerMode mode = TokenizerMode.NEUTRAL; if (length == 0) { delimiters.add(""); content.add(""); return; } while (offset < length) { switch (mode) { case NEUTRAL: { updatePositions(); int smallest = min(oneLineComment, multiLineComment, parameter, doubleQuote, singleQuote); if (smallest == length) { delimiters.add(""); content.add(input.substring(offset)); offset = length; } else if (smallest == doubleQuote) { delimiters.add("\""); content.add(input.substring(offset, doubleQuote)); offset = doubleQuote + 1; mode = TokenizerMode.BEGIN_DOUBLE_QUOTE; } else if (smallest == singleQuote) { delimiters.add("\'"); content.add(input.substring(offset, singleQuote)); offset = singleQuote + 1; mode = TokenizerMode.BEGIN_SINGLE_QUOTE; } else if (smallest == oneLineComment) { delimiters.add(" content.add(input.substring(offset, oneLineComment)); offset = oneLineComment + 2; mode = TokenizerMode.BEGIN_SINGLELINE_COMMENT; } else if (smallest == multiLineComment) { delimiters.add("", offset), length); content.add(input.substring(offset, endComment)); if (endComment == length) { delimiters.add(""); } else { delimiters.add("*/"); } offset = endComment + 2; mode = TokenizerMode.NEUTRAL; break; } case BEGIN_PARAMETER: { int endParameter = convertMissing(input.indexOf("]%", offset), length); content.add(input.substring(offset, endParameter)); if (endParameter == length) { delimiters.add(""); } else { delimiters.add("]%"); } offset = endParameter + 2; mode = TokenizerMode.NEUTRAL; break; } } } } CommentTokenizer(@Nonnull String input); }
CommentTokenizer { public void tokenize(List<String> content, List<String> delimiters) { assert (content != null && content.size() == 0); assert (delimiters != null && delimiters.size() == 0); TokenizerMode mode = TokenizerMode.NEUTRAL; if (length == 0) { delimiters.add(""); content.add(""); return; } while (offset < length) { switch (mode) { case NEUTRAL: { updatePositions(); int smallest = min(oneLineComment, multiLineComment, parameter, doubleQuote, singleQuote); if (smallest == length) { delimiters.add(""); content.add(input.substring(offset)); offset = length; } else if (smallest == doubleQuote) { delimiters.add("\""); content.add(input.substring(offset, doubleQuote)); offset = doubleQuote + 1; mode = TokenizerMode.BEGIN_DOUBLE_QUOTE; } else if (smallest == singleQuote) { delimiters.add("\'"); content.add(input.substring(offset, singleQuote)); offset = singleQuote + 1; mode = TokenizerMode.BEGIN_SINGLE_QUOTE; } else if (smallest == oneLineComment) { delimiters.add(" content.add(input.substring(offset, oneLineComment)); offset = oneLineComment + 2; mode = TokenizerMode.BEGIN_SINGLELINE_COMMENT; } else if (smallest == multiLineComment) { delimiters.add("", offset), length); content.add(input.substring(offset, endComment)); if (endComment == length) { delimiters.add(""); } else { delimiters.add("*/"); } offset = endComment + 2; mode = TokenizerMode.NEUTRAL; break; } case BEGIN_PARAMETER: { int endParameter = convertMissing(input.indexOf("]%", offset), length); content.add(input.substring(offset, endParameter)); if (endParameter == length) { delimiters.add(""); } else { delimiters.add("]%"); } offset = endParameter + 2; mode = TokenizerMode.NEUTRAL; break; } } } } CommentTokenizer(@Nonnull String input); void tokenize(List<String> content, List<String> delimiters); }
CommentTokenizer { public void tokenize(List<String> content, List<String> delimiters) { assert (content != null && content.size() == 0); assert (delimiters != null && delimiters.size() == 0); TokenizerMode mode = TokenizerMode.NEUTRAL; if (length == 0) { delimiters.add(""); content.add(""); return; } while (offset < length) { switch (mode) { case NEUTRAL: { updatePositions(); int smallest = min(oneLineComment, multiLineComment, parameter, doubleQuote, singleQuote); if (smallest == length) { delimiters.add(""); content.add(input.substring(offset)); offset = length; } else if (smallest == doubleQuote) { delimiters.add("\""); content.add(input.substring(offset, doubleQuote)); offset = doubleQuote + 1; mode = TokenizerMode.BEGIN_DOUBLE_QUOTE; } else if (smallest == singleQuote) { delimiters.add("\'"); content.add(input.substring(offset, singleQuote)); offset = singleQuote + 1; mode = TokenizerMode.BEGIN_SINGLE_QUOTE; } else if (smallest == oneLineComment) { delimiters.add(" content.add(input.substring(offset, oneLineComment)); offset = oneLineComment + 2; mode = TokenizerMode.BEGIN_SINGLELINE_COMMENT; } else if (smallest == multiLineComment) { delimiters.add("", offset), length); content.add(input.substring(offset, endComment)); if (endComment == length) { delimiters.add(""); } else { delimiters.add("*/"); } offset = endComment + 2; mode = TokenizerMode.NEUTRAL; break; } case BEGIN_PARAMETER: { int endParameter = convertMissing(input.indexOf("]%", offset), length); content.add(input.substring(offset, endParameter)); if (endParameter == length) { delimiters.add(""); } else { delimiters.add("]%"); } offset = endParameter + 2; mode = TokenizerMode.NEUTRAL; break; } } } } CommentTokenizer(@Nonnull String input); void tokenize(List<String> content, List<String> delimiters);  }