CndTokenizer implements Tokenizer { @Override public void tokenize( CharacterStream input, Tokens tokens ) throws ParsingException { while (input.hasNext()) { char c = input.next(); switch (c) { case ' ': case '\t': case '\n': case '\r': break; case '[': case ']': case '<': case '>': case '=': case '-': case '+': case '(': case ')': case ',': tokens.addToken(input.position(input.index()), input.index(), input.index() + 1, SYMBOL); break; case '{': int startIndex = input.index(); Position startingPosition = input.position(startIndex); boolean foundClosingBrace = false; while (input.hasNext()) { c = input.next(); if (c == '\\' && input.isNext('}')) { c = input.next(); } else if (c == '}') { foundClosingBrace = true; break; } } if (!foundClosingBrace) { String msg = CndI18n.vendorBlockWasNotClosed.text(startingPosition.getLine(), startingPosition.getColumn()); throw new ParsingException(startingPosition, msg); } int endIndex = input.index() + 1; if (useVendorExtensions) { tokens.addToken(startingPosition, startIndex, endIndex, VENDOR_EXTENSION); } break; case '\"': startIndex = input.index(); startingPosition = input.position(startIndex); boolean foundClosingQuote = false; while (input.hasNext()) { c = input.next(); if (c == '\\' && input.isNext('"')) { c = input.next(); } else if (c == '"') { foundClosingQuote = true; break; } } if (!foundClosingQuote) { String msg = CommonI18n.noMatchingDoubleQuoteFound.text(startingPosition.getLine(), startingPosition.getColumn()); throw new ParsingException(startingPosition, msg); } endIndex = input.index() + 1; tokens.addToken(startingPosition, startIndex, endIndex, DOUBLE_QUOTED_STRING); break; case '\'': startIndex = input.index(); startingPosition = input.position(startIndex); foundClosingQuote = false; while (input.hasNext()) { c = input.next(); if (c == '\\' && input.isNext('\'')) { c = input.next(); } else if (c == '\'') { foundClosingQuote = true; break; } } if (!foundClosingQuote) { String msg = CommonI18n.noMatchingSingleQuoteFound.text(startingPosition.getLine(), startingPosition.getColumn()); throw new ParsingException(startingPosition, msg); } endIndex = input.index() + 1; tokens.addToken(startingPosition, startIndex, endIndex, SINGLE_QUOTED_STRING); break; case '/': startIndex = input.index(); startingPosition = input.position(startIndex); if (input.isNext('/')) { boolean foundLineTerminator = false; while (input.hasNext()) { c = input.next(); if (c == '\n' || c == '\r') { foundLineTerminator = true; break; } } endIndex = input.index(); if (!foundLineTerminator) ++endIndex; if (c == '\r' && input.isNext('\n')) input.next(); if (useComments) { tokens.addToken(startingPosition, startIndex, endIndex, COMMENT); } } else if (input.isNext('*')) { while (input.hasNext() && !input.isNext('*', '/')) { c = input.next(); } if (input.hasNext()) input.next(); if (input.hasNext()) input.next(); if (useComments) { endIndex = input.index() + 1; tokens.addToken(startingPosition, startIndex, endIndex, COMMENT); } } else { continue; } break; default: startIndex = input.index(); startingPosition = input.position(startIndex); while (input.hasNext() && !(input.isNextWhitespace() || input.isNextAnyOf("[]<>=-+(),\"'/{*|"))) { c = input.next(); } endIndex = input.index() + 1; tokens.addToken(startingPosition, startIndex, endIndex, WORD); } } } CndTokenizer( boolean useComments,
                         boolean useVendorExtensions ); @Override void tokenize( CharacterStream input,
                          Tokens tokens ); static final int WORD; static final int SYMBOL; static final int DECIMAL; static final int SINGLE_QUOTED_STRING; static final int DOUBLE_QUOTED_STRING; static final int COMMENT; static final int VENDOR_EXTENSION; }
@Test public void shouldCreateNoTokensForEmptyContent() { tokenize(""); assertNoMoreTokens(); }
@Test public void shouldCreateNoTokensForContentWithOnlyWhitespace() { tokenize(" \t \n \r\n \r "); assertNoMoreTokens(); }
@Test public void shouldCreateTokenForEachSymbolCharacter() { String content = "[]<>=-+(),"; int numSymbols = content.length(); tokenize(content); for (int i = 0; i != numSymbols; ++i) { assertNextTokenIs(i, i + 1, CndTokenizer.SYMBOL); } assertNoMoreTokens(); }
@Test public void shouldNotIncludeColonInListOfSymbolsSinceTheyCanAppearInNames() { tokenizer = new CndTokenizer(true, true); String content = "dna:someName"; tokenize(content); assertNextTokenIs(0, content.length(), CndTokenizer.WORD); assertNoMoreTokens(); }
@Test public void shouldCreateVendorExtensionToken() { tokenizer = new CndTokenizer(true, true); String content = "{vendor extension}"; tokenize(content); assertNextTokenIs(0, content.length(), CndTokenizer.VENDOR_EXTENSION); assertNoMoreTokens(); }
@Test public void shouldNotCreateVendorExtensionTokenIfTokenizerIsNotUsingThem() { tokenizer = new CndTokenizer(true, false); String content = "{vendor extension}"; tokenize(content); assertNoMoreTokens(); }
@Test public void shouldCreateTokenForEndOfLineComment() { String content = "-- tokenize(content); assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); assertNextTokenIs(2, content.length() - 1, CndTokenizer.COMMENT); assertNoMoreTokens(); }
@Test public void shouldCreateTokenForEndOfLineCommentThatEndsWithEndOfString() { String content = "-- tokenize(content); assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); assertNextTokenIs(2, content.length(), CndTokenizer.COMMENT); assertNoMoreTokens(); }
@Test public void shouldCreateTokenForMultiLineComment() { String content = "---"; tokenize(content); assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); assertNextTokenIs(2, content.length() - 1, CndTokenizer.COMMENT); assertNextTokenIs(content.length() - 1, content.length(), CndTokenizer.SYMBOL); assertNoMoreTokens(); }
@Test public void shouldCreateTokenForMultiLineCommentAtEndOfContent() { String content = "--"; tokenize(content); assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); assertNextTokenIs(2, content.length(), CndTokenizer.COMMENT); assertNoMoreTokens(); }
@Test public void shouldCreateTokenForMultiLineCommentWithoutTerminatingCharacters() { String content = "--/*this is a comment"; tokenize(content); assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); assertNextTokenIs(2, content.length(), CndTokenizer.COMMENT); assertNoMoreTokens(); }
@Test public void shouldCreateTokenForMultiLineCommentWithoutAllTerminatingCharacters() { String content = "--/*this is a comment*"; tokenize(content); assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); assertNextTokenIs(2, content.length(), CndTokenizer.COMMENT); assertNoMoreTokens(); }
@Test public void shouldCreateTokenForSingleQuotedString() { String content = "--'this is a single-quoted \n string'-"; assertThat(content.charAt(2), is('\'')); assertThat(content.charAt(35), is('\'')); tokenize(content); assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); assertNextTokenIs(2, 36, CndTokenizer.SINGLE_QUOTED_STRING); assertNextTokenIs(36, 37, CndTokenizer.SYMBOL); assertNoMoreTokens(); }
@Test public void shouldCreateTokenForSingleQuotedStringWithEscapedSingleQuoteCharacters() { String content = "--'this \"is\" a \\'single-quoted\\' \n string'-"; assertThat(content.charAt(2), is('\'')); assertThat(content.charAt(41), is('\'')); tokenize(content); assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); assertNextTokenIs(2, 42, CndTokenizer.SINGLE_QUOTED_STRING); assertNextTokenIs(42, 43, CndTokenizer.SYMBOL); assertNoMoreTokens(); }
@Test public void shouldCreateTokenForSingleQuotedStringAtEndOfContent() { String content = "--'this is a single-quoted \n string'"; assertThat(content.charAt(2), is('\'')); assertThat(content.charAt(35), is('\'')); tokenize(content); assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); assertNextTokenIs(2, 36, CndTokenizer.SINGLE_QUOTED_STRING); assertNoMoreTokens(); }
@Test( expected = ParsingException.class ) public void shouldCreateTokenForSingleQuotedStringWithoutClosingQuote() { String content = "--'this is a single-quoted \n string"; tokenize(content); }
@Test public void shouldCreateTokenForDoubleQuotedString() { String content = "--\"this is a double-quoted \n string\"-"; assertThat(content.charAt(2), is('"')); assertThat(content.charAt(35), is('"')); tokenize(content); assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); assertNextTokenIs(2, 36, CndTokenizer.DOUBLE_QUOTED_STRING); assertNextTokenIs(36, 37, CndTokenizer.SYMBOL); assertNoMoreTokens(); }
@Test public void shouldCreateTokenForDoubleQuotedStringWithEscapedDoubleQuoteCharacters() { String content = "--\"this 'is' a \\\"double-quoted\\\" \n string\"-"; assertThat(content.charAt(2), is('"')); assertThat(content.charAt(41), is('"')); tokenize(content); assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); assertNextTokenIs(2, 42, CndTokenizer.DOUBLE_QUOTED_STRING); assertNextTokenIs(42, 43, CndTokenizer.SYMBOL); assertNoMoreTokens(); }
@Test public void shouldCreateTokenForDoubleQuotedStringAtEndOfContent() { String content = "--\"this is a double-quoted \n string\""; assertThat(content.charAt(2), is('"')); assertThat(content.charAt(35), is('"')); tokenize(content); assertNextTokenIs(0, 1, CndTokenizer.SYMBOL); assertNextTokenIs(1, 2, CndTokenizer.SYMBOL); assertNextTokenIs(2, 36, CndTokenizer.DOUBLE_QUOTED_STRING); assertNoMoreTokens(); }
@Test( expected = ParsingException.class ) public void shouldCreateTokenForDoubleQuotedStringWithoutClosingQuote() { String content = "--\"this is a double-quoted \n string"; tokenize(content); }
@Test public void shouldCreateTokensForWordsWithAlphabeticCharacters() { String content = "This is a series of words."; tokenize(content); assertNextTokenIs(0, 4, CndTokenizer.WORD); assertNextTokenIs(5, 7, CndTokenizer.WORD); assertNextTokenIs(8, 9, CndTokenizer.WORD); assertNextTokenIs(10, 16, CndTokenizer.WORD); assertNextTokenIs(17, 19, CndTokenizer.WORD); assertNextTokenIs(20, 26, CndTokenizer.WORD); assertNoMoreTokens(); }
@Test public void shouldCreateTokensForWordsWithNumericCharacters() { String content = "1234 4 5353.324"; tokenize(content); assertNextTokenIs(0, 4, CndTokenizer.WORD); assertNextTokenIs(5, 6, CndTokenizer.WORD); assertNextTokenIs(7, 15, CndTokenizer.WORD); assertNoMoreTokens(); }
@Test public void shouldCreateTokensForWordsWithAlphaNumericCharacters() { String content = "123a 5353.324e100"; tokenize(content); assertNextTokenIs(0, 4, CndTokenizer.WORD); assertNextTokenIs(5, 17, CndTokenizer.WORD); assertNoMoreTokens(); }