@Test void testTokenizeText() { assertThat(t("1 \" 1\n2 3 '\""), contains( tokenId(HotaruTokenId.NUMBER), tokenId(HotaruTokenId.TEXT) )); assertThat(t("1 ' 1\n2 3 ' 2"), contains( tokenId(HotaruTokenId.NUMBER), tokenId(HotaruTokenId.TEXT), tokenId(HotaruTokenId.NUMBER) )); assertThrows(LexerException.class, () -> { all("' ... "); }); assertThat(single("'\\\''").getText(), is("'")); assertThat(single("\'\\\"\'").getText(), is("\\\"")); assertThat(single("\"\\\"\"").getText(), is("\"")); assertThat(single("\"\\\'\"").getText(), is("\\\'")); }
private Token tokenizeText(char openChar) { next(); clearBuffer(); int startPos = getPos() - 1; char current = peek(0); while (true) { if (current == '\\') { final var buffer = getBuffer(); current = next(); if (current == openChar) { current = next(); buffer.append(openChar); continue; } switch (current) { case '0': current = next(); buffer.append('\0'); continue; case 'b': current = next(); buffer.append('\b'); continue; case 'f': current = next(); buffer.append('\f'); continue; case 'n': current = next(); buffer.append('\n'); continue; case 'r': current = next(); buffer.append('\r'); continue; case 't': current = next(); buffer.append('\t'); continue; case 'u': int rollbackPosition = getPos(); while (current == 'u') { current = next(); } int escapedValue = 0; for (int i = 12; i >= 0 && escapedValue != -1; i -= 4) { if (isHexNumber(current)) { escapedValue |= (Character.digit(current, 16) << i); } else { escapedValue = -1; } current = next(); } if (escapedValue >= 0) { buffer.append((char) escapedValue); } else { buffer.append("\\u"); setPos(rollbackPosition); } continue; } buffer.append('\\'); continue; } if (current == openChar) break; if (current == '\0') { throw error("Reached end of file while parsing text"); } getBuffer().append(current); current = next(); } next(); int actualLength = getPos() - startPos; return addToken(HotaruTokenId.TEXT, getBuffer().toString(), actualLength); }
HotaruLexer extends Lexer { private Token tokenizeText(char openChar) { next(); clearBuffer(); int startPos = getPos() - 1; char current = peek(0); while (true) { if (current == '\\') { final var buffer = getBuffer(); current = next(); if (current == openChar) { current = next(); buffer.append(openChar); continue; } switch (current) { case '0': current = next(); buffer.append('\0'); continue; case 'b': current = next(); buffer.append('\b'); continue; case 'f': current = next(); buffer.append('\f'); continue; case 'n': current = next(); buffer.append('\n'); continue; case 'r': current = next(); buffer.append('\r'); continue; case 't': current = next(); buffer.append('\t'); continue; case 'u': int rollbackPosition = getPos(); while (current == 'u') { current = next(); } int escapedValue = 0; for (int i = 12; i >= 0 && escapedValue != -1; i -= 4) { if (isHexNumber(current)) { escapedValue |= (Character.digit(current, 16) << i); } else { escapedValue = -1; } current = next(); } if (escapedValue >= 0) { buffer.append((char) escapedValue); } else { buffer.append("\\u"); setPos(rollbackPosition); } continue; } buffer.append('\\'); continue; } if (current == openChar) break; if (current == '\0') { throw error("Reached end of file while parsing text"); } getBuffer().append(current); current = next(); } next(); int actualLength = getPos() - startPos; return addToken(HotaruTokenId.TEXT, getBuffer().toString(), actualLength); } }
HotaruLexer extends Lexer { private Token tokenizeText(char openChar) { next(); clearBuffer(); int startPos = getPos() - 1; char current = peek(0); while (true) { if (current == '\\') { final var buffer = getBuffer(); current = next(); if (current == openChar) { current = next(); buffer.append(openChar); continue; } switch (current) { case '0': current = next(); buffer.append('\0'); continue; case 'b': current = next(); buffer.append('\b'); continue; case 'f': current = next(); buffer.append('\f'); continue; case 'n': current = next(); buffer.append('\n'); continue; case 'r': current = next(); buffer.append('\r'); continue; case 't': current = next(); buffer.append('\t'); continue; case 'u': int rollbackPosition = getPos(); while (current == 'u') { current = next(); } int escapedValue = 0; for (int i = 12; i >= 0 && escapedValue != -1; i -= 4) { if (isHexNumber(current)) { escapedValue |= (Character.digit(current, 16) << i); } else { escapedValue = -1; } current = next(); } if (escapedValue >= 0) { buffer.append((char) escapedValue); } else { buffer.append("\\u"); setPos(rollbackPosition); } continue; } buffer.append('\\'); continue; } if (current == openChar) break; if (current == '\0') { throw error("Reached end of file while parsing text"); } getBuffer().append(current); current = next(); } next(); int actualLength = getPos() - startPos; return addToken(HotaruTokenId.TEXT, getBuffer().toString(), actualLength); } HotaruLexer(String input); }
HotaruLexer extends Lexer { private Token tokenizeText(char openChar) { next(); clearBuffer(); int startPos = getPos() - 1; char current = peek(0); while (true) { if (current == '\\') { final var buffer = getBuffer(); current = next(); if (current == openChar) { current = next(); buffer.append(openChar); continue; } switch (current) { case '0': current = next(); buffer.append('\0'); continue; case 'b': current = next(); buffer.append('\b'); continue; case 'f': current = next(); buffer.append('\f'); continue; case 'n': current = next(); buffer.append('\n'); continue; case 'r': current = next(); buffer.append('\r'); continue; case 't': current = next(); buffer.append('\t'); continue; case 'u': int rollbackPosition = getPos(); while (current == 'u') { current = next(); } int escapedValue = 0; for (int i = 12; i >= 0 && escapedValue != -1; i -= 4) { if (isHexNumber(current)) { escapedValue |= (Character.digit(current, 16) << i); } else { escapedValue = -1; } current = next(); } if (escapedValue >= 0) { buffer.append((char) escapedValue); } else { buffer.append("\\u"); setPos(rollbackPosition); } continue; } buffer.append('\\'); continue; } if (current == openChar) break; if (current == '\0') { throw error("Reached end of file while parsing text"); } getBuffer().append(current); current = next(); } next(); int actualLength = getPos() - startPos; return addToken(HotaruTokenId.TEXT, getBuffer().toString(), actualLength); } HotaruLexer(String input); static List<Token> tokenize(String input); Token nextToken(); }
HotaruLexer extends Lexer { private Token tokenizeText(char openChar) { next(); clearBuffer(); int startPos = getPos() - 1; char current = peek(0); while (true) { if (current == '\\') { final var buffer = getBuffer(); current = next(); if (current == openChar) { current = next(); buffer.append(openChar); continue; } switch (current) { case '0': current = next(); buffer.append('\0'); continue; case 'b': current = next(); buffer.append('\b'); continue; case 'f': current = next(); buffer.append('\f'); continue; case 'n': current = next(); buffer.append('\n'); continue; case 'r': current = next(); buffer.append('\r'); continue; case 't': current = next(); buffer.append('\t'); continue; case 'u': int rollbackPosition = getPos(); while (current == 'u') { current = next(); } int escapedValue = 0; for (int i = 12; i >= 0 && escapedValue != -1; i -= 4) { if (isHexNumber(current)) { escapedValue |= (Character.digit(current, 16) << i); } else { escapedValue = -1; } current = next(); } if (escapedValue >= 0) { buffer.append((char) escapedValue); } else { buffer.append("\\u"); setPos(rollbackPosition); } continue; } buffer.append('\\'); continue; } if (current == openChar) break; if (current == '\0') { throw error("Reached end of file while parsing text"); } getBuffer().append(current); current = next(); } next(); int actualLength = getPos() - startPos; return addToken(HotaruTokenId.TEXT, getBuffer().toString(), actualLength); } HotaruLexer(String input); static List<Token> tokenize(String input); Token nextToken();  }