@Test public void medium() throws FileNotFoundException { FileReader fr = new FileReader(CAL10NTestConstants.TEST_CLASSES + "/parser/medium.properties"); TokenStream ts = new TokenStream(fr); List<Token> tokenList = ts.tokenize(); List<Token> witness = new ArrayList<Token>(); witness.add(new Token(TokenType.KEY, "K0")); witness.add(new Token(TokenType.SEPARATOR, "=")); witness.add(new Token(TokenType.VALUE, "V0 ")); witness.add(Token.TRAILING_BACKSLASH); witness.add(Token.EOL); witness.add(new Token(TokenType.VALUE, "X")); witness.add(Token.EOL); witness.add(Token.EOL); witness.add(new Token(TokenType.KEY, "K1")); witness.add(new Token(TokenType.SEPARATOR, "=")); witness.add(new Token(TokenType.VALUE, "V1")); witness.add(Token.EOL); assertEquals(witness, tokenList); }
List<Token> tokenize() { List<Token> tokenList = new ArrayList<Token>(); while (true) { String currentLine; try { currentLine = lineReader.readLine(); } catch (IOException e) { throw new MessageConveyorException("Failed to read input stream", e); } if (currentLine == null) { break; } if(state != State.TRAILING_BACKSLASH) { state = State.START; } tokenizeLine(tokenList, currentLine); tokenList.add(Token.EOL); } return tokenList; }
TokenStream { List<Token> tokenize() { List<Token> tokenList = new ArrayList<Token>(); while (true) { String currentLine; try { currentLine = lineReader.readLine(); } catch (IOException e) { throw new MessageConveyorException("Failed to read input stream", e); } if (currentLine == null) { break; } if(state != State.TRAILING_BACKSLASH) { state = State.START; } tokenizeLine(tokenList, currentLine); tokenList.add(Token.EOL); } return tokenList; } }
TokenStream { List<Token> tokenize() { List<Token> tokenList = new ArrayList<Token>(); while (true) { String currentLine; try { currentLine = lineReader.readLine(); } catch (IOException e) { throw new MessageConveyorException("Failed to read input stream", e); } if (currentLine == null) { break; } if(state != State.TRAILING_BACKSLASH) { state = State.START; } tokenizeLine(tokenList, currentLine); tokenList.add(Token.EOL); } return tokenList; } TokenStream(Reader reader); }
TokenStream { List<Token> tokenize() { List<Token> tokenList = new ArrayList<Token>(); while (true) { String currentLine; try { currentLine = lineReader.readLine(); } catch (IOException e) { throw new MessageConveyorException("Failed to read input stream", e); } if (currentLine == null) { break; } if(state != State.TRAILING_BACKSLASH) { state = State.START; } tokenizeLine(tokenList, currentLine); tokenList.add(Token.EOL); } return tokenList; } TokenStream(Reader reader);  }
TokenStream { List<Token> tokenize() { List<Token> tokenList = new ArrayList<Token>(); while (true) { String currentLine; try { currentLine = lineReader.readLine(); } catch (IOException e) { throw new MessageConveyorException("Failed to read input stream", e); } if (currentLine == null) { break; } if(state != State.TRAILING_BACKSLASH) { state = State.START; } tokenizeLine(tokenList, currentLine); tokenList.add(Token.EOL); } return tokenList; } TokenStream(Reader reader);   }