StructuredLexer { public static ArrayList<Token> tokens(String query) throws IOException { ArrayList<Token> tokens = new ArrayList<Token>(); HashSet<Character> tokenCharacters = new HashSet<Character>(); tokenCharacters.add('#'); tokenCharacters.add(':'); tokenCharacters.add('.'); tokenCharacters.add('='); tokenCharacters.add(')'); tokenCharacters.add('('); tokenCharacters.add(','); int start = 0; int qlen = query.length (); for (int i = 0; i < qlen; ++i) { char c = query.charAt(i); boolean special = tokenCharacters.contains(c) || c == '@' || c == '"'; boolean isSpace = Character.isWhitespace(c); if (special || isSpace) { if (start != i) { tokens.add(new Token(query.substring(start, i), start, TokenType.TERM)); } if (c == '@') { if (i + 1 < qlen) { char escapeChar = query.charAt(i + 1); int endChar = query.indexOf(escapeChar, i + 2); if (endChar < 0) { throw new IOException("Lex failure: No end found to '@' escape sequence."); } tokens.add(new Token(query.substring(i + 2, endChar), i, TokenType.SPECIALQUOTE)); i = endChar; } else { throw new IOException("Lex failure: '@' at end of input sequence."); } } else if (c == '"') { int endChar = query.indexOf('"', i + 1); if (endChar < 0) { throw new IOException("Lex failure: No ending quote found."); } String quotedString = query.substring(i + 1, endChar); addQuotedTokens(quotedString, tokens, i); i = endChar; } else if (!isSpace) { boolean badDot = false; if (c == '.') { badDot = ( (i+1 < qlen && Character.isWhitespace (query.charAt (i+1))) || (i+1 == qlen) ); } if (!badDot) { tokens.add(new Token(Character.toString(c), i, TokenType.TERM)); } } start = i + 1; } } if (start != query.length()) { tokens.add(new Token(query.substring(start), start, TokenType.TERM)); } return tokens; }  static ArrayList<Token> tokens(String query);  }
@Test public void testTokens() throws Exception { StructuredLexer lexer = new StructuredLexer(); List<Token> tokens = lexer.tokens("#op:this=that:a( b c d ).e"); Iterator<Token> iterator = tokens.iterator(); Token t = iterator.next(); assertEquals("#", t.text); assertEquals(0, t.position); t = iterator.next(); assertEquals("op", t.text); assertEquals(1, t.position); t = iterator.next(); assertEquals(":", t.text); assertEquals(3, t.position); t = iterator.next(); assertEquals("this", t.text); assertEquals(4, t.position); t = iterator.next(); assertEquals("=", t.text); assertEquals(8, t.position); t = iterator.next(); assertEquals("that", t.text); assertEquals(9, t.position); t = iterator.next(); assertEquals(":", t.text); assertEquals(13, t.position); t = iterator.next(); assertEquals("a", t.text); assertEquals(14, t.position); t = iterator.next(); assertEquals("(", t.text); assertEquals(15, t.position); t = iterator.next(); assertEquals("b", t.text); assertEquals(17, t.position); t = iterator.next(); assertEquals("c", t.text); assertEquals(19, t.position); t = iterator.next(); assertEquals("d", t.text); assertEquals(21, t.position); t = iterator.next(); assertEquals(")", t.text); assertEquals(23, t.position); t = iterator.next(); assertEquals(".", t.text); assertEquals(24, t.position); t = iterator.next(); assertEquals("e", t.text); assertEquals(25, t.position); assertFalse(iterator.hasNext()); }
@Test public void testEscapes() throws IOException { StructuredLexer lexer = new StructuredLexer(); List<Token> tokens = lexer.tokens("@/b c d/"); Iterator<Token> iterator = tokens.iterator(); Token t = iterator.next(); assertEquals("b c d", t.text); assertEquals(0, t.position); }
@Test public void testSimpleDotTokens () throws Exception { StructuredLexer lexer = new StructuredLexer (); List<Token> tokens = lexer.tokens ("world. "); Iterator<Token> iterator = tokens.iterator(); Token t = iterator.next(); assertEquals("world", t.text); assertEquals(0, t.position); assertFalse(iterator.hasNext()); }
@Test public void testStructuredDotTokens () throws Exception { StructuredLexer lexer = new StructuredLexer (); List<Token> tokens = lexer.tokens ("#combine(world.headline)"); Iterator<Token> iterator = tokens.iterator(); Token t = iterator.next(); assertEquals("#", t.text); assertEquals(0, t.position); t = iterator.next(); assertEquals("combine", t.text); assertEquals(1, t.position); t = iterator.next(); assertEquals("(", t.text); assertEquals(8, t.position); t = iterator.next(); assertEquals("world", t.text); assertEquals(9, t.position); t = iterator.next(); assertEquals(".", t.text); assertEquals(14, t.position); t = iterator.next(); assertEquals("headline", t.text); assertEquals(15, t.position); t = iterator.next(); assertEquals(")", t.text); assertEquals(23, t.position); assertFalse(iterator.hasNext()); }
@Test public void testCombinedSimpleStructuredDotTokens () throws Exception { StructuredLexer lexer = new StructuredLexer (); List<Token> tokens = lexer.tokens ("#combine(world. world.headline)"); Iterator<Token> iterator = tokens.iterator(); Token t = iterator.next(); assertEquals("#", t.text); assertEquals(0, t.position); t = iterator.next(); assertEquals("combine", t.text); assertEquals(1, t.position); t = iterator.next(); assertEquals("(", t.text); assertEquals(8, t.position); t = iterator.next(); assertEquals("world", t.text); assertEquals(9, t.position); t = iterator.next(); assertEquals("world", t.text); assertEquals(17, t.position); t = iterator.next(); assertEquals(".", t.text); assertEquals(22, t.position); t = iterator.next(); assertEquals("headline", t.text); assertEquals(23, t.position); t = iterator.next(); assertEquals(")", t.text); assertEquals(31, t.position); assertFalse(iterator.hasNext()); }