giving lexer utf-8 capability
+use syntax::ast::punc::Punctuator;
use syntax::ast::token::{Token, TokenData};
/// A javascript Lexer
@@ -26,4 +27,28 @@ impl Lexer {
self.tokens
.push(Token::new(tk, self.line_number, self.column_number))
}
+
+ /// Push a punctuation token
+ fn push_punc(&mut self, punc: Punctuator) {
+ self.push_token(TokenData::TPunctuator(punc));
+ }
+
+ /// Processes an input stream from a string into an array of tokens
+ pub fn lex_str(script: String) -> Vec<Token> {
+ let mut lexer = Lexer::new(script);
+ lexer.tokens
+ }
+
+ fn next(&mut self) -> Option<char> {
+ self.buffer.chars().next()
+ }
+
+ pub fn lex(&mut self) -> Result<(), &str> {
+ loop {
+ let ch = match self.next() {
+ Some(ch) => ch,
+ None => return Err("oh my days"),
+ };
+ }
+ }
}
