data/train/cpp/bbd23e970556d5d4fda7dc9ef8c3e9fbd9868999Tokenizer.cpp
std::string GetToken( std::istream & stream ) {
		// ignore whitespace
		for( ; stream.peek( ) == 9 || stream.peek( ) == 10 || stream.peek( ) == 13 || stream.peek( ) == 32; stream.get( ) );

		int c = stream.peek( );
		if( c == -1 ) {
			// return blank token indicating end of stream
			return std::string( );
		} else if( c == '(' || c == ')' || c == ',' || c == ';' || c == '{' || c == '}' || c == '=' ) {
			// single character tokens
			return std::string( 1, (char)stream.get( ) );
		} else if( c >= '0' && c <= '9' || c == '-' || c == '+' ) {
			return GetNumber( stream );
		} else if( c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c == '_' ) {
			return GetIdentifier( stream );
		} else if( c == '"' ) {
			return GetStringConst( stream );
		} else if( c == '#' ) {
			return GetComment( stream );
		} else {
			throw TokenizerError( "unknown start of token: " + std::string( 1, (char)c ) );
		}
	}
// reads the stream for the next token
reads the stream for the next token
reads the stream for the next token
std::string GetToken( std::istream & stream ) {
		for( ; stream.peek( ) == 9 || stream.peek( ) == 10 || stream.peek( ) == 13 || stream.peek( ) == 32; stream.get( ) );
		int c = stream.peek( );
		if( c == -1 ) {
			return std::string( );
		} else if( c == '(' || c == ')' || c == ',' || c == ';' || c == '{' || c == '}' || c == '=' ) {
			return std::string( 1, (char)stream.get( ) );
		} else if( c >= '0' && c <= '9' || c == '-' || c == '+' ) {
			return GetNumber( stream );
		} else if( c >= 'A' && c <= 'Z' || c >= 'a' && c <= 'z' || c == '_' ) {
			return GetIdentifier( stream );
		} else if( c == '"' ) {
			return GetStringConst( stream );
		} else if( c == '#' ) {
			return GetComment( stream );
		} else {
			throw TokenizerError( "unknown start of token: " + std::string( 1, (char)c ) );
		}
	}
std :: string GetToken ( std :: istream & stream ) { for ( ; stream . peek ( ) == 9 || stream . peek ( ) == 10 || stream . peek ( ) == 13 || stream . peek ( ) == 32 ; stream . get ( ) ) ; int c = stream . peek ( ) ; if ( c == -1 ) { return std :: string ( ) ; } else if ( c == ' ' || c == ' ' || c == ' ' || c == ' ' || c == ' ' || c == ' ' || c == ' ' ) { return std :: string ( 1 , ( char ) stream . get ( ) ) ; } else if ( c >= ' ' && c <= ' ' || c == ' ' || c == ' ' ) { return GetNumber ( stream ) ; } else if ( c >= ' ' && c <= ' ' || c >= ' ' && c <= ' ' || c == ' ' ) { return GetIdentifier ( stream ) ; } else if ( c == ' ' ) { return GetStringConst ( stream ) ; } else if ( c == ' ' ) { return GetComment ( stream ) ; } else { throw TokenizerError ( " " + std :: string ( 1 , ( char ) c ) ) ; } }
reads the stream for the next token
reads the stream for the next token
// ignore whitespace // return blank token indicating end of stream // single character tokens