src/tokeniser/tokeniser.c
hubbub_error hubbub_tokeniser_emit_token(hubbub_tokeniser *tokeniser,
		hubbub_token *token)
{
	hubbub_error err = HUBBUB_OK;

	assert(tokeniser != NULL);
	assert(token != NULL);
	assert(tokeniser->insert_buf->length == 0);

#ifndef NDEBUG
	/* Sanity checks */
	switch (token->type) {
	case HUBBUB_TOKEN_DOCTYPE:
		assert(memchr(token->data.doctype.name.ptr, 0xff, 
				token->data.doctype.name.len) == NULL);
		if (token->data.doctype.public_missing == false)
			assert(memchr(token->data.doctype.public_id.ptr, 0xff,
				token->data.doctype.public_id.len) == NULL);
		if (token->data.doctype.system_missing == false)
			assert(memchr(token->data.doctype.system_id.ptr, 0xff,
				token->data.doctype.system_id.len) == NULL);
		break;
	case HUBBUB_TOKEN_START_TAG:
	case HUBBUB_TOKEN_END_TAG:
	{
		uint32_t i;
		assert(memchr(token->data.tag.name.ptr, 0xff, 
				token->data.tag.name.len) == NULL);
		for (i = 0; i < token->data.tag.n_attributes; i++) {
			hubbub_attribute *attr = &token->data.tag.attributes[i];

			assert(memchr(attr->name.ptr, 0xff, attr->name.len) == 
					NULL);
			assert(memchr(attr->value.ptr, 0xff, attr->value.len) ==
					NULL);
		}
	}
		break;
	case HUBBUB_TOKEN_COMMENT:
		assert(memchr(token->data.comment.ptr, 0xff, 
				token->data.comment.len) == NULL);
		break;
	case HUBBUB_TOKEN_CHARACTER:
		assert(memchr(token->data.character.ptr, 0xff,
				token->data.character.len) == NULL);
		break;
	case HUBBUB_TOKEN_EOF:
		break;
	}
#endif

	/* Emit the token */
	if (tokeniser->token_handler) {
		err = tokeniser->token_handler(token, tokeniser->token_pw);
	}

	/* Discard current buffer */
	if (tokeniser->buffer->length) {
		parserutils_buffer_discard(tokeniser->buffer, 0,
				tokeniser->buffer->length);
	}

	/* Advance the pointer */
	if (tokeniser->context.pending) {
		parserutils_inputstream_advance(tokeniser->input,
				tokeniser->context.pending);
		tokeniser->context.pending = 0;
	}

	if (tokeniser->insert_buf->length > 0) {
		parserutils_inputstream_insert(tokeniser->input,
				tokeniser->insert_buf->data,
				tokeniser->insert_buf->length);
		parserutils_buffer_discard(tokeniser->insert_buf, 0,
				tokeniser->insert_buf->length);
	}

	/* Ensure callback can pause the tokenise */
	if (err == HUBBUB_PAUSED) {
		tokeniser->paused = true;
	}

	return err;
}
/**
 * Emit a token, performing sanity checks if necessary
 *
 * \param tokeniser  Tokeniser instance
 * \param token      Token to emit
 */
Emit a token, performing sanity checks if necessary
\param tokeniser  Tokeniser instance
\param token      Token to emit
Emit a token performing sanity checks if necessary \ param tokeniser Tokeniser instance \ param token Token to emit
hubbub_error hubbub_tokeniser_emit_token(hubbub_tokeniser *tokeniser,
		hubbub_token *token)
{
	hubbub_error err = HUBBUB_OK;
	assert(tokeniser != NULL);
	assert(token != NULL);
	assert(tokeniser->insert_buf->length == 0);
#ifndef NDEBUG
	switch (token->type) {
	case HUBBUB_TOKEN_DOCTYPE:
		assert(memchr(token->data.doctype.name.ptr, 0xff, 
				token->data.doctype.name.len) == NULL);
		if (token->data.doctype.public_missing == false)
			assert(memchr(token->data.doctype.public_id.ptr, 0xff,
				token->data.doctype.public_id.len) == NULL);
		if (token->data.doctype.system_missing == false)
			assert(memchr(token->data.doctype.system_id.ptr, 0xff,
				token->data.doctype.system_id.len) == NULL);
		break;
	case HUBBUB_TOKEN_START_TAG:
	case HUBBUB_TOKEN_END_TAG:
	{
		uint32_t i;
		assert(memchr(token->data.tag.name.ptr, 0xff, 
				token->data.tag.name.len) == NULL);
		for (i = 0; i < token->data.tag.n_attributes; i++) {
			hubbub_attribute *attr = &token->data.tag.attributes[i];
			assert(memchr(attr->name.ptr, 0xff, attr->name.len) == 
					NULL);
			assert(memchr(attr->value.ptr, 0xff, attr->value.len) ==
					NULL);
		}
	}
		break;
	case HUBBUB_TOKEN_COMMENT:
		assert(memchr(token->data.comment.ptr, 0xff, 
				token->data.comment.len) == NULL);
		break;
	case HUBBUB_TOKEN_CHARACTER:
		assert(memchr(token->data.character.ptr, 0xff,
				token->data.character.len) == NULL);
		break;
	case HUBBUB_TOKEN_EOF:
		break;
	}
#endif
	if (tokeniser->token_handler) {
		err = tokeniser->token_handler(token, tokeniser->token_pw);
	}
	if (tokeniser->buffer->length) {
		parserutils_buffer_discard(tokeniser->buffer, 0,
				tokeniser->buffer->length);
	}
	if (tokeniser->context.pending) {
		parserutils_inputstream_advance(tokeniser->input,
				tokeniser->context.pending);
		tokeniser->context.pending = 0;
	}
	if (tokeniser->insert_buf->length > 0) {
		parserutils_inputstream_insert(tokeniser->input,
				tokeniser->insert_buf->data,
				tokeniser->insert_buf->length);
		parserutils_buffer_discard(tokeniser->insert_buf, 0,
				tokeniser->insert_buf->length);
	}
	if (err == HUBBUB_PAUSED) {
		tokeniser->paused = true;
	}
	return err;
}
hubbub_error hubbub_tokeniser_emit_token ( hubbub_tokeniser * tokeniser , hubbub_token * token ) { hubbub_error err = HUBBUB_OK ; assert ( tokeniser != NULL ) ; assert ( token != NULL ) ; assert ( tokeniser -> insert_buf -> length == 0 ) ; #ifndef NDEBUG switch ( token -> type ) { case HUBBUB_TOKEN_DOCTYPE : assert ( memchr ( token -> data . doctype . name . ptr , 0xff , token -> data . doctype . name . len ) == NULL ) ; if ( token -> data . doctype . public_missing == false ) assert ( memchr ( token -> data . doctype . public_id . ptr , 0xff , token -> data . doctype . public_id . len ) == NULL ) ; if ( token -> data . doctype . system_missing == false ) assert ( memchr ( token -> data . doctype . system_id . ptr , 0xff , token -> data . doctype . system_id . len ) == NULL ) ; break ; case HUBBUB_TOKEN_START_TAG : case HUBBUB_TOKEN_END_TAG : { uint32_t i ; assert ( memchr ( token -> data . tag . name . ptr , 0xff , token -> data . tag . name . len ) == NULL ) ; for ( i = 0 ; i < token -> data . tag . n_attributes ; i ++ ) { hubbub_attribute * attr = & token -> data . tag . attributes [ i ] ; assert ( memchr ( attr -> name . ptr , 0xff , attr -> name . len ) == NULL ) ; assert ( memchr ( attr -> value . ptr , 0xff , attr -> value . len ) == NULL ) ; } } break ; case HUBBUB_TOKEN_COMMENT : assert ( memchr ( token -> data . comment . ptr , 0xff , token -> data . comment . len ) == NULL ) ; break ; case HUBBUB_TOKEN_CHARACTER : assert ( memchr ( token -> data . character . ptr , 0xff , token -> data . character . len ) == NULL ) ; break ; case HUBBUB_TOKEN_EOF : break ; } #endif if ( tokeniser -> token_handler ) { err = tokeniser -> token_handler ( token , tokeniser -> token_pw ) ; } if ( tokeniser -> buffer -> length ) { parserutils_buffer_discard ( tokeniser -> buffer , 0 , tokeniser -> buffer -> length ) ; } if ( tokeniser -> context . pending ) { parserutils_inputstream_advance ( tokeniser -> input , tokeniser -> context . pending ) ; tokeniser -> context . pending = 0 ; } if ( tokeniser -> insert_buf -> length > 0 ) { parserutils_inputstream_insert ( tokeniser -> input , tokeniser -> insert_buf -> data , tokeniser -> insert_buf -> length ) ; parserutils_buffer_discard ( tokeniser -> insert_buf , 0 , tokeniser -> insert_buf -> length ) ; } if ( err == HUBBUB_PAUSED ) { tokeniser -> paused = true ; } return err ; }
Emit a token, performing sanity checks if necessary
 \param tokeniser  Tokeniser instance
 \param token      Token to emit
Emit a token performing sanity checks if necessary \ param tokeniser Tokeniser instance \ param token Token to emit
/* Sanity checks */ /* Emit the token */ /* Discard current buffer */ /* Advance the pointer */ /* Ensure callback can pause the tokenise */