text: /* HTML parser for Wget.
     Copyright (C) 1998, 2000, 2003 Free Software Foundation, Inc.
  
  This file is part of GNU Wget.
  
  GNU Wget is free software; you can redistribute it and/or modify
  it under the terms of the GNU General Public License as published by
  the Free Software Foundation; either version 2 of the License, or (at
  your option) any later version.
  
  GNU Wget is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.
  
  You should have received a copy of the GNU General Public License
  along with Wget; if not, write to the Free Software
  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  
  In addition, as a special exception, the Free Software Foundation
  gives permission to link the code of its release of Wget with the
  OpenSSL project's "OpenSSL" library (or with modified versions of it
  that use the same license as the "OpenSSL" library), and distribute
  the linked executables.  You must obey the GNU General Public License
  in all respects for all of the code used other than "OpenSSL".  If you
  modify this file, you may extend this exception to your version of the
  file, but you are not obligated to do so.  If you do not wish to do
  so, delete this exception statement from your version.  */
  
  /* The only entry point to this module is map_html_tags(), which see.  */
  
  /* TODO:
  
     - Allow hooks for callers to process contents outside tags.  This
       is needed to implement handling <style> and <script>.  The
       taginfo structure already carries the information about where the
       tags are, but this is not enough, because one would also want to
       skip the comments.  (The funny thing is that for <style> and
       <script> you *don't* want to skip comments!)
  
     - Create a test suite for regression testing. */
  
  /* HISTORY:
  
     This is the third HTML parser written for Wget.  The first one was
     written some time during the Geturl 1.0 beta cycle, and was very
     inefficient and buggy.  It also contained some very complex code to
     remember a list of parser states, because it was supposed to be
     reentrant.
  
     The second HTML parser was written for Wget 1.4 (the first version
     by the name `Wget'), and was a complete rewrite.  Although the new
     parser behaved much better and made no claims of reentrancy, it
     still shared many of the fundamental flaws of the old version -- it
     only regarded HTML in terms tag-attribute pairs, where the
     attribute's value was a URL to be returned.  Any other property of
     HTML, such as <base href=...>, or strange way to specify a URL,
     such as <meta http-equiv=Refresh content="0; URL=..."> had to be
     crudely hacked in -- and the caller had to be aware of these hacks.
     Like its predecessor, this parser did not support HTML comments.
  
     After Wget 1.5.1 was released, I set out to write a third HTML
     parser.  The objectives of the new parser were to: (1) provide a
     clean way to analyze HTML lexically, (2) separate interpretation of
     the markup from the parsing process, (3) be as correct as possible,
     e.g. correctly skipping comments and other SGML declarations, (4)
     understand the most common errors in markup and skip them or be
     relaxed towrds them, and (5) be reasonably efficient (no regexps,
     minimum copying and minimum or no heap allocation).
  
     I believe this parser meets all of the above goals.  It is
     reasonably well structured, and could be relatively easily
     separated from Wget and used elsewhere.  While some of its
     intrinsic properties limit its value as a general-purpose HTML
     parser, I believe that, with minimum modifications, it could serve
     as a backend for one.
  
     Due to time and other constraints, this parser was not integrated
     into Wget until the version 1.7. */
  
  /* DESCRIPTION:
  
     The single entry point of this parser is map_html_tags(), which
     works by calling a function you specify for each tag.  The function
     gets called with the pointer to a structure describing the tag and
     its attributes.  */
  
  /* To test as standalone, compile with `-DSTANDALONE -I.'.  You'll
     still need Wget headers to compile.  */
  
  #include <config.h>
  
  #ifdef STANDALONE
  # define I_REALLY_WANT_CTYPE_MACROS
  #endif
  
  #include <stdio.h>
  #include <stdlib.h>
  #ifdef HAVE_STRING_H
  # include <string.h>
  #else
  # include <strings.h>
  #endif
  #include <assert.h>
  
  #include "wget.h"
  #include "html-parse.h"
  
  #ifdef STANDALONE
  # undef xmalloc
  # undef xrealloc
  # undef xfree
  # define xmalloc malloc
  # define xrealloc realloc
  # define xfree free
  
  # undef ISSPACE
  # undef ISDIGIT
  # undef ISXDIGIT
  # undef ISALPHA
  # undef ISALNUM
  # undef TOLOWER
  # undef TOUPPER
  
  # define ISSPACE(x) isspace (x)
  # define ISDIGIT(x) isdigit (x)
  # define ISXDIGIT(x) isxdigit (x)
  # define ISALPHA(x) isalpha (x)
  # define ISALNUM(x) isalnum (x)
  # define TOLOWER(x) tolower (x)
  # define TOUPPER(x) toupper (x)
  
  struct hash_table {
    int dummy;
  };
  static void *
  hash_table_get (const struct hash_table *ht, void *ptr)
  {
    return ptr;
  }
  #else  /* not STANDALONE */
  # include "hash.h"
  #endif
  
  /* Pool support.  A pool is a resizable chunk of memory.  It is first
     allocated on the stack, and moved to the heap if it needs to be
     larger than originally expected.  map_html_tags() uses it to store
     the zero-terminated names and values of tags and attributes.
  
     Thus taginfo->name, and attr->name and attr->value for each
     attribute, do not point into separately allocated areas, but into
     different parts of the pool, separated only by terminating zeros.
     This ensures minimum amount of allocation and, for most tags, no
     allocation because the entire pool is kept on the stack.  */
  
  struct pool {
    char *contents;		/* pointer to the contents. */
    int size;			/* size of the pool. */
    int tail;			/* next available position index. */
    int resized;			/* whether the pool has been resized
  				   using malloc. */
  
    char *orig_contents;		/* original pool contents, usually
                                     stack-allocated.  used by POOL_FREE
                                     to restore the pool to the initial
                                     state. */
    int orig_size;
  };
  
  /* Initialize the pool to hold INITIAL_SIZE bytes of storage. */
  
  #define POOL_INIT(p, initial_storage, initial_size) do {	\
    struct pool *P = (p);						\
    P->contents = (initial_storage);				\
    P->size = (initial_size);					\
    P->tail = 0;							\
    P->resized = 0;						\
    P->orig_contents = P->contents;				\
    P->orig_size = P->size;					\
  } while (0)
  
  /* Grow the pool to accomodate at least SIZE new bytes.  If the pool
     already has room to accomodate SIZE bytes of data, this is a no-op.  */
  
  #define POOL_GROW(p, increase)					\
    GROW_ARRAY ((p)->contents, (p)->size, (p)->tail + (increase),	\
  	      (p)->resized, char)
  
  /* Append text in the range [beg, end) to POOL.  No zero-termination
     is done.  */
  
  #define POOL_APPEND(p, beg, end) do {			\
    const char *PA_beg = (beg);				\
    int PA_size = (end) - PA_beg;				\
    POOL_GROW (p, PA_size);				\
    memcpy ((p)->contents + (p)->tail, PA_beg, PA_size);	\
    (p)->tail += PA_size;					\
  } while (0)
  
  /* Append one character to the pool.  Can be used to zero-terminate
     pool strings.  */
  
  #define POOL_APPEND_CHR(p, ch) do {		\
    char PAC_char = (ch);				\
    POOL_GROW (p, 1);				\
    (p)->contents[(p)->tail++] = PAC_char;	\
  } while (0)
  
  /* Forget old pool contents.  The allocated memory is not freed. */
  #define POOL_REWIND(p) (p)->tail = 0
  
  /* Free heap-allocated memory for contents of POOL.  This calls
     xfree() if the memory was allocated through malloc.  It also
     restores `contents' and `size' to their original, pre-malloc
     values.  That way after POOL_FREE, the pool is fully usable, just
     as if it were freshly initialized with POOL_INIT.  */
  
  #define POOL_FREE(p) do {			\
    struct pool *P = p;				\
    if (P->resized)				\
      xfree (P->contents);			\
    P->contents = P->orig_contents;		\
    P->size = P->orig_size;			\
    P->tail = 0;					\
    P->resized = 0;				\
  } while (0)
  
  /* Used for small stack-allocated memory chunks that might grow.  Like
     DO_REALLOC, this macro grows BASEVAR as necessary to take
     NEEDED_SIZE items of TYPE.
  
     The difference is that on the first resize, it will use
     malloc+memcpy rather than realloc.  That way you can stack-allocate
     the initial chunk, and only resort to heap allocation if you
     stumble upon large data.
  
     After the first resize, subsequent ones are performed with realloc,
     just like DO_REALLOC.  */
  
  #define GROW_ARRAY(basevar, sizevar, needed_size, resized, type) do {		\
    long ga_needed_size = (needed_size);						\
    long ga_newsize = (sizevar);							\
    while (ga_newsize < ga_needed_size)						\
      ga_newsize <<= 1;								\
    if (ga_newsize != (sizevar))							\
      {										\
        if (resized)								\
  	basevar = (type *)xrealloc (basevar, ga_newsize * sizeof (type));	\
        else									\
  	{									\
  	  void *ga_new = xmalloc (ga_newsize * sizeof (type));			\
  	  memcpy (ga_new, basevar, (sizevar) * sizeof (type));			\
  	  (basevar) = ga_new;							\
  	  resized = 1;								\
  	}									\
        (sizevar) = ga_newsize;							\
      }										\
  } while (0)
  
  /* Test whether n+1-sized entity name fits in P.  We don't support
     IE-style non-terminated entities, e.g. "&ltfoo" -> "<foo".
     However, "&lt;foo" will work, as will "&lt!foo", "&lt", etc.  In
     other words an entity needs to be terminated by either a
     non-alphanumeric or the end of string.  */
  #define FITS(p, n) (p + n == end || (p + n < end && !ISALNUM (p[n])))
  
  /* Macros that test entity names by returning true if P is followed by
     the specified characters.  */
  #define ENT1(p, c0) (FITS (p, 1) && p[0] == c0)
  #define ENT2(p, c0, c1) (FITS (p, 2) && p[0] == c0 && p[1] == c1)
  #define ENT3(p, c0, c1, c2) (FITS (p, 3) && p[0]==c0 && p[1]==c1 && p[2]==c2)
  
  /* Increment P by INC chars.  If P lands at a semicolon, increment it
     past the semicolon.  This ensures that e.g. "&lt;foo" is converted
     to "<foo", but "&lt,foo" to "<,foo".  */
  #define SKIP_SEMI(p, inc) (p += inc, p < end && *p == ';' ? ++p : p)
  
  /* Decode the HTML character entity at *PTR, considering END to be end
     of buffer.  It is assumed that the "&" character that marks the
     beginning of the entity has been seen at *PTR-1.  If a recognized
     ASCII entity is seen, it is returned, and *PTR is moved to the end
     of the entity.  Otherwise, -1 is returned and *PTR left unmodified.
  
     The recognized entities are: &lt, &gt, &amp, &apos, and &quot.  */
  
  static int
  decode_entity (const char **ptr, const char *end)
  {
    const char *p = *ptr;
    int value = -1;
  
    if (++p == end)
      return -1;
  
    switch (*p++)
      {
      case '#':
        /* Process numeric entities "&#DDD;" and "&#xHH;".  */
        {
  	int digits = 0;
  	value = 0;
  	if (*p == 'x')
  	  for (++p; value < 256 && p < end && ISXDIGIT (*p); p++, digits++)
  	    value = (value << 4) + XDIGIT_TO_NUM (*p);
  	else
  	  for (; value < 256 && p < end && ISDIGIT (*p); p++, digits++)
  	    value = (value * 10) + (*p - '0');
  	if (!digits)
  	  return -1;
  	/* Don't interpret 128+ codes and NUL because we cannot
  	   portably reinserted them into HTML.  */
  	if (!value || (value & ~0x7f))
  	  return -1;
  	*ptr = SKIP_SEMI (p, 0);
  	return value;
        }
      /* Process named ASCII entities.  */
      case 'g':
        if (ENT1 (p, 't'))
  	value = '>', *ptr = SKIP_SEMI (p, 1);
        break;
      case 'l':
        if (ENT1 (p, 't'))
  	value = '<', *ptr = SKIP_SEMI (p, 1);
        break;
      case 'a':
        if (ENT2 (p, 'm', 'p'))
  	value = '&', *ptr = SKIP_SEMI (p, 2);
        else if (ENT3 (p, 'p', 'o', 's'))
  	/* handle &apos for the sake of the XML/XHTML crowd. */
  	value = '\'', *ptr = SKIP_SEMI (p, 3);
        break;
      case 'q':
        if (ENT3 (p, 'u', 'o', 't'))
  	value = '\"', *ptr = SKIP_SEMI (p, 3);
        break;
      }
    return value;
  }
  #undef ENT1
  #undef ENT2
  #undef ENT3
  #undef FITS
  #undef SKIP_SEMI
  
  enum {
    AP_DOWNCASE		= 1,
    AP_DECODE_ENTITIES	= 2,
    AP_TRIM_BLANKS	= 4
  };
  
  /* Copy the text in the range [BEG, END) to POOL, optionally
     performing operations specified by FLAGS.  FLAGS may be any
     combination of AP_DOWNCASE, AP_DECODE_ENTITIES and AP_TRIM_BLANKS
     with the following meaning:
  
     * AP_DOWNCASE -- downcase all the letters;
  
     * AP_DECODE_ENTITIES -- decode the named and numeric entities in
       the ASCII range when copying the string.
  
     * AP_TRIM_BLANKS -- ignore blanks at the beginning and at the end
       of text, as well as embedded newlines.  */
  
  static void
  convert_and_copy (struct pool *pool, const char *beg, const char *end, int flags)
  {
    int old_tail = pool->tail;
  
    /* Skip blanks if required.  We must do this before entities are
       processed, so that blanks can still be inserted as, for instance,
       `&#32;'.  */
    if (flags & AP_TRIM_BLANKS)
      {
        while (beg < end && ISSPACE (*beg))
  	++beg;
        while (end > beg && ISSPACE (end[-1]))
  	--end;
      }
  
    if (flags & AP_DECODE_ENTITIES)
      {
        /* Grow the pool, then copy the text to the pool character by
  	 character, processing the encountered entities as we go
  	 along.
  
  	 It's safe (and necessary) to grow the pool in advance because
  	 processing the entities can only *shorten* the string, it can
  	 never lengthen it.  */
        const char *from = beg;
        char *to;
        int squash_newlines = flags & AP_TRIM_BLANKS;
  
        POOL_GROW (pool, end - beg);
        to = pool->contents + pool->tail;
  
        while (from < end)
  	{
  	  if (*from == '&')
  	    {
  	      int entity = decode_entity (&from, end);
  	      if (entity != -1)
  		*to++ = entity;
  	      else
  		*to++ = *from++;
  	    }
  	  else if ((*from == '\n' || *from == '\r') && squash_newlines)
  	    ++from;
  	  else
  	    *to++ = *from++;
  	}
        /* Verify that we haven't exceeded the original size.  (It
  	 shouldn't happen, hence the assert.)  */
        assert (to - (pool->contents + pool->tail) <= end - beg);
  
        /* Make POOL's tail point to the position following the string
  	 we've written.  */
        pool->tail = to - pool->contents;
        POOL_APPEND_CHR (pool, '\0');
      }
    else
      {
        /* Just copy the text to the pool.  */
        POOL_APPEND (pool, beg, end);
        POOL_APPEND_CHR (pool, '\0');
      }
  
    if (flags & AP_DOWNCASE)
      {
        char *p = pool->contents + old_tail;
        for (; *p; p++)
  	*p = TOLOWER (*p);
      }
  }
  
  /* Originally we used to adhere to rfc 1866 here, and allowed only
     letters, digits, periods, and hyphens as names (of tags or
     attributes).  However, this broke too many pages which used
     proprietary or strange attributes, e.g. <img src="a.gif"
     v:shapes="whatever">.
  
     So now we allow any character except:
       * whitespace
       * 8-bit and control chars
       * characters that clearly cannot be part of name:
         '=', '>', '/'.
  
     This only affects attribute and tag names; attribute values allow
     an even greater variety of characters.  */
  
  #define NAME_CHAR_P(x) ((x) > 32 && (x) < 127				\
  			&& (x) != '=' && (x) != '>' && (x) != '/')
  
  #ifdef STANDALONE
  static int comment_backout_count;
  #endif
  
  /* Advance over an SGML declaration, such as <!DOCTYPE ...>.  In
     strict comments mode, this is used for skipping over comments as
     well.
  
     To recap: any SGML declaration may have comments associated with
     it, e.g.
         <!MY-DECL -- isn't this fun? -- foo bar>
  
     An HTML comment is merely an empty declaration (<!>) with a comment
     attached, like this:
         <!-- some stuff here -->
  
     Several comments may be embedded in one comment declaration:
         <!-- have -- -- fun -->
  
     Whitespace is allowed between and after the comments, but not
     before the first comment.  Additionally, this function attempts to
     handle double quotes in SGML declarations correctly.  */
  
  static const char *
  advance_declaration (const char *beg, const char *end)
  {
    const char *p = beg;
    char quote_char = '\0';	/* shut up, gcc! */
    char ch;
  
    enum {
      AC_S_DONE,
      AC_S_BACKOUT,
      AC_S_BANG,
      AC_S_DEFAULT,
      AC_S_DCLNAME,
      AC_S_DASH1,
      AC_S_DASH2,
      AC_S_COMMENT,
      AC_S_DASH3,
      AC_S_DASH4,
      AC_S_QUOTE1,
      AC_S_IN_QUOTE,
      AC_S_QUOTE2
    } state = AC_S_BANG;
  
    if (beg == end)
      return beg;
    ch = *p++;
  
    /* It looked like a good idea to write this as a state machine, but
       now I wonder...  */
  
    while (state != AC_S_DONE && state != AC_S_BACKOUT)
      {
        if (p == end)
  	state = AC_S_BACKOUT;
        switch (state)
  	{
  	case AC_S_DONE:
  	case AC_S_BACKOUT:
  	  break;
  	case AC_S_BANG:
  	  if (ch == '!')
  	    {
  	      ch = *p++;
  	      state = AC_S_DEFAULT;
  	    }
  	  else
  	    state = AC_S_BACKOUT;
  	  break;
  	case AC_S_DEFAULT:
  	  switch (ch)
  	    {
  	    case '-':
  	      state = AC_S_DASH1;
  	      break;
  	    case ' ':
  	    case '\t':
  	    case '\r':
  	    case '\n':
  	      ch = *p++;
  	      break;
  	    case '>':
  	      state = AC_S_DONE;
  	      break;
  	    case '\'':
  	    case '\"':
  	      state = AC_S_QUOTE1;
  	      break;
  	    default:
  	      if (NAME_CHAR_P (ch))
  		state = AC_S_DCLNAME;
  	      else
  		state = AC_S_BACKOUT;
  	      break;
  	    }
  	  break;
  	case AC_S_DCLNAME:
  	  if (ch == '-')
  	    state = AC_S_DASH1;
  	  else if (NAME_CHAR_P (ch))
  	    ch = *p++;
  	  else
  	    state = AC_S_DEFAULT;
  	  break;
  	case AC_S_QUOTE1:
  	  /* We must use 0x22 because broken assert macros choke on
  	     '"' and '\"'.  */
  	  assert (ch == '\'' || ch == 0x22);
  	  quote_char = ch;	/* cheating -- I really don't feel like
  				   introducing more different states for
  				   different quote characters. */
  	  ch = *p++;
  	  state = AC_S_IN_QUOTE;
  	  break;
  	case AC_S_IN_QUOTE:
  	  if (ch == quote_char)
  	    state = AC_S_QUOTE2;
  	  else
  	    ch = *p++;
  	  break;
  	case AC_S_QUOTE2:
  	  assert (ch == quote_char);
  	  ch = *p++;
  	  state = AC_S_DEFAULT;
  	  break;
  	case AC_S_DASH1:
  	  assert (ch == '-');
  	  ch = *p++;
  	  state = AC_S_DASH2;
  	  break;
  	case AC_S_DASH2:
  	  switch (ch)
  	    {
  	    case '-':
  	      ch = *p++;
  	      state = AC_S_COMMENT;
  	      break;
  	    default:
  	      state = AC_S_BACKOUT;
  	    }
  	  break;
  	case AC_S_COMMENT:
  	  switch (ch)
  	    {
  	    case '-':
  	      state = AC_S_DASH3;
  	      break;
  	    default:
  	      ch = *p++;
  	      break;
  	    }
  	  break;
  	case AC_S_DASH3:
  	  assert (ch == '-');
  	  ch = *p++;
  	  state = AC_S_DASH4;
  	  break;
  	case AC_S_DASH4:
  	  switch (ch)
  	    {
  	    case '-':
  	      ch = *p++;
  	      state = AC_S_DEFAULT;
  	      break;
  	    default:
  	      state = AC_S_COMMENT;
  	      break;
  	    }
  	  break;
  	}
      }
  
    if (state == AC_S_BACKOUT)
      {
  #ifdef STANDALONE
        ++comment_backout_count;
  #endif
        return beg + 1;
      }
    return p;
  }
  
  /* Find the first occurrence of the substring "-->" in [BEG, END) and
     return the pointer to the character after the substring.  If the
     substring is not found, return NULL.  */
  
  static const char *
  find_comment_end (const char *beg, const char *end)
  {
    /* Open-coded Boyer-Moore search for "-->".  Examine the third char;
       if it's not '>' or '-', advance by three characters.  Otherwise,
       look at the preceding characters and try to find a match.  */
  
    const char *p = beg - 1;
  
    while ((p += 3) < end)
      switch (p[0])
        {
        case '>':
  	if (p[-1] == '-' && p[-2] == '-')
  	  return p + 1;
  	break;
        case '-':
        at_dash:
  	if (p[-1] == '-')
  	  {
  	  at_dash_dash:
  	    if (++p == end) return NULL;
  	    switch (p[0])
  	      {
  	      case '>': return p + 1;
  	      case '-': goto at_dash_dash;
  	      }
  	  }
  	else
  	  {
  	    if ((p += 2) >= end) return NULL;
  	    switch (p[0])
  	      {
  	      case '>':
  		if (p[-1] == '-')
  		  return p + 1;
  		break;
  	      case '-':
  		goto at_dash;
  	      }
  	  }
        }
    return NULL;
  }
  
  /* Return non-zero of the string inside [b, e) are present in hash
     table HT.  */
  
  static int
  name_allowed (const struct hash_table *ht, const char *b, const char *e)
  {
    char *copy;
    if (!ht)
      return 1;
    BOUNDED_TO_ALLOCA (b, e, copy);
    return hash_table_get (ht, copy) != NULL;
  }
  
  /* Advance P (a char pointer), with the explicit intent of being able
     to read the next character.  If this is not possible, go to finish.  */
  
  #define ADVANCE(p) do {				\
    ++p;						\
    if (p >= end)					\
      goto finish;				\
  } while (0)
  
  /* Skip whitespace, if any. */
  
  #define SKIP_WS(p) do {				\
    while (ISSPACE (*p)) {			\
      ADVANCE (p);				\
    }						\
  } while (0)
  
  /* Skip non-whitespace, if any. */
  
  #define SKIP_NON_WS(p) do {			\
    while (!ISSPACE (*p)) {			\
      ADVANCE (p);				\
    }						\
  } while (0)
  
  #ifdef STANDALONE
  static int tag_backout_count;
  #endif
  
  /* Map MAPFUN over HTML tags in TEXT, which is SIZE characters long.
     MAPFUN will be called with two arguments: pointer to an initialized
     struct taginfo, and MAPARG.
  
     ALLOWED_TAGS and ALLOWED_ATTRIBUTES are hash tables the keys of
     which are the tags and attribute names that this function should
     use.  If ALLOWED_TAGS is NULL, all tags are processed; if
     ALLOWED_ATTRIBUTES is NULL, all attributes are returned.
  
     (Obviously, the caller can filter out unwanted tags and attributes
     just as well, but this is just an optimization designed to avoid
     unnecessary copying of tags/attributes which the caller doesn't
     care about.)  */
  
  void
  map_html_tags (const char *text, int size,
  	       void (*mapfun) (struct taginfo *, void *), void *maparg,
  	       int flags,
  	       const struct hash_table *allowed_tags,
  	       const struct hash_table *allowed_attributes)
  {
    /* storage for strings passed to MAPFUN callback; if 256 bytes is
       too little, POOL_APPEND allocates more with malloc. */
    char pool_initial_storage[256];
    struct pool pool;
  
    const char *p = text;
    const char *end = text + size;
  
    struct attr_pair attr_pair_initial_storage[8];
    int attr_pair_size = countof (attr_pair_initial_storage);
    int attr_pair_resized = 0;
    struct attr_pair *pairs = attr_pair_initial_storage;
  
    if (!size)
      return;
  
    POOL_INIT (&pool, pool_initial_storage, countof (pool_initial_storage));
  
    {
      int nattrs, end_tag;
      const char *tag_name_begin, *tag_name_end;
      const char *tag_start_position;
      int uninteresting_tag;
  
    look_for_tag:
      POOL_REWIND (&pool);
  
      nattrs = 0;
      end_tag = 0;
  
      /* Find beginning of tag.  We use memchr() instead of the usual
         looping with ADVANCE() for speed. */
      p = memchr (p, '<', end - p);
      if (!p)
        goto finish;
  
      tag_start_position = p;
      ADVANCE (p);
  
      /* Establish the type of the tag (start-tag, end-tag or
         declaration).  */
      if (*p == '!')
        {
  	if (!(flags & MHT_STRICT_COMMENTS)
  	    && p < end + 3 && p[1] == '-' && p[2] == '-')
  	  {
  	    /* If strict comments are not enforced and if we know
  	       we're looking at a comment, simply look for the
  	       terminating "-->".  Non-strict is the default because
  	       it works in other browsers and most HTML writers can't
  	       be bothered with getting the comments right.  */
  	    const char *comment_end = find_comment_end (p + 3, end);
  	    if (comment_end)
  	      p = comment_end;
  	  }
  	else
  	  {
  	    /* Either in strict comment mode or looking at a non-empty
  	       declaration.  Real declarations are much less likely to
  	       be misused the way comments are, so advance over them
  	       properly regardless of strictness.  */
  	    p = advance_declaration (p, end);
  	  }
  	if (p == end)
  	  goto finish;
  	goto look_for_tag;
        }
      else if (*p == '/')
        {
  	end_tag = 1;
  	ADVANCE (p);
        }
      tag_name_begin = p;
      while (NAME_CHAR_P (*p))
        ADVANCE (p);
      if (p == tag_name_begin)
        goto look_for_tag;
      tag_name_end = p;
      SKIP_WS (p);
      if (end_tag && *p != '>')
        goto backout_tag;
  
      if (!name_allowed (allowed_tags, tag_name_begin, tag_name_end))
        /* We can't just say "goto look_for_tag" here because we need
           the loop below to properly advance over the tag's attributes.  */
        uninteresting_tag = 1;
      else
        {
  	uninteresting_tag = 0;
  	convert_and_copy (&pool, tag_name_begin, tag_name_end, AP_DOWNCASE);
        }
  
      /* Find the attributes. */
      while (1)
        {
  	const char *attr_name_begin, *attr_name_end;
  	const char *attr_value_begin, *attr_value_end;
  	const char *attr_raw_value_begin, *attr_raw_value_end;
  	int operation = AP_DOWNCASE; /* stupid compiler. */
  
  	SKIP_WS (p);
  
  	if (*p == '/')
  	  {
  	    /* A slash at this point means the tag is about to be
  	       closed.  This is legal in XML and has been popularized
  	       in HTML via XHTML.  */
  	    /* <foo a=b c=d /> */
  	    /*              ^  */
  	    ADVANCE (p);
  	    SKIP_WS (p);
  	    if (*p != '>')
  	      goto backout_tag;
  	  }
  
  	/* Check for end of tag definition. */
  	if (*p == '>')
  	  break;
  
  	/* Establish bounds of attribute name. */
  	attr_name_begin = p;	/* <foo bar ...> */
  				/*      ^        */
  	while (NAME_CHAR_P (*p))
  	  ADVANCE (p);
  	attr_name_end = p;	/* <foo bar ...> */
  				/*         ^     */
  	if (attr_name_begin == attr_name_end)
  	  goto backout_tag;
  
  	/* Establish bounds of attribute value. */
  	SKIP_WS (p);
  	if (NAME_CHAR_P (*p) || *p == '/' || *p == '>')
  	  {
  	    /* Minimized attribute syntax allows `=' to be omitted.
                 For example, <UL COMPACT> is a valid shorthand for <UL
                 COMPACT="compact">.  Even if such attributes are not
                 useful to Wget, we need to support them, so that the
                 tags containing them can be parsed correctly. */
  	    attr_raw_value_begin = attr_value_begin = attr_name_begin;
  	    attr_raw_value_end = attr_value_end = attr_name_end;
  	  }
  	else if (*p == '=')
  	  {
  	    ADVANCE (p);
  	    SKIP_WS (p);
  	    if (*p == '\"' || *p == '\'')
  	      {
  		int newline_seen = 0;
  		char quote_char = *p;
  		attr_raw_value_begin = p;
  		ADVANCE (p);
  		attr_value_begin = p; /* <foo bar="baz"> */
  				      /*           ^     */
  		while (*p != quote_char)
  		  {
  		    if (!newline_seen && *p == '\n')
  		      {
  			/* If a newline is seen within the quotes, it
  			   is most likely that someone forgot to close
  			   the quote.  In that case, we back out to
  			   the value beginning, and terminate the tag
  			   at either `>' or the delimiter, whichever
  			   comes first.  Such a tag terminated at `>'
  			   is discarded.  */
  			p = attr_value_begin;
  			newline_seen = 1;
  			continue;
  		      }
  		    else if (newline_seen && *p == '>')
  		      break;
  		    ADVANCE (p);
  		  }
  		attr_value_end = p; /* <foo bar="baz"> */
  				    /*              ^  */
  		if (*p == quote_char)
  		  ADVANCE (p);
  		else
  		  goto look_for_tag;
  		attr_raw_value_end = p;	/* <foo bar="baz"> */
  					/*               ^ */
  		operation = AP_DECODE_ENTITIES;
  		if (flags & MHT_TRIM_VALUES)
  		  operation |= AP_TRIM_BLANKS;
  	      }
  	    else
  	      {
  		attr_value_begin = p; /* <foo bar=baz> */
  				      /*          ^    */
  		/* According to SGML, a name token should consist only
  		   of alphanumerics, . and -.  However, this is often
  		   violated by, for instance, `%' in `width=75%'.
  		   We'll be liberal and allow just about anything as
  		   an attribute value.  */
  		while (!ISSPACE (*p) && *p != '>')
  		  ADVANCE (p);
  		attr_value_end = p; /* <foo bar=baz qux=quix> */
  				    /*             ^          */
  		if (attr_value_begin == attr_value_end)
  		  /* <foo bar=> */
  		  /*          ^ */
  		  goto backout_tag;
  		attr_raw_value_begin = attr_value_begin;
  		attr_raw_value_end = attr_value_end;
  		operation = AP_DECODE_ENTITIES;
  	      }
  	  }
  	else
  	  {
  	    /* We skipped the whitespace and found something that is
  	       neither `=' nor the beginning of the next attribute's
  	       name.  Back out.  */
  	    goto backout_tag;	/* <foo bar [... */
  				/*          ^    */
  	  }
  
  	/* If we're not interested in the tag, don't bother with any
             of the attributes.  */
  	if (uninteresting_tag)
  	  continue;
  
  	/* If we aren't interested in the attribute, skip it.  We
             cannot do this test any sooner, because our text pointer
             needs to correctly advance over the attribute.  */
  	if (!name_allowed (allowed_attributes, attr_name_begin, attr_name_end))
  	  continue;
  
  	GROW_ARRAY (pairs, attr_pair_size, nattrs + 1, attr_pair_resized,
  		    struct attr_pair);
  
  	pairs[nattrs].name_pool_index = pool.tail;
  	convert_and_copy (&pool, attr_name_begin, attr_name_end, AP_DOWNCASE);
  
  	pairs[nattrs].value_pool_index = pool.tail;
  	convert_and_copy (&pool, attr_value_begin, attr_value_end, operation);
  	pairs[nattrs].value_raw_beginning = attr_raw_value_begin;
  	pairs[nattrs].value_raw_size = (attr_raw_value_end
  					- attr_raw_value_begin);
  	++nattrs;
        }
  
      if (uninteresting_tag)
        {
  	ADVANCE (p);
  	goto look_for_tag;
        }
  
      /* By now, we have a valid tag with a name and zero or more
         attributes.  Fill in the data and call the mapper function.  */
      {
        int i;
        struct taginfo taginfo;
  
        taginfo.name      = pool.contents;
        taginfo.end_tag_p = end_tag;
        taginfo.nattrs    = nattrs;
        /* We fill in the char pointers only now, when pool can no
  	 longer get realloc'ed.  If we did that above, we could get
  	 hosed by reallocation.  Obviously, after this point, the pool
  	 may no longer be grown.  */
        for (i = 0; i < nattrs; i++)
  	{
  	  pairs[i].name = pool.contents + pairs[i].name_pool_index;
  	  pairs[i].value = pool.contents + pairs[i].value_pool_index;
  	}
        taginfo.attrs = pairs;
        taginfo.start_position = tag_start_position;
        taginfo.end_position   = p + 1;
        /* Ta-dam! */
        (*mapfun) (&taginfo, maparg);
        ADVANCE (p);
      }
      goto look_for_tag;
  
    backout_tag:
  #ifdef STANDALONE
      ++tag_backout_count;
  #endif
      /* The tag wasn't really a tag.  Treat its contents as ordinary
         data characters. */
      p = tag_start_position + 1;
      goto look_for_tag;
    }
  
   finish:
    POOL_FREE (&pool);
    if (attr_pair_resized)
      xfree (pairs);
  }
  
  #undef ADVANCE
  #undef SKIP_WS
  #undef SKIP_NON_WS
  
  #ifdef STANDALONE
  static void
  test_mapper (struct taginfo *taginfo, void *arg)
  {
    int i;
  
    printf ("%s%s", taginfo->end_tag_p ? "/" : "", taginfo->name);
    for (i = 0; i < taginfo->nattrs; i++)
      printf (" %s=%s", taginfo->attrs[i].name, taginfo->attrs[i].value);
    putchar ('\n');
    ++*(int *)arg;
  }
  
  int main ()
  {
    int size = 256;
    char *x = (char *)xmalloc (size);
    int length = 0;
    int read_count;
    int tag_counter = 0;
  
    while ((read_count = fread (x + length, 1, size - length, stdin)))
      {
        length += read_count;
        size <<= 1;
        x = (char *)xrealloc (x, size);
      }
  
    map_html_tags (x, length, test_mapper, &tag_counter, 0, NULL, NULL);
    printf ("TAGS: %d\n", tag_counter);
    printf ("Tag backouts:     %d\n", tag_backout_count);
    printf ("Comment backouts: %d\n", comment_backout_count);
    return 0;
  }
  #endif /* STANDALONE */
  