text: /* tokenizer - A tokenizer structure that will chop up file into
   * tokens.  It is aware of quoted strings and otherwise tends to return
   * white-space or punctuated-separated words, with punctuation in
   * a separate token.  This is used by autoSql. */
  
  #ifndef TOKENIZER_H
  #define TOKENIZER_H
  
  struct tokenizer
  /* This handles reading in tokens. */
      {
      bool reuse;	         /* True if want to reuse this token. */
      bool eof;            /* True at end of file. */
      int leadingSpaces;	 /* Number of leading spaces before token. */
      struct lineFile *lf; /* Underlying file. */
      char *curLine;       /* Current line of text. */
      char *linePt;        /* Start position within current line. */
      char *string;        /* String value of token */
      int sSize;           /* Size of string. */
      int sAlloc;          /* Allocated string size. */
        /* Some variables set after tokenizerNew to control details of
         * parsing. */
      bool leaveQuotes;	 /* Leave quotes in string. */
      bool uncommentC;	 /* Take out C (and C++) style comments. */
      bool uncommentShell; /* Take out # style comments. */
      };
  
  struct tokenizer *tokenizerNew(char *fileName);
  /* Return a new tokenizer. */
  
  struct tokenizer *tokenizerOnLineFile(struct lineFile *lf);
  /* Create a new tokenizer on open lineFile. */
  
  void tokenizerFree(struct tokenizer **pTkz);
  /* Tear down a tokenizer. */
  
  void tokenizerReuse(struct tokenizer *tkz);
  /* Reuse token. */
  
  int tokenizerLineCount(struct tokenizer *tkz);
  /* Return line of current token. */
  
  char *tokenizerFileName(struct tokenizer *tkz);
  /* Return name of file. */
  
  char *tokenizerNext(struct tokenizer *tkz);
  /* Return token's next string (also available as tkz->string) or
   * NULL at EOF. This string will be overwritten with the next call
   * to tokenizerNext, so cloneString if you need to save it. */
  
  void tokenizerErrAbort(struct tokenizer *tkz, char *format, ...);
  /* Print error message followed by file and line number and
   * abort. */
  
  void tokenizerNotEnd(struct tokenizer *tkz);
  /* Squawk if at end. */
  
  char *tokenizerMustHaveNext(struct tokenizer *tkz);
  /* Get next token, which must be there. */
  
  void tokenizerMustMatch(struct tokenizer *tkz, char *string);
  /* Require next token to match string.  Return next token
   * if it does, otherwise abort. */
  
  #endif /* TOKENIZER_H */
  
  