text: // file      : libbuild2/cc/lexer.cxx -*- C++ -*-
  // license   : MIT; see accompanying LICENSE file
  
  #include <libbuild2/cc/lexer.hxx>
  
  using namespace std;
  using namespace butl;
  
  // bit 0 - identifier character (_0-9A-Ba-b).
  //
  static const uint8_t char_flags[256] =
  //0    1    2    3    4    5    6    7      8    9    A    B    C    D    E    F
  {
    0,   0,   0,   0,   0,   0,   0,   0,     0,   0,   0,   0,   0,   0,   0,   0, // 0
    0,   0,   0,   0,   0,   0,   0,   0,     0,   0,   0,   0,   0,   0,   0,   0, // 1
    0,   0,   0,   0,   0,   0,   0,   0,     0,   0,   0,   0,   0,   0,   0,   0, // 2
    1,   1,   1,   1,   1,   1,   1,   1,     1,   1,   0,   0,   0,   0,   0,   0, // 3
    0,   1,   1,   1,   1,   1,   1,   1,     1,   1,   1,   1,   1,   1,   1,   1, // 4
    1,   1,   1,   1,   1,   1,   1,   1,     1,   1,   1,   0,   0,   0,   0,   1, // 5
    0,   1,   1,   1,   1,   1,   1,   1,     1,   1,   1,   1,   1,   1,   1,   1, // 6
    1,   1,   1,   1,   1,   1,   1,   1,     1,   1,   1,   0,   0,   0,   0,   0, // 7
  
    // 128-255
    0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0,
    0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,  0,0,0,0,0,0,0,0
  };
  
  // Diagnostics plumbing.
  //
  namespace butl // ADL
  {
    inline build2::location
    get_location (const butl::char_scanner<>::xchar& c, const void* data)
    {
      using namespace build2;
  
      assert (data != nullptr); // E.g., must be &lexer::name_.
      return location (*static_cast<const path_name*> (data), c.line, c.column);
    }
  }
  
  namespace build2
  {
    namespace cc
    {
      auto lexer::
      peek (bool e) -> xchar
      {
        if (ungetn_ != 0)
          return ungetb_[ungetn_ - 1];
  
        if (unpeek_)
          return unpeekc_;
  
        xchar c (base::peek ());
  
        if (e && c == '\\')
        {
          get (c);
          xchar p (base::peek ());
  
          // Handle Windows CRLF sequence. Similar to char_scanner, we treat a
          // single CR as if it was followed by LF and also collapse multiple
          // CRs.
          //
          while (p == '\r')
          {
            get (p);
            p = base::peek ();
  
            if (p == '\n')
              break;
  
            // Pretend '\n' was there and recurse.
            //
            if (p != '\r')
              return peek (e);
          }
  
          if (p == '\n')
          {
            get (p);
            return peek (e); // Recurse.
          }
  
          // Save in the unpeek buffer so that it is returned on the subsequent
          // calls to peek() (until get()).
          //
          unpeek_ = true;
          unpeekc_ = c;
        }
  
        return c;
      }
  
      inline auto lexer::
      get (bool e) -> xchar
      {
        if (ungetn_ != 0)
          return ungetb_[--ungetn_];
        else
        {
          xchar c (peek (e));
          get (c);
          return c;
        }
      }
  
      inline void lexer::
      get (const xchar& c)
      {
        // Increment the logical line similar to how base will increment the
        // physical (the column counts are the same).
        //
        if (log_line_ && c == '\n' && ungetn_ == 0)
          ++*log_line_;
  
        base::get (c);
      }
  
      inline auto lexer::
      geth (bool e) -> xchar
      {
        xchar c (get (e));
        cs_.append (c);
        return c;
      }
  
      inline void lexer::
      geth (const xchar& c)
      {
        get (c);
        cs_.append (c);
      }
  
      using type = token_type;
  
      void lexer::
      next (token& t, xchar c, bool ignore_pp)
      {
        for (;; c = skip_spaces ())
        {
          t.file = &log_file_;
          t.line = log_line_ ? *log_line_ : c.line;
          t.column = c.column;
  
          if (eos (c))
          {
            t.type = type::eos;
            return;
          }
  
          const location l (name_, c.line, c.column);
  
          // Hash the token's line. The reason is debug info. In fact, doing
          // this will make quite a few "noop" changes (like adding a newline
          // anywhere in the source) cause the checksum change. But there
          // doesn't seem to be any way around it: the case where we benefit
          // from the precise change detection the most (development) is also
          // where we will most likely have debug info enable.
          //
          // Note that in order not to make this completely useless we don't
          // hash the column. Even if it is part of the debug info, having it a
          // bit off shouldn't cause any significant mis-positioning. We also
          // don't hash the file path for each token instead only hashing it
          // when changed with the #line directive (as well as in the
          // constructor for the initial path).
          //
          cs_.append (t.line);
          cs_.append (c);
  
          switch (c)
          {
            // Preprocessor lines.
            //
          case '#':
            {
              // It is tempting to simply scan until the newline ignoring
              // anything in between. However, these lines can start a
              // multi-line C-style comment. So we have to tokenize them (and
              // hash the data for each token).
              //
              // Note that this may not work for things like #error that can
              // contain pretty much anything. Also note that lines that start
              // with '#' can contain '#' further down. In this case we need to
              // be careful not to recurse (and consume multiple newlines). Thus
              // the ignore_pp flag.
              //
              // Finally, to support diagnostics properly we need to recognize
              // #line directives.
              //
              if (ignore_pp)
              {
                for (bool first (true);;)
                {
                  // Note that we keep using the passed token for buffers.
                  //
                  c = skip_spaces (false); // Stop at newline.
  
                  if (eos (c) || c == '\n')
                    break;
  
                  if (first)
                  {
                    first = false;
  
                    // Recognize #line and its shorthand version:
                    //
                    // #line <integer> [<string literal>] ...
                    // #     <integer> [<string literal>] ...
                    //
                    // Also diagnose #include while at it.
                    //
                    if (!(c >= '0' && c <= '9'))
                    {
                      next (t, c, false);
  
                      if (t.type == type::identifier)
                      {
                        if (t.value == "include")
                          fail (l) << "unexpected #include directive";
                        else if (t.value != "line")
                          continue;
                      }
                      else
                        continue;
  
                      if (t.type != type::identifier || t.value != "line")
                        continue;
  
                      c = skip_spaces (false);
  
                      if (!(c >= '0' && c <= '9'))
                        fail (c) << "line number expected after #line directive";
                    }
  
                    // Ok, this is #line and next comes the line number.
                    //
                    line_directive (t, c);
                    continue; // Parse the tail, if any.
                  }
  
                  next (t, c, false);
                }
                break;
              }
              else
              {
                t.type = type::punctuation;
                return;
              }
            }
            // Single-letter punctuation.
            //
          case ';': t.type = type::semi;    return;
          case '{': t.type = type::lcbrace; return;
          case '}': t.type = type::rcbrace; return;
            // Other single-letter punctuation.
            //
          case '(':
          case ')':
          case '[':
          case ']':
          case ',':
          case '?':
          case '~':
          case '\\': t.type = type::punctuation; return;
            // Potentially multi-letter punctuation.
            //
          case '.': // . .* .<N> ...
            {
              xchar p (peek ());
  
              if (p == '*')
              {
                geth (p);
                t.type = type::punctuation;
                return;
              }
              else if (p >= '0' && p <= '9')
              {
                number_literal (t, c);
                return;
              }
              else if (p == '.')
              {
                get (p);
  
                xchar q (peek ());
                if (q == '.')
                {
                  cs_.append (p);
  
                  geth (q);
                  t.type = type::punctuation;
                  return;
                }
                unget (p);
                // Fall through.
              }
  
              t.type = type::dot;
              return;
            }
          case '=': // = ==
          case '!': // ! !=
          case '*': // * *=
          case '/': // / /=   (/* and // handled by skip_spaced() above)
          case '%': // % %=
          case '^': // ^ ^=
            {
              xchar p (peek ());
  
              if (p == '=')
                geth (p);
  
              t.type = type::punctuation;
              return;
            }
          case '<': // < <= << <<=
          case '>': // > >= >> >>=
            {
              xchar p (peek ());
  
              if (p == c)
              {
                geth (p);
                if ((p = peek ()) == '=')
                  geth (p);
                t.type = type::punctuation;
              }
              else if (p == '=')
              {
                geth (p);
                t.type = type::punctuation;
              }
              else
                t.type = (c == '<' ? type::less : type::greater);
  
              return;
            }
          case '+': // + ++ +=
          case '-': // - -- -= -> ->*
            {
              xchar p (peek ());
  
              if (p == c || p == '=')
                geth (p);
              else if (c == '-' && p == '>')
              {
                geth (p);
                if ((p = peek ()) == '*')
                  geth (p);
              }
  
              t.type = type::punctuation;
              return;
            }
          case '&': // & && &=
          case '|': // | || |=
            {
              xchar p (peek ());
  
              if (p == c || p == '=')
                geth (p);
  
              t.type = type::punctuation;
              return;
            }
          case ':': // : ::
            {
              xchar p (peek ());
  
              if (p == ':')
                geth (p);
  
              t.type = type::punctuation;
              return;
            }
            // Number (and also .<N> above).
            //
          case '0':
          case '1':
          case '2':
          case '3':
          case '4':
          case '5':
          case '6':
          case '7':
          case '8':
          case '9':
            {
              number_literal (t, c);
              return;
            }
            // Char/string literal, identifier, or other (\, $, @, `).
            //
          default:
            {
              bool raw (false); // Raw string literal.
  
              // Note: known not to be a digit (see above).
              //
              if (char_flags[static_cast<uint8_t> (c)] & 0x01)
              {
                // This smells a little: we know skip_spaces() did not peek at
                // the next character because this is not '/'. Which means the
                // position in the stream must be of this character + 1.
                //
                t.position = buf_->tellg () - 1;
  
                string& id (t.value);
                id = c;
  
                while (char_flags[static_cast<uint8_t> (c = peek ())] & 0x01)
                {
                  geth (c);
                  id += c;
  
                  // Direct buffer scan. Note that we always follow up with the
                  // normal peek() call which may load the next chunk, handle
                  // line continuations, etc. In other words, the end of the
                  // "raw" scan doesn't necessarily mean the end.
                  //
                  const char* b (gptr_);
                  const char* p (b);
  
                  for (const char* e (egptr_);
                       p != e && char_flags[static_cast<uint8_t> (*p)] & 0x01;
                       ++p) ;
  
                  // Unrolling this loop doesn't make a difference.
                  //
                  // for (const char* e (egptr_ - 4); p < e; p += 4)
                  // {
                  //   uint8_t c;
                  //
                  //  c = static_cast<uint8_t> (p[0]);
                  //  if (!(char_flags[c] & 0x01)) break;
                  //
                  //  c = static_cast<uint8_t> (p[1]);
                  //  if (!(char_flags[c] & 0x01)) {p += 1; break;}
                  //
                  //  c = static_cast<uint8_t> (p[2]);
                  //  if (!(char_flags[c] & 0x01)) {p += 2; break;}
                  //
                  //  c = static_cast<uint8_t> (p[3]);
                  //  if (!(char_flags[c] & 0x01)) {p += 3; break;}
                  // }
  
                  size_t n (p - b);
                  id.append (b, n); cs_.append (b, n);
                  gptr_ = p; buf_->gbump (static_cast<int> (n)); column += n;
                }
  
                // If the following character is a quote, see if the identifier
                // is one of the literal prefixes.
                //
                if (c == '\'' || c == '\"')
                {
                  size_t n (id.size ()), i (0);
                  switch (id[0])
                  {
                  case 'u':
                    {
                      if (n > 1 && id[1] == '8')
                        ++i;
                    }
                    // Fall through.
                  case 'L':
                  case 'U':
                    {
                      ++i;
  
                      if (c == '\"' && n > i && id[i] == 'R')
                      {
                        ++i;
                        raw = true;
                      }
                      break;
                    }
                  case 'R':
                    {
                      if (c == '\"')
                      {
                        ++i;
                        raw = true;
                      }
                      break;
                    }
                  }
  
                  if (i == n) // All characters "consumed".
                  {
                    geth (c);
                    id.clear ();
                  }
                }
  
                if (!id.empty ())
                {
                  t.type = type::identifier;
                  return;
                }
              }
  
              switch (c)
              {
              case '\'':
                {
                  char_literal (t, c);
                  return;
                }
              case '\"':
                {
                  if (raw)
                    raw_string_literal (t, c);
                  else
                    string_literal (t, c);
                  return;
                }
              default:
                {
                  t.type = type::other;
                  return;
                }
              }
            }
          }
        }
      }
  
      void lexer::
      number_literal (token& t, xchar c)
      {
        // note: c is hashed
  
        // A number (integer or floating point literal) can:
        //
        // 1. Start with a dot (which must be followed by a digit, e.g., .123).
        //
        // 2. Can have a radix prefix (0b101, 0123, 0X12AB).
        //
        // 3. Can have an exponent (1e10, 0x1.p-10, 1.).
        //
        // 4. Digits can be separated with ' (123'456, 0xff00'00ff).
        //
        // 5. End with a built-in or user defined literal (123f, 123UL, 123_X)
        //
        // Quoting from GCC's preprocessor documentation:
        //
        // "Formally preprocessing numbers begin with an optional period, a
        // required decimal digit, and then continue with any sequence of
        // letters, digits, underscores, periods, and exponents. Exponents are
        // the two-character sequences 'e+', 'e-', 'E+', 'E-', 'p+', 'p-', 'P+',
        // and 'P-'."
        //
        // So it looks like a "C++ number" is then any unseparated (with
        // whitespace or punctuation) sequence of those plus '. The only mildly
        // tricky part is then to recognize +/- as being part of the exponent.
        //
        while (!eos ((c = peek ())))
        {
          switch (c)
          {
            // All the whitespace, punctuation, and other characters that end
            // the number.
            //
          case ' ':
          case '\n':
          case '\t':
          case '\r':
          case '\f':
          case '\v':
  
          case '#':
          case ';':
          case '{':
          case '}':
          case '(':
          case ')':
          case '[':
          case ']':
          case ',':
          case '?':
          case '~':
          case '=':
          case '!':
          case '*':
          case '/':
          case '%':
          case '^':
          case '>':
          case '<':
          case '&':
          case '|':
          case ':':
          case '+': // The exponent case is handled below.
          case '-': // The exponent case is handled below.
          case '"':
          case '\\':
  
          case '@':
          case '$':
          case '`':
            break;
  
            // Recognize +/- after the exponent.
            //
          case 'e':
          case 'E':
          case 'p':
          case 'P':
            {
              geth (c);
              c = peek ();
              if (c == '+' || c == '-')
                geth (c);
              continue;
            }
  
          case '_':
          case '.':
          case '\'':
          default: // Digits and letters.
            {
              geth (c);
              continue;
            }
          }
  
          break;
        }
  
        t.type = type::number;
      }
  
      void lexer::
      char_literal (token& t, xchar c)
      {
        // note: c is hashed
  
        const location l (name_, c.line, c.column);
  
        for (char p (c);;) // Previous character (see below).
        {
          c = geth ();
  
          if (eos (c) || c == '\n')
            fail (l) << "unterminated character literal";
  
          if (c == '\'' && p != '\\')
            break;
  
          // Keep track of \\-escapings so we don't confuse them with \', as in
          // '\\'.
          //
          p = (c == '\\' && p == '\\') ? '\0' : static_cast<char> (c);
        }
  
        // See if we have a user-defined suffix (which is an identifier).
        //
        if ((c = peek ()) == '_' || alpha (c))
          literal_suffix (c);
  
        t.type = type::character;
      }
  
      void lexer::
      string_literal (token& t, xchar c)
      {
        // note: c is hashed
  
        const location l (name_, c.line, c.column);
  
        for (char p (c);;) // Previous character (see below).
        {
          c = geth ();
  
          if (eos (c) || c == '\n')
            fail (l) << "unterminated string literal";
  
          if (c == '\"' && p != '\\')
            break;
  
          // Keep track of \\-escapings so we don't confuse them with \", as in
          // "\\".
          //
          p = (c == '\\' && p == '\\') ? '\0' : static_cast<char> (c);
  
          // Direct buffer scan.
          //
          if (p != '\\')
          {
            const char* b (gptr_);
            const char* e (egptr_);
            const char* p (b);
  
            for (char c;
                 p != e &&
                 (c = *p) != '\"' && c != '\\' && c != '\n' && c != '\r';
                 ++p) ;
  
            size_t n (p - b);
            cs_.append (b, n);
            gptr_ = p; buf_->gbump (static_cast<int> (n)); column += n;
          }
        }
  
        // See if we have a user-defined suffix (which is an identifier).
        //
        if ((c = peek ()) == '_' || alpha (c))
          literal_suffix (c);
  
        t.type = type::string;
      }
  
      void lexer::
      raw_string_literal (token& t, xchar c)
      {
        // note: c is hashed
  
        // The overall form is:
        //
        // R"<delimiter>(<raw_characters>)<delimiter>"
        //
        // Where <delimiter> is a potentially-empty character sequence made of
        // any source character but parentheses, backslash and spaces. It can be
        // at most 16 characters long.
        //
        // Note that the <raw_characters> are not processed in any way, not even
        // for line continuations.
        //
        const location l (name_, c.line, c.column);
  
        // As a first step, parse the delimiter (including the openning paren).
        //
        string d (1, ')');
  
        for (;;)
        {
          c = geth ();
  
          if (eos (c) || c == '\"' || c == ')' || c == '\\' || c == ' ')
            fail (l) << "invalid raw string literal";
  
          if (c == '(')
            break;
  
          d += c;
        }
  
        d += '"';
  
        // Now parse the raw characters while trying to match the closing
        // delimiter.
        //
        for (size_t i (0);;) // Position to match in d.
        {
          c = geth (false); // No newline escaping.
  
          if (eos (c)) // Note: newline is ok.
            fail (l) << "invalid raw string literal";
  
          if (c != d[i] && i != 0) // Restart from the beginning.
            i = 0;
  
          if (c == d[i])
          {
            if (++i == d.size ())
              break;
          }
        }
  
        // See if we have a user-defined suffix (which is an identifier).
        //
        if ((c = peek ()) == '_' || alpha (c))
          literal_suffix (c);
  
        t.type = type::string;
      }
  
      void lexer::
      literal_suffix (xchar c)
      {
        // note: c is unhashed
  
        // Parse a user-defined literal suffix identifier.
        //
        for (geth (c); (c = peek ()) == '_' || alnum (c); geth (c)) ;
      }
  
      void lexer::
      line_directive (token& t, xchar c)
      {
        // enter: first digit of the line number
        // leave: last character of the line number or file string
        // note:  c is unhashed
  
        // If our number and string tokens contained the literal values, then we
        // could have used that. However, we ignore the value (along with escape
        // processing, etc), for performance. Let's keep it that way and instead
        // handle it ourselves.
        //
        // Note also that we are not hashing these at the character level
        // instead hashing the switch to a new file path below and leaving the
        // line number to the token line hashing.
        //
        {
          string& s (t.value);
  
          for (s = c; (c = peek ()) >= '0' && c <= '9'; get (c))
            s += c;
  
          // The newline that ends the directive will increment the logical line
          // so subtract one to compensate. Note: can't be 0 and shouldn't throw
          // for valid lines.
          //
          log_line_ = stoull (s.c_str ()) - 1;
        }
  
        // See if we have the file.
        //
        c = skip_spaces (false);
  
        if (c == '\"')
        {
          const location l (name_, c.line, c.column);
  
          // It is common to have a large number of #line directives that don't
          // change the file (they seem to be used to track macro locations or
          // some such). So we are going to optimize for this by comparing the
          // current path to what's in #line.
          //
          string& s (tmp_file_);
          s.clear ();
  
          for (char p ('\0'); p != '\"'; ) // Previous character.
          {
            c = get ();
  
            if (eos (c) || c == '\n')
              fail (l) << "unterminated string literal";
  
            // Handle escapes.
            //
            if (p == '\\')
            {
              p = '\0'; // Clear so we don't confuse \" and \\".
  
              // We only handle what can reasonably be expected in a file name.
              //
              switch (c)
              {
              case '\\':
              case '\'':
              case '\"': break; // Add as is.
              default:
                fail (c) << "unsupported escape sequence in #line directive";
              }
            }
            else
            {
              p = c;
  
              switch (c)
              {
              case '\\':
              case '\"': continue;
              }
            }
  
            s += c;
  
            // Direct buffer scan.
            //
            if (p != '\\')
            {
              const char* b (gptr_);
              const char* e (egptr_);
              const char* p (b);
  
              for (char c;
                   p != e &&
                   (c = *p) != '\"' && c != '\\' && c != '\n' && c != '\r';
                   ++p) ;
  
              size_t n (p - b);
              s.append (b, n);
              gptr_ = p; buf_->gbump (static_cast<int> (n)); column += n;
            }
          }
  
          try
          {
            if (s.empty ())
              throw invalid_path ("");
  
            // Handle special names (<stdin>, <built-in>, etc).
            //
            if (s.front () == '<' && s.back () == '>')
            {
              if (log_file_.name)
              {
                if (*log_file_.name == s)
                  return;
  
                log_file_.name->swap (s);
              }
              else
                log_file_.name = move (s);
  
              log_file_.path.clear ();
            }
            else
            {
              if (log_file_.path.string () == s)
                return;
  
              string r (move (log_file_.path).string ()); // Move string rep out.
              r.swap (s);
              log_file_.path = path (move (r)); // Move back in.
  
              log_file_.name = nullopt;
            }
          }
          catch (const invalid_path&)
          {
            fail (l) << "invalid path in #line directive";
          }
  
          // If the path is relative, then prefix it with the current working
          // directory. Failed that, we will end up with different checksums for
          // invocations from different directories.
          //
          // While this should work fine for normal cross-compilation, it's an
          // entirely different story for the emulated case (e.g., msvc-linux
          // where the preprocessed output contains absolute Windows paths). So
          // we try to sense if things look fishy and leave the path alone.
          //
          // Also detect special names like <built-in> and <command-line>. Plus
          // GCC sometimes adds what looks like working directory (has trailing
          // slash). So ignore that as well.
          //
          // We now switched to using absolute translation unit paths (because
          // of __FILE__/assert(); see compile.cxx for details). But we might
          // still need this logic when we try to calculate location-independent
          // hash for distributed compilation/caching. The idea is to only hash
          // the part starting from the project root which is immutable. Plus
          // we will need -ffile-prefix-map to deal with __FILE__.
          //
          if (!log_file_.path.to_directory ()) // Also covers special names.
            cs_.append (log_file_.name
                        ? *log_file_.name
                        : log_file_.path.string ());
  #if 0
          {
            using tr = path::traits_type;
            const string& f (log_file_.path.string ());
  
            if (log_file_.name               ||
                f.find (':') != string::npos ||
                log_file_.path.absolute ())
              cs_.append (log_file_.name ? *log_file_.name : f);
            else
            {
              // This gets complicated and slow: the path may contain '..' and
              // '.'  so strictly speaking we would need to normalize it.
              // Instead, we are going to handle leading '..'s ourselves (the
              // sane case) and ignore everything else (so if you have '..'  or
              // '.' somewhere in the middle, then things might not work
              // optimally for you).
              //
              const string& d (work.string ());
  
              // Iterate over leading '..' in f "popping" the corresponding
              // number of trailing components from d.
              //
              size_t fp (0);
              size_t dp (d.size () - 1);
  
              for (size_t p;; )
              {
                // Note that in file we recognize any directory separator, not
                // just of this platform (see note about emulation above).
                //
                if (f.compare (fp, 2, "..") != 0  ||
                    (f[fp + 2] != '/' && f[fp + 2] != '\\') || // Could be '\0'.
                    (p = tr::rfind_separator (d, dp)) == string::npos)
                  break;
  
                fp += 3;
                dp = p - 1;
              }
  
              cs_.append (d.c_str (), dp + 1);
              cs_.append (tr::directory_separator); // Canonical in work.
              cs_.append (f.c_str () + fp);
            }
          }
  #endif
        }
        else
          unget (c);
      }
  
      auto lexer::
      skip_spaces (bool nl) -> xchar
      {
        xchar c (get ());
  
        for (; !eos (c); c = get ())
        {
          switch (c)
          {
          case '\n':
            if (!nl) break;
            // Fall through.
          case ' ':
          case '\t':
          case '\r':
          case '\f':
          case '\v':
            {
              // Direct buffer scan.
              //
              const char* b (gptr_);
              const char* e (egptr_);
              const char* p (b);
  
              for (char c;
                   p != e && ((c = *p) == ' ' || c == '\t');
                   ++p) ;
  
              size_t n (p - b);
              gptr_ = p; buf_->gbump (static_cast<int> (n)); column += n;
  
              continue;
            }
          case '/':
            {
              xchar p (peek ());
  
              // C++ comment.
              //
              if (p == '/')
              {
                get (p);
  
                for (;;)
                {
                  c = get ();
                  if (c == '\n' || eos (c))
                    break;
  
                  // Direct buffer scan.
                  //
                  const char* b (gptr_);
                  const char* e (egptr_);
                  const char* p (b);
  
                  for (char c;
                       p != e && (c = *p) != '\n' && c != '\\';
                       ++p) ;
  
                  size_t n (p - b);
                  gptr_ = p; buf_->gbump (static_cast<int> (n)); column += n;
                }
  
                if (!nl)
                  break;
  
                continue;
              }
  
              // C comment.
              //
              if (p == '*')
              {
                get (p);
  
                for (;;)
                {
                  c = get ();
  
                  if (eos (c))
                    fail (p) << "unterminated comment";
  
                  if (c == '*' && (c = peek ()) == '/')
                  {
                    get (c);
                    break;
                  }
  
                  if (c != '*' && c != '\\')
                  {
                    // Direct buffer scan.
                    //
                    // Note that we should call get() prior to the direct buffer
                    // scan (see butl::char_scanner for details).
                    //
                    get (c);
  
                    const char* b (gptr_);
                    const char* e (egptr_);
                    const char* p (b);
  
                    for (char c;
                         p != e && (c = *p) != '*' && c != '\\';
                         ++p)
                    {
                      if (c == '\n')
                      {
                        if (log_line_) ++*log_line_;
                        ++line;
                        column = 1;
                      }
                      else
                        ++column;
                    }
  
                    gptr_ = p; buf_->gbump (static_cast<int> (p - b));
                  }
                }
                continue;
              }
              break;
            }
          }
          break;
        }
  
        return c;
      }
  
      ostream&
      operator<< (ostream& o, const token& t)
      {
        switch (t.type)
        {
        case type::dot:         o << "'.'";                   break;
        case type::semi:        o << "';'";                   break;
        case type::less:        o << "'<'";                   break;
        case type::greater:     o << "'>'";                   break;
        case type::lcbrace:     o << "'{'";                   break;
        case type::rcbrace:     o << "'}'";                   break;
        case type::punctuation: o << "<punctuation>";         break;
  
        case type::identifier:  o << '\'' << t.value << '\''; break;
  
        case type::number:      o << "<number literal>";      break;
        case type::character:   o << "<char literal>";        break;
        case type::string:      o << "<string literal>";      break;
  
        case type::other:       o << "<other>";               break;
        case type::eos:         o << "<end of file>";         break;
        }
  
        return o;
      }
    }
  }
  