text: #include <cassert>
  #include <memory>
  
  #include "exp.h"
  #include "scanner.h"
  #include "token.h"
  #include "yaml-cpp/exceptions.h"  // IWYU pragma: keep
  
  namespace YAML {
  Scanner::Scanner(std::istream& in)
      : INPUT(in),
        m_startedStream(false),
        m_endedStream(false),
        m_simpleKeyAllowed(false),
        m_canBeJSONFlow(false) {}
  
  Scanner::~Scanner() {}
  
  // empty
  // . Returns true if there are no more tokens to be read
  bool Scanner::empty() {
    EnsureTokensInQueue();
    return m_tokens.empty();
  }
  
  // pop
  // . Simply removes the next token on the queue.
  void Scanner::pop() {
    EnsureTokensInQueue();
    if (!m_tokens.empty())
      m_tokens.pop();
  }
  
  // peek
  // . Returns (but does not remove) the next token on the queue.
  Token& Scanner::peek() {
    EnsureTokensInQueue();
    assert(!m_tokens.empty());  // should we be asserting here? I mean, we really
                                // just be checking
                                // if it's empty before peeking.
  
  #if 0
  		static Token *pLast = 0;
  		if(pLast != &m_tokens.front())
  			std::cerr << "peek: " << m_tokens.front() << "\n";
  		pLast = &m_tokens.front();
  #endif
  
    return m_tokens.front();
  }
  
  // mark
  // . Returns the current mark in the stream
  Mark Scanner::mark() const { return INPUT.mark(); }
  
  // EnsureTokensInQueue
  // . Scan until there's a valid token at the front of the queue,
  //   or we're sure the queue is empty.
  void Scanner::EnsureTokensInQueue() {
    while (1) {
      if (!m_tokens.empty()) {
        Token& token = m_tokens.front();
  
        // if this guy's valid, then we're done
        if (token.status == Token::VALID)
          return;
  
        // here's where we clean up the impossible tokens
        if (token.status == Token::INVALID) {
          m_tokens.pop();
          continue;
        }
  
        // note: what's left are the unverified tokens
      }
  
      // no token? maybe we've actually finished
      if (m_endedStream)
        return;
  
      // no? then scan...
      ScanNextToken();
    }
  }
  
  // ScanNextToken
  // . The main scanning function; here we branch out and
  //   scan whatever the next token should be.
  void Scanner::ScanNextToken() {
    if (m_endedStream)
      return;
  
    if (!m_startedStream)
      return StartStream();
  
    // get rid of whitespace, etc. (in between tokens it should be irrelevent)
    ScanToNextToken();
  
    // maybe need to end some blocks
    PopIndentToHere();
  
    // *****
    // And now branch based on the next few characters!
    // *****
  
    // end of stream
    if (!INPUT)
      return EndStream();
  
    if (INPUT.column() == 0 && INPUT.peek() == Keys::Directive)
      return ScanDirective();
  
    // document token
    if (INPUT.column() == 0 && Exp::DocStart().Matches(INPUT))
      return ScanDocStart();
  
    if (INPUT.column() == 0 && Exp::DocEnd().Matches(INPUT))
      return ScanDocEnd();
  
    // flow start/end/entry
    if (INPUT.peek() == Keys::FlowSeqStart || INPUT.peek() == Keys::FlowMapStart)
      return ScanFlowStart();
  
    if (INPUT.peek() == Keys::FlowSeqEnd || INPUT.peek() == Keys::FlowMapEnd)
      return ScanFlowEnd();
  
    if (INPUT.peek() == Keys::FlowEntry)
      return ScanFlowEntry();
  
    // block/map stuff
    if (Exp::BlockEntry().Matches(INPUT))
      return ScanBlockEntry();
  
    if ((InBlockContext() ? Exp::Key() : Exp::KeyInFlow()).Matches(INPUT))
      return ScanKey();
  
    if (GetValueRegex().Matches(INPUT))
      return ScanValue();
  
    // alias/anchor
    if (INPUT.peek() == Keys::Alias || INPUT.peek() == Keys::Anchor)
      return ScanAnchorOrAlias();
  
    // tag
    if (INPUT.peek() == Keys::Tag)
      return ScanTag();
  
    // special scalars
    if (InBlockContext() && (INPUT.peek() == Keys::LiteralScalar ||
                             INPUT.peek() == Keys::FoldedScalar))
      return ScanBlockScalar();
  
    if (INPUT.peek() == '\'' || INPUT.peek() == '\"')
      return ScanQuotedScalar();
  
    // plain scalars
    if ((InBlockContext() ? Exp::PlainScalar() : Exp::PlainScalarInFlow())
            .Matches(INPUT))
      return ScanPlainScalar();
  
    // don't know what it is!
    throw ParserException(INPUT.mark(), ErrorMsg::UNKNOWN_TOKEN);
  }
  
  // ScanToNextToken
  // . Eats input until we reach the next token-like thing.
  void Scanner::ScanToNextToken() {
    while (1) {
      // first eat whitespace
      while (INPUT && IsWhitespaceToBeEaten(INPUT.peek())) {
        if (InBlockContext() && Exp::Tab().Matches(INPUT))
          m_simpleKeyAllowed = false;
        INPUT.eat(1);
      }
  
      // then eat a comment
      if (Exp::Comment().Matches(INPUT)) {
        // eat until line break
        while (INPUT && !Exp::Break().Matches(INPUT))
          INPUT.eat(1);
      }
  
      // if it's NOT a line break, then we're done!
      if (!Exp::Break().Matches(INPUT))
        break;
  
      // otherwise, let's eat the line break and keep going
      int n = Exp::Break().Match(INPUT);
      INPUT.eat(n);
  
      // oh yeah, and let's get rid of that simple key
      InvalidateSimpleKey();
  
      // new line - we may be able to accept a simple key now
      if (InBlockContext())
        m_simpleKeyAllowed = true;
    }
  }
  
  ///////////////////////////////////////////////////////////////////////
  // Misc. helpers
  
  // IsWhitespaceToBeEaten
  // . We can eat whitespace if it's a space or tab
  // . Note: originally tabs in block context couldn't be eaten
  //         "where a simple key could be allowed
  //         (i.e., not at the beginning of a line, or following '-', '?', or
  // ':')"
  //   I think this is wrong, since tabs can be non-content whitespace; it's just
  //   that they can't contribute to indentation, so once you've seen a tab in a
  //   line, you can't start a simple key
  bool Scanner::IsWhitespaceToBeEaten(char ch) {
    if (ch == ' ')
      return true;
  
    if (ch == '\t')
      return true;
  
    return false;
  }
  
  // GetValueRegex
  // . Get the appropriate regex to check if it's a value token
  const RegEx& Scanner::GetValueRegex() const {
    if (InBlockContext())
      return Exp::Value();
  
    return m_canBeJSONFlow ? Exp::ValueInJSONFlow() : Exp::ValueInFlow();
  }
  
  // StartStream
  // . Set the initial conditions for starting a stream.
  void Scanner::StartStream() {
    m_startedStream = true;
    m_simpleKeyAllowed = true;
    std::auto_ptr<IndentMarker> pIndent(new IndentMarker(-1, IndentMarker::NONE));
    m_indentRefs.push_back(pIndent);
    m_indents.push(&m_indentRefs.back());
  }
  
  // EndStream
  // . Close out the stream, finish up, etc.
  void Scanner::EndStream() {
    // force newline
    if (INPUT.column() > 0)
      INPUT.ResetColumn();
  
    PopAllIndents();
    PopAllSimpleKeys();
  
    m_simpleKeyAllowed = false;
    m_endedStream = true;
  }
  
  Token* Scanner::PushToken(Token::TYPE type) {
    m_tokens.push(Token(type, INPUT.mark()));
    return &m_tokens.back();
  }
  
  Token::TYPE Scanner::GetStartTokenFor(IndentMarker::INDENT_TYPE type) const {
    switch (type) {
      case IndentMarker::SEQ:
        return Token::BLOCK_SEQ_START;
      case IndentMarker::MAP:
        return Token::BLOCK_MAP_START;
      case IndentMarker::NONE:
        assert(false);
        break;
    }
    assert(false);
    throw std::runtime_error("yaml-cpp: internal error, invalid indent type");
  }
  
  // PushIndentTo
  // . Pushes an indentation onto the stack, and enqueues the
  //   proper token (sequence start or mapping start).
  // . Returns the indent marker it generates (if any).
  Scanner::IndentMarker* Scanner::PushIndentTo(int column,
                                               IndentMarker::INDENT_TYPE type) {
    // are we in flow?
    if (InFlowContext())
      return 0;
  
    std::auto_ptr<IndentMarker> pIndent(new IndentMarker(column, type));
    IndentMarker& indent = *pIndent;
    const IndentMarker& lastIndent = *m_indents.top();
  
    // is this actually an indentation?
    if (indent.column < lastIndent.column)
      return 0;
    if (indent.column == lastIndent.column &&
        !(indent.type == IndentMarker::SEQ &&
          lastIndent.type == IndentMarker::MAP))
      return 0;
  
    // push a start token
    indent.pStartToken = PushToken(GetStartTokenFor(type));
  
    // and then the indent
    m_indents.push(&indent);
    m_indentRefs.push_back(pIndent);
    return &m_indentRefs.back();
  }
  
  // PopIndentToHere
  // . Pops indentations off the stack until we reach the current indentation
  // level,
  //   and enqueues the proper token each time.
  // . Then pops all invalid indentations off.
  void Scanner::PopIndentToHere() {
    // are we in flow?
    if (InFlowContext())
      return;
  
    // now pop away
    while (!m_indents.empty()) {
      const IndentMarker& indent = *m_indents.top();
      if (indent.column < INPUT.column())
        break;
      if (indent.column == INPUT.column() && !(indent.type == IndentMarker::SEQ &&
                                               !Exp::BlockEntry().Matches(INPUT)))
        break;
  
      PopIndent();
    }
  
    while (!m_indents.empty() && m_indents.top()->status == IndentMarker::INVALID)
      PopIndent();
  }
  
  // PopAllIndents
  // . Pops all indentations (except for the base empty one) off the stack,
  //   and enqueues the proper token each time.
  void Scanner::PopAllIndents() {
    // are we in flow?
    if (InFlowContext())
      return;
  
    // now pop away
    while (!m_indents.empty()) {
      const IndentMarker& indent = *m_indents.top();
      if (indent.type == IndentMarker::NONE)
        break;
  
      PopIndent();
    }
  }
  
  // PopIndent
  // . Pops a single indent, pushing the proper token
  void Scanner::PopIndent() {
    const IndentMarker& indent = *m_indents.top();
    m_indents.pop();
  
    if (indent.status != IndentMarker::VALID) {
      InvalidateSimpleKey();
      return;
    }
  
    if (indent.type == IndentMarker::SEQ)
      m_tokens.push(Token(Token::BLOCK_SEQ_END, INPUT.mark()));
    else if (indent.type == IndentMarker::MAP)
      m_tokens.push(Token(Token::BLOCK_MAP_END, INPUT.mark()));
  }
  
  // GetTopIndent
  int Scanner::GetTopIndent() const {
    if (m_indents.empty())
      return 0;
    return m_indents.top()->column;
  }
  
  // ThrowParserException
  // . Throws a ParserException with the current token location
  //   (if available).
  // . Does not parse any more tokens.
  void Scanner::ThrowParserException(const std::string& msg) const {
    Mark mark = Mark::null_mark();
    if (!m_tokens.empty()) {
      const Token& token = m_tokens.front();
      mark = token.mark;
    }
    throw ParserException(mark, msg);
  }
  }
  