#include "tree_fragment_tokenizer.h"

#include <cctype>

namespace MosesTraining
{
namespace Syntax
{

TreeFragmentToken::TreeFragmentToken(TreeFragmentTokenType t,
                                     StringPiece v, std::size_t p)
    : type(t)
    , value(v)
    , pos(p) {
}

TreeFragmentTokenizer::TreeFragmentTokenizer()
    : value_(TreeFragmentToken_EOS, "", -1) {
}

TreeFragmentTokenizer::TreeFragmentTokenizer(const StringPiece &s)
    : str_(s)
    , value_(TreeFragmentToken_EOS, "", -1)
    , iter_(s.begin())
    , end_(s.end())
    , pos_(0) {
  ++(*this);
}

TreeFragmentTokenizer &TreeFragmentTokenizer::operator++() {
  while (iter_ != end_ && (*iter_ == ' ' || *iter_ == '\t')) {
    ++iter_;
    ++pos_;
  }

  if (iter_ == end_) {
    value_ = TreeFragmentToken(TreeFragmentToken_EOS, "", pos_);
    return *this;
  }

  if (*iter_ == '[') {
    value_ = TreeFragmentToken(TreeFragmentToken_LSB, "[", pos_);
    ++iter_;
    ++pos_;
  } else if (*iter_ == ']') {
    value_ = TreeFragmentToken(TreeFragmentToken_RSB, "]", pos_);
    ++iter_;
    ++pos_;
  } else {
    std::size_t start = pos_;
    while (true) {
      ++iter_;
      ++pos_;
      if (iter_ == end_ || *iter_ == ' ' || *iter_ == '\t') {
        break;
      }
      if (*iter_ == '[' || *iter_ == ']') {
        break;
      }
    }
    StringPiece word = str_.substr(start, pos_-start);
    value_ = TreeFragmentToken(TreeFragmentToken_WORD, word, start);
  }

  return *this;
}

TreeFragmentTokenizer TreeFragmentTokenizer::operator++(int) {
  TreeFragmentTokenizer tmp(*this);
  ++*this;
  return tmp;
}

bool operator==(const TreeFragmentTokenizer &lhs,
                const TreeFragmentTokenizer &rhs) {
  if (lhs.value_.type == TreeFragmentToken_EOS ||
      rhs.value_.type == TreeFragmentToken_EOS) {
    return lhs.value_.type == TreeFragmentToken_EOS &&
           rhs.value_.type == TreeFragmentToken_EOS;
  }
  return lhs.iter_ == rhs.iter_;
}

bool operator!=(const TreeFragmentTokenizer &lhs,
                const TreeFragmentTokenizer &rhs) {
  return !(lhs == rhs);
}

}  // namespace Syntax
}  // namespace MosesTraining
