message: A more nuanced lexing of streams that are being parsed
diff: @@ -23,6 +23,7 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.*/
  #include "newParser.h"
  #include "Log.h"
  +#include <cctype>
  #include <fstream>
  @@ -97,7 +98,10 @@ void commonItems::parser::parseFile(const std::string& filename)
  std::optional<std::string> commonItems::parser::getNextToken(std::istream& theStream)
  {
  + theStream >> std::noskipws;
  +
  std::string toReturn;
  +
  bool gotToken = false;
  while (!gotToken)
  {
  @@ -106,14 +110,37 @@ std::optional<std::string> commonItems::parser::getNextToken(std::istream& theSt
  return {};
  }
  - theStream >> toReturn;
  - if (toReturn.substr(0, 1) == "#")
  + toReturn = "";
  + while (true)
  + {
  + char inputChar;
  + theStream >> inputChar;
  + if (theStream.eof())
  + {
  + break;
  + }
  + else if (inputChar == '#')
  {
  std::string bitbucket;
  std::getline(theStream, bitbucket);
  + if (toReturn.size() > 0)
  + {
  + break;
  + }
  + }
  + else if (std::isspace(inputChar))
  + {
  + if (toReturn.size() > 0)
  + {
  + break;
  + }
  }
  else
  {
  + toReturn += inputChar;
  + }
  + }
  +
  bool matched = false;
  for (auto registration: registeredKeywords)
  {
  @@ -131,7 +158,6 @@ std::optional<std::string> commonItems::parser::getNextToken(std::istream& theSt
  gotToken = true;
  }
  }
  - }
  return toReturn;
  }
  