text: #include <rff/parsing.hpp>
  
  #include <algorithm>
  #include <cctype>
  
  bool rff::parsing::is_space(char c)
  {
  	return std::isspace(static_cast<unsigned char>(c));
  };
  
  std::string rff::parsing::remove_comments(const std::string& str, std::string_view single_line_comment, 
                                            std::string_view multiline_comment_start, std::string_view multiline_comment_end,
                                            const char string_literal, const char escape_character)
  {
  	std::string ret(std::size(str), '\0');
  	std::string::iterator cur = std::begin(ret);
  	
  	enum
  	{
  		IN_SINGLE_LINE_COMMENT,
  		IN_MULTILINE_COMMENT,
  		IN_STRING_LITERAL,
  		NONE
  	} state = NONE;
  
  	// Hold last n characters and compare them against single_line_comment and multiline_comment when needed
  	// Ensure that size is at least 2 (minimum current + escape char)
  	std::string char_tracker(
  		std::max(
  			{
  				std::size(single_line_comment),
  				std::size(multiline_comment_start),
  				std::size(multiline_comment_end),
  				static_cast<size_t>(2)
  			}),
  		'\0');
  
  	// Iterate over every character, update char_tracker and state machine
  	// fill ret string if needed
  	for(char c : str)
  	{
  		// Update char_tracker
  		std::shift_left(std::begin(char_tracker), std::end(char_tracker), 1);
  		char_tracker.back() = c;
  
  		auto remove_comment = [&cur](const std::string_view& sv) -> void
  		{
  			if (std::size(sv) != 1) // remove only if size is greater that 1
  				cur -= std::size(sv) - 1;
  		};
  		
  		if(state == NONE && c == string_literal) // entering string literal
  		{
  			state = IN_STRING_LITERAL;
  		}
  		else if(state == IN_STRING_LITERAL && c == string_literal && *(std::end(char_tracker) - 2) != escape_character) // leaving string literal
  			// we are in string literal, current character is a string literal, previous character wasn't escape character
  			// for example: \" won't escape
  		{
  			state = NONE;
  		}
  		else if(state == NONE && char_tracker.ends_with(single_line_comment)) // entering single line comment
  		{
  			state = IN_SINGLE_LINE_COMMENT;
  			remove_comment(single_line_comment);
  		}
  		else if(state == IN_SINGLE_LINE_COMMENT && c == '\n') // leaving single line comment
  		{
  			state = NONE;
  		}
  		else if(state == NONE && char_tracker.ends_with(multiline_comment_start)) // entering multi-line comment
  		{
  			state = IN_MULTILINE_COMMENT;
  			remove_comment(multiline_comment_start);
  		}
  		else if(state == IN_MULTILINE_COMMENT && char_tracker.ends_with(multiline_comment_end))
  		{
  			state = NONE;
  			remove_comment(multiline_comment_end);
  			continue; // we don't want to output last char of multiline_comment_end to our finished product
  		}
  
  		if(state == IN_SINGLE_LINE_COMMENT || state == IN_MULTILINE_COMMENT)
  			continue; // remove comments
  
  		// Update current char and increase output
  		*(cur++) = c;
  	}
  	
  	ret.erase(cur, std::end(ret));
  	ret.shrink_to_fit(); // TODO: Is this needed here?
  	return ret;
  }
  
  std::vector<std::string> rff::parsing::tokenize(const std::string& str,
  	const std::initializer_list<std::string_view>& unique_tokens, const char string_literal,
  	const char escape_character, bool(* is_space)(char))
  {
  	std::vector<std::string> ret;
  
  	enum
  	{
  		IN_STRING_LITERAL,
  		NONE
  	} state = NONE;
  
  	// Hold last n characters and compare them against single_line_comment and multiline_comment when needed
  	// Ensure that size is at least 2 (minimum current + escape char)
  	size_t char_tracker_size = 2;
  	for (auto& t : unique_tokens)
  		char_tracker_size = std::max(char_tracker_size, std::size(t));
  
  	std::string char_tracker(char_tracker_size, '\0');
  
  	std::string token;
  	auto new_token = [&token, &ret]() -> void
  	{
  		if (!token.empty())
  		{
  			ret.shrink_to_fit(); // TODO: What happens with capacity on copy?
  			ret.push_back(token);
  			token.clear();
  		}
  	};
  
  	for (char c : str)
  	{
  		// Update char_tracker
  		std::shift_left(std::begin(char_tracker), std::end(char_tracker), 1);
  		char_tracker.back() = c;
  
  		if (state == NONE && c == string_literal) // entering string literal
  		{
  			state = IN_STRING_LITERAL;
  			new_token();
  		}
  		else if (state == IN_STRING_LITERAL && c == string_literal && *(std::end(char_tracker) - 2) != escape_character) // leaving string literal
  			// we are in string literal, current character is a string literal, previous character wasn't escape character
  			// for example: \" won't escape
  		{
  			state = NONE;
  			new_token();
  		}
  		// TODO: Insert escape characters like \n \t \0
  		else if(state == NONE && is_space(c))
  			// Separate by white space
  		{
  			new_token();
  		}
  		else if(state == NONE)
  			// Separate by special characters
  		{
  			const auto res = std::ranges::find_if(unique_tokens,
  				[&char_tracker](const std::string_view& str)-> bool
  				{
  					return char_tracker.ends_with(str);
  				});
  
  			if(res != std::end(unique_tokens))
  			{
  				if(std::size(*res) != 1) // remove unique token from token
  					token.erase(std::end(token) - std::size(*res), std::end(token));
  
  				new_token(); // push old token
  				ret.emplace_back(*res); // push found token
  			}
  			else
  			{
  				goto update_token;
  			}
  		}
  		else
  		{
  		update_token:
  			token.push_back(c);
  		}
  	}
  
  	// push last token if it is there
  	new_token();
  
  	ret.shrink_to_fit();
  	return ret;
  }
  