Rewrite line tokenizer
Ditch regex expression in favor of a state-based tokenization. This gives a bit better control over delimiting quote-based strings.
Also fixes an issue with having escaped '"' and ':' characters inside string literals.
MODIFY
src/assembler/assemblerbase.cpp
src/assembler/assemblerbase.cpp
@@ -72,10 +72,12 @@ void AssemblerBase::setDirectives(const DirectiveVec& directives) {
}
}
-std::variant<Error, LineTokens> AssemblerBase::tokenize(const QString& line, const int sourceLine) const {
- auto tokens = line.split(m_splitterRegex);
- tokens.removeAll(QLatin1String(""));
- auto joinedtokens = joinParentheses(tokens);
+std::variant<Error, LineTokens> AssemblerBase::tokenize(const QString& line, int sourceLine) const {
+ auto quoteTokenized = tokenizeQuotes(line, sourceLine);
+ if (auto* error = std::get_if<Error>(&quoteTokenized))
+ return {*error};
+
+ auto joinedtokens = joinParentheses(std::get<QStringList>(quoteTokenized));
if (auto* err = std::get_if<Error>(&joinedtokens)) {
err->first = sourceLine;
return *err;
@@ -114,6 +116,11 @@ std::variant<Error, SymbolLinePair> AssemblerBase::splitSymbolsFromLine(const Li
LineTokens splitTokens;
splitTokens.reserve(tokens.size());
for (const auto& token : tokens) {
+ if (token.startsWith('\"') && token.endsWith('\"')) {
+ // Skip quoted strings.
+ splitTokens.push_back(token);
+ continue;
+ }
Token buffer;
for (const auto& ch : token) {
buffer.append(ch);

MODIFY
src/assembler/assemblerbase.h
src/assembler/assemblerbase.h
@@ -63,7 +63,7 @@ public:
protected:
/// Creates a set of LineTokens by tokenizing a line of source code.
QRegularExpression m_splitterRegex;
- std::variant<Error, LineTokens> tokenize(const QString& line, const int sourceLine) const;
+ std::variant<Error, LineTokens> tokenize(const QString& line, int sourceLine) const;
HandleDirectiveRes assembleDirective(const DirectiveArg& arg, bool& ok, bool skipEarlyDirectives = true) const;

MODIFY
src/assembler/parserutilities.cpp
src/assembler/parserutilities.cpp
@@ -125,5 +125,47 @@ std::variant<Error, LineTokens> joinParentheses(const QStringList& tokens) {
}
}
+std::variant<Error, QStringList> tokenizeQuotes(const QString& line, unsigned sourceLine) {
+ QStringList tokens;
+ bool inQuotes = false;
+ bool escape = false;
+ QString substr;
+ auto pushSubstr = [&]{
+ tokens.push_back(substr);
+ substr.clear();
+ };
+ for (auto& ch : line) {
+ if (inQuotes) {
+ if (!escape) {
+ if (inQuotes && ch == '"') {
+ inQuotes = false;
+ substr += ch;
+ pushSubstr();
+ continue;
+ }
+ if (ch == '\\')
+ escape = true;
+ } else
+ escape = false;
+ substr.push_back(ch);
+ } else {
+ if (ch == " " || ch == "," || ch == "\t") {
+ if (!substr.isEmpty())
+ pushSubstr();
+ } else
+ substr.push_back(ch);
+ if (ch == "\"")
+ inQuotes = true;
+ }
+ }
+
+ if (inQuotes)
+ return {Error({sourceLine, "Missing terminating '\"' character."})};
+
+ tokens.push_back(substr);
+ tokens.removeAll(QLatin1String(""));
+ return {tokens};
+}
+
} // namespace Assembler
} // namespace Ripes

MODIFY
src/assembler/parserutilities.h
src/assembler/parserutilities.h
@@ -24,5 +24,7 @@ int64_t getImmediateSext32(const QString& string, bool& canConvert);
*/
std::variant<Error, LineTokens> joinParentheses(const QStringList& tokens);
+/// Quote-aware string tokenization.
+std::variant<Error, QStringList> tokenizeQuotes(const QString& line, unsigned sourceLine);
} // namespace Assembler
} // namespace Ripes

MODIFY
test/tst_assembler.cpp
test/tst_assembler.cpp
@@ -43,6 +43,7 @@ private slots:
void tst_expression();
void tst_invalidLabel();
void tst_directives();
+ void tst_stringDirectives();
void tst_riscv();
private:
@@ -331,6 +332,22 @@ void tst_Assembler::tst_labelWithPseudo() {
Expect::Success);
}
+void tst_Assembler::tst_stringDirectives() {
+ testAssemble(QStringList() << R"(A: .string "a b c")", Expect::Success);
+ testAssemble(QStringList() << R"(A: .string "a : c")", Expect::Success);
+ testAssemble(QStringList() << R"(A: .string "a \" c")", Expect::Success);
+ testAssemble(QStringList() << R"(A: .string "a \" \" c")", Expect::Success);
+ testAssemble(QStringList() << R"(A: .string "a \\" c")", Expect::Fail);
+ testAssemble(QStringList() << R"(A: .string "a c" "a c")", Expect::Fail);
+ testAssemble(QStringList() << R"(A: .string "a c\")", Expect::Fail);
+ testAssemble(QStringList() << R"(A: .string "a c)", Expect::Fail);
+ testAssemble(QStringList() << R"(A: .string a c ")", Expect::Fail);
+ testAssemble(QStringList() << R"(A: .string ")", Expect::Fail);
+ testAssemble(QStringList() << R"(A: .string a c\")", Expect::Fail);
+ testAssemble(QStringList() << R"(A: .string """")", Expect::Fail);
+ testAssemble(QStringList() << R"(A: .string "")", Expect::Success);
+}
+
void tst_Assembler::tst_matcher() {
auto isa = std::make_unique<ISAInfo<ISA::RV32I>>(QStringList());
auto assembler = RV32I_Assembler(isa.get());

Rewrite line tokenizer

Ditch regex expression in favor of a state-based tokenization. This gives a bit better control over delimiting quote-based strings.
Also fixes an issue with having escaped '"' and ':' characters inside string literals.