[Assembler] Remove comments from tokens
MODIFY
src/assembler/assembler.h
src/assembler/assembler.h
@@ -139,15 +139,18 @@ private:
if (line.isEmpty())
continue;
TokenizedSrcLine tsl;
+ tsl.sourceLine = i;
runOperation(tokens, LineTokens, tokenize, program[i]);
- runOperation(directivesAndRest, DirectivesLinePair, splitDirectivesFromLine, tokens, i);
+ // Symbols precede directives
+ runOperation(symbolsAndRest, SymbolLinePair, splitSymbolsFromLine, tokens, i);
+ tsl.symbols = symbolsAndRest.first;
+
+ runOperation(directivesAndRest, DirectivesLinePair, splitDirectivesFromLine, symbolsAndRest.second, i);
tsl.directives = directivesAndRest.first;
- runOperation(symbolsAndRest, SymbolLinePair, splitSymbolsFromLine, directivesAndRest.second, i);
- tsl.sourceLine = i;
- tsl.symbols = symbolsAndRest.first;
- tsl.tokens = symbolsAndRest.second;
+ runOperation(remainingTokens, LineTokens, splitCommentFromLine, symbolsAndRest.second, i);
+ tsl.tokens = remainingTokens;
if (!tsl.symbols.empty() && tsl.tokens.empty()) {
carry.insert(tsl.symbols.begin(), tsl.symbols.end());
@@ -180,7 +183,7 @@ private:
runOperation(expandedOps, std::optional<std::vector<LineTokens>>, expandPseudoOp, tokenizedLine);
if (expandedOps) {
/** @note: Original source line is kept for all resulting lines after pseudo-op expantion.
- * Labels, directives and comments are only kept for the first expanded op.
+ * Labels and directives are only kept for the first expanded op.
*/
const auto& eops = expandedOps.value();
for (int j = 0; j < eops.size(); j++) {
@@ -188,7 +191,6 @@ private:
tsl.tokens = eops.at(j);
tsl.sourceLine = tokenizedLine.sourceLine;
if (j == 0) {
- tsl.comments = tokenizedLine.comments;
tsl.directives = tokenizedLine.directives;
tsl.symbols = tokenizedLine.symbols;
}
@@ -398,8 +400,26 @@ protected:
return std::pair<Directives, LineTokens>(directives, remainingTokens);
}
+ virtual std::variant<Error, LineTokens> splitCommentFromLine(const LineTokens& tokens, int sourceLine) const {
+ if (tokens.size() == 0) {
+ return {tokens};
+ }
+
+ LineTokens preCommentTokens;
+ preCommentTokens.reserve(tokens.size());
+ for (const auto& token : tokens) {
+ if (token.contains(commentDelimiter())) {
+ break;
+ } else {
+ preCommentTokens.append(token);
+ }
+ }
+ return {preCommentTokens};
+ }
+
virtual QString instrSegment() const { return QStringLiteral(".text"); }
virtual QString dataSegment() const { return QStringLiteral(".data"); }
+ virtual QChar commentDelimiter() const = 0;
/**
* @brief m_instructions is the set of instructions which can be matched from an instruction string as well as be

MODIFY
src/assembler/assembler_defines.h
src/assembler/assembler_defines.h
@@ -23,7 +23,6 @@ using ReverseSymbolMap = std::map<uint32_t, Symbol>;
using AssembleDirectiveRes = std::variant<Error, std::optional<QByteArray>>;
struct TokenizedSrcLine {
Symbols symbols;
- LineTokens comments;
LineTokens tokens;
Directives directives;
unsigned sourceLine = 0;

MODIFY
src/assembler/rv32i_assembler.h
src/assembler/rv32i_assembler.h
@@ -29,6 +29,9 @@ public:
void enableExtI(RVInstrVec& instructions, RVPseudoInstrVec& pseudoInstructions) const;
void enableExtM(RVInstrVec& instructions, RVPseudoInstrVec& pseudoInstructions) const;
void enableExtF(RVInstrVec& instructions, RVPseudoInstrVec& pseudoInstructions) const;
+
+protected:
+ QChar commentDelimiter() const override { return '#'; }
};
} // namespace AssemblerTmp

MODIFY
test/tst_assembler.cpp
test/tst_assembler.cpp
@@ -29,7 +29,7 @@ private:
void tst_Assembler::tst_simpleprogram() {
auto assembler = RV32I_Assembler({});
- QStringList program = QStringList() << "addi a0 a0 123"
+ QStringList program = QStringList() << "addi a0 a0 123 # Hello world"
<< "nop";
auto res = assembler.assemble(program);
if (res.errors.size() != 0) {

[Assembler] Remove comments from tokens