Added better support to paragraphs, lists and dividers
MODIFY
packages/@romefrontend/ast/markdown/blocks/MarkdownDividerBlock.ts
packages/@romefrontend/ast/markdown/blocks/MarkdownDividerBlock.ts
@@ -3,6 +3,7 @@ import {createBuilder} from "../../utils";
export type MarkdownDividerBlock = NodeBaseWithComments & {
type: "MarkdownDividerBlock";
+ value: string;
};
export const markdownDividerBlock = createBuilder<MarkdownDividerBlock>(

MODIFY
packages/@romefrontend/ast/markdown/blocks/MarkdownListBlock.ts
packages/@romefrontend/ast/markdown/blocks/MarkdownListBlock.ts
@@ -3,7 +3,7 @@ import {createBuilder} from "../../utils";
export type MarkdownListBlock = NodeBaseWithComments & {
type: "MarkdownListBlock";
- kind: "dot-list" | "numeric-list";
+ ordered: boolean;
children: Array<MarkdownListItem>;
};

MODIFY
packages/@romefrontend/ast/markdown/core/MarkdownListItem.ts
packages/@romefrontend/ast/markdown/core/MarkdownListItem.ts
@@ -4,8 +4,10 @@ import {MarkdownListChildren} from "@romefrontend/ast/markdown/unions";
export type MarkdownListItem = NodeBaseWithComments & {
type: "MarkdownListItem";
- checked: boolean | null;
+ checked: boolean | undefined;
children: Array<MarkdownListChildren>;
+ // the value of ordered list: 1./-/*
+ value: string | undefined;
};
export const markdownListItem = createBuilder<MarkdownListItem>(

MODIFY
packages/@romefrontend/formatter/builders/markdown/blocks/MarkdownDividerBlock.ts
packages/@romefrontend/formatter/builders/markdown/blocks/MarkdownDividerBlock.ts
@@ -5,5 +5,5 @@ export default function MarkdownDividerBlock(
builder: Builder,
node: MarkdownDividerBlock,
): Token {
- throw new Error("unimplemented");
+ return node.value;
}

MODIFY
packages/@romefrontend/formatter/builders/markdown/blocks/MarkdownHeadingBlock.ts
packages/@romefrontend/formatter/builders/markdown/blocks/MarkdownHeadingBlock.ts
import {MarkdownHeadingBlock} from "@romefrontend/ast";
-import {Builder, Token} from "@romefrontend/formatter";
+import {Builder, Token, concat, hardline, space} from "@romefrontend/formatter";
export default function MarkdownHeadingBlock(
builder: Builder,
node: MarkdownHeadingBlock,
): Token {
- throw new Error("unimplemented");
+ return concat(["#".repeat(node.level), space, node.value, hardline]);
}

MODIFY
packages/@romefrontend/formatter/builders/markdown/blocks/MarkdownListBlock.ts
packages/@romefrontend/formatter/builders/markdown/blocks/MarkdownListBlock.ts
import {MarkdownListBlock} from "@romefrontend/ast";
-import {Builder, Token} from "@romefrontend/formatter";
+import {
+ Builder,
+ Token,
+ Tokens,
+ concat,
+ hardline,
+ space,
+} from "@romefrontend/formatter";
export default function MarkdownListBlock(
builder: Builder,
node: MarkdownListBlock,
): Token {
- throw new Error("unimplemented");
+ const tokens: Tokens = node.children.reduce(
+ (tokens, child, index) => {
+ if (node.ordered) {
+ tokens.push(`${index + 1}.`);
+ tokens.push(space);
+ } else {
+ if (child.value) {
+ tokens.push(`${child.value}`);
+ tokens.push(space);
+ }
+ }
+ tokens.push(builder.tokenize(child, node));
+ if (index + 1 < node.children.length) {
+ tokens.push(hardline);
+ }
+
+ return tokens;
+ },
+ ([] as Tokens),
+ );
+ return concat(tokens);
}

MODIFY
packages/@romefrontend/formatter/builders/markdown/core/MarkdownListItem.ts
packages/@romefrontend/formatter/builders/markdown/core/MarkdownListItem.ts
import {MarkdownListItem} from "@romefrontend/ast";
-import {Builder, Token} from "@romefrontend/formatter";
+import {Builder, Token, Tokens, concat} from "@romefrontend/formatter";
export default function MarkdownListItem(
builder: Builder,
node: MarkdownListItem,
): Token {
- throw new Error("unimplemented");
+ const tokens: Tokens = [];
+ if (node.checked !== undefined) {
+ tokens.push(`[${node.checked ? "x" : " "}]`);
+ }
+
+ tokens.push(builder.tokenizeStatementList(node.children, node));
+
+ return concat(tokens);
}

MODIFY
packages/@romefrontend/formatter/builders/markdown/core/MarkdownParagraph.ts
packages/@romefrontend/formatter/builders/markdown/core/MarkdownParagraph.ts
import {MarkdownParagraph} from "@romefrontend/ast";
-import {Builder, Token} from "@romefrontend/formatter";
+import {Builder, Token, Tokens, concat} from "@romefrontend/formatter";
export default function MarkdownParagraph(
builder: Builder,
node: MarkdownParagraph,
): Token {
- throw new Error("unimplemented");
+ const tokens: Tokens = node.children.map((child) => {
+ return builder.tokenize(child, node);
+ });
+ return concat(tokens);
}

MODIFY
packages/@romefrontend/markdown-parser/parse.ts
packages/@romefrontend/markdown-parser/parse.ts
@@ -4,7 +4,11 @@ import {
isDigit,
readUntilLineBreak,
} from "@romefrontend/parser-core";
-import {State, Tokens} from "@romefrontend/markdown-parser/types";
+import {
+ ListProperties,
+ State,
+ Tokens,
+} from "@romefrontend/markdown-parser/types";
import {
MarkdownDividerBlock,
MarkdownHeadingBlock,
@@ -60,13 +64,13 @@ export const createMarkdownParser = createParser((
const nextNextChar = this.getInputCharOnly(index, 2);
if (hasThematicBreak([currentChar, nextChar, nextNextChar].join(""))) {
// by spec, should be at least 3, with an infinite number
- const [, endIndex] = this.readInputFrom(
+ const [value, endIndex] = this.readInputFrom(
index,
(char) => {
return char === blockChar;
},
);
- return this.finishToken("Break", endIndex);
+ return this.finishValueToken("Break", value, endIndex);
}
return undefined;
}
@@ -90,6 +94,32 @@ export const createMarkdownParser = createParser((
state,
};
}
+ if (nextChar === "\n") {
+ return {
+ token: this.finishToken("EndParagraph", ob1Add(index, 1)),
+ state,
+ };
+ }
+
+ if (isDigit(nextChar)) {
+ const [, endIndex] = this.readInputFrom(index, isDigit);
+ const nextChar = this.getInputCharOnly(endIndex);
+ const nextNextChar = this.getInputCharOnly(endIndex, 1);
+
+ if (nextChar === "." && nextNextChar === " ") {
+ return {
+ token: this.finishComplexToken<"ListItem", ListProperties>(
+ "ListItem",
+ {
+ numeric: true,
+ checked: undefined,
+ },
+ ob1Add(endIndex, 2),
+ ),
+ state,
+ };
+ }
+ }
return {
token: this.finishToken("NewLine"),
@@ -103,6 +133,21 @@ export const createMarkdownParser = createParser((
if (block) {
return {token: block, state};
}
+ const nextChar = this.getInputCharOnly(index, 1);
+ if (nextChar === " ") {
+ return {
+ token: this.finishComplexToken<"ListItem", ListProperties>(
+ "ListItem",
+ {
+ numeric: false,
+ checked: undefined,
+ value: "-",
+ },
+ ob1Add(index, 2),
+ ),
+ state,
+ };
+ }
}
if (char === "_") {
const block = this.consumeBlock("_", index, char);
@@ -115,17 +160,36 @@ export const createMarkdownParser = createParser((
if (block) {
return {token: block, state};
}
+ const nextChar = this.getInputCharOnly(index, 1);
+ if (nextChar === " ") {
+ return {
+ token: this.finishComplexToken<"ListItem", ListProperties>(
+ "ListItem",
+ {
+ numeric: false,
+ checked: undefined,
+ value: "*",
+ },
+ ob1Add(index, 2),
+ ),
+ state,
+ };
+ }
}
if (isDigit(char)) {
- const nextChar = this.getInputCharOnly(index, 1);
- const nextNextChar = this.getInputCharOnly(index, 2);
+ const [, endIndex] = this.readInputFrom(index, isDigit);
+ const nextChar = this.getInputCharOnly(endIndex);
+ const nextNextChar = this.getInputCharOnly(endIndex, 1);
if (nextChar === "." && nextNextChar === " ") {
return {
- token: this.finishValueToken(
+ token: this.finishComplexToken<"ListItem", ListProperties>(
"ListItem",
- "numeric-list",
- ob1Add(index, 3),
+ {
+ numeric: true,
+ checked: undefined,
+ },
+ ob1Add(endIndex, 2),
),
state,
};
@@ -175,20 +239,50 @@ export const createMarkdownParser = createParser((
);
}
- parseParagraph(): MarkdownParagraph {
+ parseParagraph(isList?: boolean): MarkdownParagraph {
const start = this.getPosition();
const children: Array<AnyMarkdownInlineNode> = [];
-
- while (!this.matchToken("EOF") && !this.matchToken("NewLine")) {
+ while (
+ !this.matchToken("EOF") &&
+ !this.matchToken("EndParagraph") &&
+ !this.matchToken("Break")
+ ) {
const token = this.getToken();
+ if (isList && token.type === "NewLine") {
+ this.nextToken();
+ break;
+ }
switch (token.type) {
+ case "Break": {
+ break;
+ }
case "Text": {
children.push(this.parseText());
break;
}
+ case "NewLine": {
+ const pos = this.getPosition();
+ children.push(
+ this.finishNode(
+ pos,
+ {
+ type: "MarkdownText",
+ value: "\n",
+ },
+ ),
+ );
+ this.nextToken();
+ break;
}
+ default: {
+ // TODO: to remove once all cases are handled
+ this.unexpectedDiagnostic({
+ description: descriptions.MARKDOWN_PARSER.INVALID_SEQUENCE,
+ });
this.nextToken();
}
+ }
+ }
return this.finishNode(
start,
@@ -200,33 +294,39 @@ export const createMarkdownParser = createParser((
}
parseBreak(): MarkdownDividerBlock {
- this.expectToken("Break");
+ const token = this.expectToken("Break");
const start = this.getPosition();
return this.finishNode(
start,
{
type: "MarkdownDividerBlock",
+ value: token.value,
},
);
}
parseItem(): MarkdownListItem {
- this.expectToken("ListItem");
+ const token = this.expectToken("ListItem");
const pos = this.getPosition();
const children: Array<MarkdownListChildren> = [];
- while (!this.matchToken("EOF") && this.matchToken("Text")) {
- children.push(this.parseParagraph());
+ while (
+ !this.matchToken("EOF") &&
+ !this.matchToken("NewLine") &&
+ this.matchToken("Text")
+ ) {
+ children.push(this.parseParagraph(true));
}
return this.finishNode(
pos,
{
// TODO handle check
- checked: null,
+ checked: token.checked,
type: "MarkdownListItem",
children,
+ value: token.value,
},
);
}
@@ -234,7 +334,13 @@ export const createMarkdownParser = createParser((
parseListBlock(): MarkdownListBlock {
const pos = this.getPosition();
const children: Array<MarkdownListItem> = [];
-
+ const currentToken = this.getToken();
+ let ordered = false;
+ if (currentToken.type === "ListItem") {
+ if (currentToken.numeric === true) {
+ ordered = true;
+ }
+ }
while (!this.matchToken("EOF") && this.matchToken("ListItem")) {
const item = this.parseItem();
children.push(item);
@@ -244,21 +350,22 @@ export const createMarkdownParser = createParser((
pos,
{
type: "MarkdownListBlock",
-
- // TODO to review the type of list
- kind: "dot-list",
+ ordered,
children,
},
);
}
parseBlock(): undefined | AnyMarkdownNode {
const token = this.getToken();
-
switch (token.type) {
case "NewLine": {
this.nextToken();
return undefined;
}
+ case "EndParagraph": {
+ this.nextToken();
+ return undefined;
+ }
case "HeadingLevel":
return this.parseHeading();

MODIFY
packages/@romefrontend/markdown-parser/test-fixtures/smoke/input.test.md
packages/@romefrontend/markdown-parser/test-fixtures/smoke/input.test.md
@@ -131,8 +131,8 @@ MarkdownRoot {
loc: Object {
filename: "input.md"
end: Object {
- column: 3
- index: 99
+ column: 0
+ index: 96
line: 8
}
start: Object {
@@ -158,15 +158,47 @@ MarkdownRoot {
}
}
}
+ MarkdownText {
+ value: "\n"
+ loc: Object {
+ filename: "input.md"
+ end: Object {
+ column: 32
+ index: 95
+ line: 7
+ }
+ start: Object {
+ column: 32
+ index: 95
+ line: 7
+ }
+ }
+ }
]
}
- MarkdownListBlock {
- kind: "dot-list"
+ MarkdownDividerBlock {
+ value: "___"
loc: Object {
filename: "input.md"
end: Object {
column: 3
- index: 132
+ index: 99
+ line: 8
+ }
+ start: Object {
+ column: 3
+ index: 99
+ line: 8
+ }
+ }
+ }
+ MarkdownListBlock {
+ ordered: true
+ loc: Object {
+ filename: "input.md"
+ end: Object {
+ column: 0
+ index: 129
line: 11
}
start: Object {
@@ -177,13 +209,14 @@ MarkdownRoot {
}
children: Array [
MarkdownListItem {
- checked: null
+ value: undefined
+ checked: undefined
loc: Object {
filename: "input.md"
end: Object {
- column: 3
- index: 132
- line: 11
+ column: 0
+ index: 114
+ line: 10
}
start: Object {
column: 3
@@ -196,9 +229,9 @@ MarkdownRoot {
loc: Object {
filename: "input.md"
end: Object {
- column: 3
- index: 132
- line: 11
+ column: 0
+ index: 114
+ line: 10
}
start: Object {
column: 3
@@ -223,6 +256,42 @@ MarkdownRoot {
}
}
}
+ ]
+ }
+ ]
+ }
+ MarkdownListItem {
+ value: undefined
+ checked: undefined
+ loc: Object {
+ filename: "input.md"
+ end: Object {
+ column: 0
+ index: 129
+ line: 11
+ }
+ start: Object {
+ column: 3
+ index: 117
+ line: 10
+ }
+ }
+ children: Array [
+ MarkdownParagraph {
+ loc: Object {
+ filename: "input.md"
+ end: Object {
+ column: 0
+ index: 129
+ line: 11
+ }
+ start: Object {
+ column: 3
+ index: 117
+ line: 10
+ }
+ }
+ children: Array [
MarkdownText {
value: "second item"
loc: Object {
@@ -245,12 +314,28 @@ MarkdownRoot {
}
]
}
- MarkdownParagraph {
+ MarkdownDividerBlock {
+ value: "---"
loc: Object {
filename: "input.md"
end: Object {
column: 3
- index: 146
+ index: 132
+ line: 11
+ }
+ start: Object {
+ column: 3
+ index: 132
+ line: 11
+ }
+ }
+ }
+ MarkdownParagraph {
+ loc: Object {
+ filename: "input.md"
+ end: Object {
+ column: 0
+ index: 143
line: 13
}
start: Object {
@@ -276,8 +361,40 @@ MarkdownRoot {
}
}
}
+ MarkdownText {
+ value: "\n"
+ loc: Object {
+ filename: "input.md"
+ end: Object {
+ column: 9
+ index: 142
+ line: 12
+ }
+ start: Object {
+ column: 9
+ index: 142
+ line: 12
+ }
+ }
+ }
]
}
+ MarkdownDividerBlock {
+ value: "***"
+ loc: Object {
+ filename: "input.md"
+ end: Object {
+ column: 3
+ index: 146
+ line: 13
+ }
+ start: Object {
+ column: 3
+ index: 146
+ line: 13
+ }
+ }
+ }
]
}
```

MODIFY
packages/@romefrontend/markdown-parser/types.ts
packages/@romefrontend/markdown-parser/types.ts
import {
BaseTokens,
+ ComplexToken,
ParserCoreState,
ParserOptions,
SimpleToken,
@@ -18,11 +19,18 @@ export type State = ParserCoreState & {
isBlockHead: boolean;
};
+export type ListProperties = {
+ checked: boolean | undefined;
+ numeric: boolean;
+ value?: string;
+};
+
export type Tokens = BaseTokens & {
HeadingLevel: ValueToken<"HeadingLevel", number>;
Greater: SimpleToken<"Greater">;
Text: ValueToken<"Text", string>;
NewLine: SimpleToken<"NewLine">;
- Break: SimpleToken<"Break">;
- ListItem: ValueToken<"ListItem", string>;
+ Break: ValueToken<"Break", string>;
+ ListItem: ComplexToken<"ListItem", ListProperties>;
+ EndParagraph: SimpleToken<"EndParagraph">;
};

Added better support to paragraphs, lists and dividers (#859)

Co-authored-by: Florent Cailhol <florent.cailhol@gmail.com>