fix(markdown-parser): handle text with single newline as same paragraph
MODIFY
internal/markdown-parser/index.ts
internal/markdown-parser/index.ts
@@ -70,40 +70,6 @@ const markdownParser = createParser<MarkdownParserTypes>({
if (char === "#") {
return [state, consumeHeading(parser, index)];
}
- if (char === "\n") {
- const nextChar = parser.getInputCharOnly(index.increment());
- if (nextChar === "#") {
- return [state, consumeHeading(parser, index.add(1))];
- }
-
- if (nextChar === "`") {
- const token = consumeCode(parser, index.add(1));
- if (token) {
- return [state, token];
- }
- }
-
- if (isDigit(nextChar)) {
- const token = tokenizeListItem(parser, index);
- if (token) {
- return [
- {
- isParagraph: true,
- isListItem: true,
- },
- token,
- ];
- }
- }
-
- return [
- {
- isParagraph: false,
- isListItem: false,
- },
- parser.finishToken("NewLine"),
- ];
- }
if (char === "-") {
const block = tokenizeBlock(parser, "-", index, char);

MODIFY
internal/markdown-parser/parser/paragraph.ts
internal/markdown-parser/parser/paragraph.ts
@@ -4,6 +4,7 @@ import {parseInline} from "@internal/markdown-parser/parser/inline";
import {descriptions} from "@internal/diagnostics";
import {parseText} from "@internal/markdown-parser/parser/text";
import {parseReference} from "@internal/markdown-parser/parser/reference";
+import {Position} from "@internal/parser-core";
export function parseParagraph(
parser: MarkdownParser,
@@ -11,13 +12,34 @@ export function parseParagraph(
): MarkdownParagraph {
const start = parser.getPosition();
const children: AnyMarkdownInlineNode[] = [];
- while (!(parser.matchToken("EOF") || isBlockToken(parser))) {
+ let endPos: Position | null = null;
+ while (!parser.matchToken("EOF")) {
const token = parser.getToken();
- if (isList && token.type === "NewLine") {
- parser.nextToken();
+ if (token.type === "NewLine") {
+ if (isList) {
+ break;
+ }
+ const currentPos = parser.getPosition();
+ const next = parser.nextToken();
+ if (next.type === "NewLine" || next.type === "EOF" || isBlockToken(next)) {
+ // avoid including NewLine or EOF in paragraph
+ endPos = currentPos;
break;
+ } else {
+ children.push(
+ parser.finishNode(
+ currentPos,
+ {
+ type: "MarkdownText",
+ value: "\n",
+ },
+ ),
+ );
+ continue;
+ }
}
+
switch (token.type) {
case "Strong":
case "Emphasis": {
@@ -45,20 +67,6 @@ export function parseParagraph(
parser.nextToken();
break;
}
- case "NewLine": {
- const pos = parser.getPosition();
- children.push(
- parser.finishNode(
- pos,
- {
- type: "MarkdownText",
- value: "\n",
- },
- ),
- );
- parser.nextToken();
- break;
- }
case "OpenSquareBracket": {
const reference = parseReference(parser);
if (Array.isArray(reference)) {
@@ -73,14 +81,16 @@ export function parseParagraph(
// TODO: to remove once all cases are handled
parser.unexpectedDiagnostic({
description: descriptions.MARKDOWN_PARSER.INVALID_SEQUENCE,
+ token,
});
parser.nextToken();
}
}
}
- return parser.finishNode(
+ return parser.finishNodeAt(
start,
+ endPos || parser.getLastEndPosition(),
{
type: "MarkdownParagraph",
children,

MODIFY
internal/markdown-parser/test-fixtures/smoke/input.md
internal/markdown-parser/test-fixtures/smoke/input.md
@@ -28,3 +28,6 @@ Lorem *ipsum dolor sit* amet, _consectetur adipiscing elit_.
Lorem __ipsum dolor__ sit amet, **consectetur adipiscing** elit.
Lorem ipsum __**dolor sit amet, consectetur**__ adipiscing elit.
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Lorem **ipsum** dolor sit amet, consectetur adipiscing elit.

MODIFY
internal/markdown-parser/test-fixtures/smoke/input.test.md
internal/markdown-parser/test-fixtures/smoke/input.test.md
@@ -12,7 +12,7 @@ MarkdownRoot {
corrupt: false
diagnostics: Array []
integrity: undefined
- loc: SourceLocation smoke/input.md 1:0-30:64
+ loc: SourceLocation smoke/input.md 1:0-33:60
path: UIDPath<smoke/input.md>
body: Array [
MarkdownHeadingBlock {
@@ -324,6 +324,36 @@ MarkdownRoot {
}
]
}
+ MarkdownParagraph {
+ loc: SourceLocation smoke/input.md 32:0-33:60
+ children: Array [
+ MarkdownText {
+ value: "Lorem ipsum dolor sit amet, consectetur adipiscing elit."
+ loc: SourceLocation smoke/input.md 32:0-32:0
+ }
+ MarkdownText {
+ value: "\n"
+ loc: SourceLocation smoke/input.md 32:56-33:0
+ }
+ MarkdownText {
+ value: "Lorem "
+ loc: SourceLocation smoke/input.md 33:0-33:0
+ }
+ MarkdownEmphasisInline {
+ value: Array [
+ MarkdownText {
+ value: "ipsum"
+ loc: SourceLocation smoke/input.md 33:8-33:8
+ }
+ ]
+ loc: SourceLocation smoke/input.md 33:6-33:13
+ }
+ MarkdownText {
+ value: " dolor sit amet, consectetur adipiscing elit."
+ loc: SourceLocation smoke/input.md 33:15-33:15
+ }
+ ]
+ }
]
}
```

MODIFY
internal/markdown-parser/utils.ts
internal/markdown-parser/utils.ts
+import {Tokens} from "./types";
+import {TokenValues, isDigit} from "@internal/parser-core";
import {ZeroIndexed} from "@internal/math";
import {MarkdownParser} from "@internal/markdown-parser/index";
-import {isDigit} from "@internal/parser-core";
const THEMATIC_BREAKS = new Set(["***", "---", "___"]);
@@ -106,13 +107,12 @@ export function isntInlineCharacter(char: string) {
return !INLINE_HOT_CHARACTERS.has(char);
}
-export function isBlockToken(parser: MarkdownParser) {
+export function isBlockToken(token: TokenValues<Tokens>) {
return (
- parser.matchToken("NewLine") ||
- parser.matchToken("Code") ||
- parser.matchToken("Break") ||
- parser.matchToken("HeadingLevel") ||
- parser.matchToken("ListItem")
+ token.type === "Code" ||
+ token.type === "Break" ||
+ token.type === "HeadingLevel" ||
+ token.type === "ListItem"
);
}
@@ -165,3 +165,10 @@ export function isBlock(
export function readUntilEndOfParagraph(char: string): boolean {
return char !== "\n";
}
+
+export function isBeginningOfLine(parser: MarkdownParser): boolean {
+ return (
+ parser.getCurrentToken().type === "NewLine" ||
+ parser.getCurrentToken().type === "SOF"
+ );
+}

fix(markdown-parser): handle text with single newline as same paragraph (#1381)