feat(rome_formatter): track verbatim code inside snapshots
MODIFY
crates/rome_formatter/src/format_element.rs
crates/rome_formatter/src/format_element.rs
@@ -875,6 +875,30 @@ pub enum FormatElement {
Token(Token),
LineSuffix(Content),
+
+ /// A token that tracks tokens/nodes that are printed using [`format_verbatim`](Formatter::format_verbatim) API
+ Verbatim(Verbatim),
+}
+
+/// Information of the node/token formatted verbatim
+#[derive(Clone, Eq, PartialEq)]
+pub struct Verbatim {
+ /// the range that belongs to the node/token formatted verbatim
+ pub range: TextRange,
+ /// the text that belongs to the node/token formatted verbatim
+ pub text: String,
+ /// The [FormatElement] version of the node/token
+ pub element: Box<FormatElement>,
+}
+
+impl Verbatim {
+ pub fn new(element: FormatElement, text: String, range: TextRange) -> Self {
+ Self {
+ element: Box::new(element),
+ range,
+ text,
+ }
+ }
}
impl Debug for FormatElement {
@@ -905,6 +929,10 @@ impl Debug for FormatElement {
FormatElement::LineSuffix(content) => {
fmt.debug_tuple("LineSuffix").field(content).finish()
}
+ FormatElement::Verbatim(verbatim) => fmt
+ .debug_tuple("Verbatim")
+ .field(&verbatim.element)
+ .finish(),
}
}
}
@@ -1260,6 +1288,11 @@ impl FormatElement {
FormatElement::List(list) => FormatElement::List(list.trim_start()),
FormatElement::Token(s) => FormatElement::Token(s.trim_start()),
FormatElement::LineSuffix(s) => FormatElement::LineSuffix(Box::new(s.trim_start())),
+ FormatElement::Verbatim(v) => FormatElement::Verbatim(Verbatim::new(
+ v.element.trim_start(),
+ v.text.clone(),
+ v.range,
+ )),
}
}
@@ -1279,6 +1312,11 @@ impl FormatElement {
FormatElement::List(list) => FormatElement::List(list.trim_end()),
FormatElement::Token(s) => FormatElement::Token(s.trim_end()),
FormatElement::LineSuffix(s) => FormatElement::LineSuffix(Box::new(s.trim_end())),
+ FormatElement::Verbatim(v) => FormatElement::Verbatim(Verbatim::new(
+ v.element.trim_end(),
+ v.text.clone(),
+ v.range,
+ )),
}
}
}

MODIFY
crates/rome_formatter/src/formatter.rs
crates/rome_formatter/src/formatter.rs
@@ -3,6 +3,7 @@ use std::cell::RefCell;
#[cfg(debug_assertions)]
use std::collections::HashSet;
+use crate::format_element::Verbatim;
use crate::formatter_traits::FormatTokenAndNode;
use crate::{
block_indent, concat_elements, empty_element, empty_line,
@@ -323,7 +324,9 @@ impl Formatter {
Ok(result) => result,
Err(_) => {
self.restore(snapshot);
- self.format_verbatim(module_item.syntax())
+ // Lists that yield errors are formatted as they were unknown nodes.
+ // Doing so, the formatter formats the nodes/tokens as is.
+ self.format_unknown(module_item.syntax())
.trim_start()
.trim_end()
}
@@ -411,12 +414,27 @@ impl Formatter {
///
/// You may be inclined to call `node.text` directly. However, using `text` doesn't track the nodes
///nor its children source mapping information, resulting in incorrect source maps for this subtree.
+ ///
+ /// These nodes and tokens get tracked as [FormatElement::Verbatim], useful to understand
+ /// if these nodes still need to have their own implementation.
pub fn format_verbatim(&self, node: &SyntaxNode) -> FormatElement {
- concat_elements(node.children_with_tokens().map(|child| match child {
+ let verbatim = self.format_verbatim_node_or_token(node);
+ FormatElement::Verbatim(Verbatim::new(verbatim, node.to_string(), node.text_range()))
+ }
+
+ /// Formats unknown nodes. The difference between this method and `format_verbatim` is that this method
+ /// doesn't track nodes/tokens as [FormatElement::Verbatim]. They are just printed as they are.
+ pub fn format_unknown(&self, node: &SyntaxNode) -> FormatElement {
+ self.format_verbatim_node_or_token(node)
+ }
+
+ fn format_verbatim_node_or_token(&self, node: &SyntaxNode) -> FormatElement {
+ concat_elements(node.children_with_tokens().map(|child| {
+ match child {
SyntaxElement::Node(child_node) => {
- // TODO: Add source map markers before/after node as well as any additional elements that
- // need to be tracked for every node.
- self.format_verbatim(&child_node)
+ // Here we call `format_unknown` because we don't want to track it as [FormatElement::Verbatim]
+ // all the possible children. The first node to call `format_verbatim` should be node to be tracked
+ self.format_verbatim_node_or_token(&child_node)
}
SyntaxElement::Token(syntax_token) => {
cfg_if::cfg_if! {
@@ -431,6 +449,7 @@ impl Formatter {
syntax_token.text_range(),
))
}
+ }
}))
}
}

MODIFY
crates/rome_formatter/src/js/unknown/unknown.rs
crates/rome_formatter/src/js/unknown/unknown.rs
@@ -2,6 +2,6 @@ use crate::{FormatElement, FormatResult, Formatter, ToFormatElement};
use rslint_parser::{ast::JsUnknown, AstNode};
impl ToFormatElement for JsUnknown {
fn to_format_element(&self, formatter: &Formatter) -> FormatResult<FormatElement> {
- Ok(formatter.format_verbatim(self.syntax()))
+ Ok(formatter.format_unknown(self.syntax()))
}
}

MODIFY
crates/rome_formatter/src/js/unknown/unknown_assignment.rs
crates/rome_formatter/src/js/unknown/unknown_assignment.rs
@@ -6,6 +6,6 @@ use rslint_parser::AstNode;
impl ToFormatElement for JsUnknownAssignment {
fn to_format_element(&self, formatter: &Formatter) -> FormatResult<FormatElement> {
- Ok(formatter.format_verbatim(self.syntax()))
+ Ok(formatter.format_unknown(self.syntax()))
}
}

MODIFY
crates/rome_formatter/src/js/unknown/unknown_binding.rs
crates/rome_formatter/src/js/unknown/unknown_binding.rs
@@ -6,6 +6,6 @@ use rslint_parser::AstNode;
impl ToFormatElement for JsUnknownBinding {
fn to_format_element(&self, formatter: &Formatter) -> FormatResult<FormatElement> {
- Ok(formatter.format_verbatim(self.syntax()))
+ Ok(formatter.format_unknown(self.syntax()))
}
}

MODIFY
crates/rome_formatter/src/js/unknown/unknown_expression.rs
crates/rome_formatter/src/js/unknown/unknown_expression.rs
@@ -6,6 +6,6 @@ use rslint_parser::AstNode;
impl ToFormatElement for JsUnknownExpression {
fn to_format_element(&self, formatter: &Formatter) -> FormatResult<FormatElement> {
- Ok(formatter.format_verbatim(self.syntax()))
+ Ok(formatter.format_unknown(self.syntax()))
}
}

MODIFY
crates/rome_formatter/src/js/unknown/unknown_import_assertion_entry.rs
crates/rome_formatter/src/js/unknown/unknown_import_assertion_entry.rs
@@ -6,6 +6,6 @@ use rslint_parser::AstNode;
impl ToFormatElement for JsUnknownImportAssertionEntry {
fn to_format_element(&self, formatter: &Formatter) -> FormatResult<FormatElement> {
- Ok(formatter.format_verbatim(self.syntax()))
+ Ok(formatter.format_unknown(self.syntax()))
}
}

MODIFY
crates/rome_formatter/src/js/unknown/unknown_member.rs
crates/rome_formatter/src/js/unknown/unknown_member.rs
@@ -6,6 +6,6 @@ use rslint_parser::AstNode;
impl ToFormatElement for JsUnknownMember {
fn to_format_element(&self, formatter: &Formatter) -> FormatResult<FormatElement> {
- Ok(formatter.format_verbatim(self.syntax()))
+ Ok(formatter.format_unknown(self.syntax()))
}
}

MODIFY
crates/rome_formatter/src/js/unknown/unknown_named_import_specifier.rs
crates/rome_formatter/src/js/unknown/unknown_named_import_specifier.rs
@@ -6,6 +6,6 @@ use rslint_parser::AstNode;
impl ToFormatElement for JsUnknownNamedImportSpecifier {
fn to_format_element(&self, formatter: &Formatter) -> FormatResult<FormatElement> {
- Ok(formatter.format_verbatim(self.syntax()))
+ Ok(formatter.format_unknown(self.syntax()))
}
}

MODIFY
crates/rome_formatter/src/js/unknown/unknown_parameter.rs
crates/rome_formatter/src/js/unknown/unknown_parameter.rs
@@ -6,6 +6,6 @@ use rslint_parser::AstNode;
impl ToFormatElement for JsUnknownParameter {
fn to_format_element(&self, formatter: &Formatter) -> FormatResult<FormatElement> {
- Ok(formatter.format_verbatim(self.syntax()))
+ Ok(formatter.format_unknown(self.syntax()))
}
}

MODIFY
crates/rome_formatter/src/js/unknown/unknown_statement.rs
crates/rome_formatter/src/js/unknown/unknown_statement.rs
@@ -6,6 +6,6 @@ use rslint_parser::AstNode;
impl ToFormatElement for JsUnknownStatement {
fn to_format_element(&self, formatter: &Formatter) -> FormatResult<FormatElement> {
- Ok(formatter.format_verbatim(self.syntax()))
+ Ok(formatter.format_unknown(self.syntax()))
}
}

MODIFY
crates/rome_formatter/src/lib.rs
crates/rome_formatter/src/lib.rs
@@ -203,14 +203,21 @@ pub struct Formatted {
code: String,
range: Option<TextRange>,
sourcemap: Vec<SourceMarker>,
+ verbatim_source: Vec<(String, TextRange)>,
}
impl Formatted {
- fn new(code: String, range: Option<TextRange>, sourcemap: Vec<SourceMarker>) -> Self {
+ fn new(
+ code: String,
+ range: Option<TextRange>,
+ sourcemap: Vec<SourceMarker>,
+ verbatim_source: Vec<(String, TextRange)>,
+ ) -> Self {
Self {
code,
range,
sourcemap,
+ verbatim_source,
}
}
@@ -220,6 +227,7 @@ impl Formatted {
code: String::new(),
range: None,
sourcemap: Vec::new(),
+ verbatim_source: Vec::new(),
}
}
@@ -244,6 +252,10 @@ impl Formatted {
pub fn into_code(self) -> String {
self.code
}
+
+ pub fn verbatim(&self) -> &[(String, TextRange)] {
+ &self.verbatim_source
+ }
}
/// Formats a JavaScript (and its super languages) file based on its features.
@@ -394,6 +406,7 @@ pub fn format_range(
code.into(),
Some(input_range),
formatted.sourcemap,
+ formatted.verbatim_source,
))
}
@@ -463,6 +476,7 @@ pub fn format_node(options: FormatOptions, root: &SyntaxNode) -> FormatResult<Fo
formatted.code,
Some(root.text_range()),
formatted.sourcemap,
+ formatted.verbatim_source,
))
}

MODIFY
crates/rome_formatter/src/printer.rs
crates/rome_formatter/src/printer.rs
-use std::iter::once;
-
-use rome_rowan::TextSize;
-
use crate::format_element::{ConditionalGroupContent, Group, GroupPrintMode, LineMode, List};
use crate::intersperse::Intersperse;
use crate::{
hard_line_break, space_token, FormatElement, FormatOptions, Formatted, IndentStyle,
- SourceMarker,
+ SourceMarker, TextRange,
};
+use rome_rowan::TextSize;
+use std::iter::once;
/// Options that affect how the [Printer] prints the format tokens
#[derive(Clone, Debug, Eq, PartialEq)]
@@ -136,7 +134,12 @@ impl<'a> Printer<'a> {
}
}
- Formatted::new(self.state.buffer, None, self.state.source_markers)
+ Formatted::new(
+ self.state.buffer,
+ None,
+ self.state.source_markers,
+ self.state.verbatim_markers,
+ )
}
/// Prints a single element and push the following elements to queue
@@ -245,6 +248,12 @@ impl<'a> Printer<'a> {
.line_suffixes
.push(PrintElementCall::new(&**suffix, args));
}
+ FormatElement::Verbatim(verbatim) => {
+ self.state
+ .verbatim_markers
+ .push((verbatim.text.clone(), verbatim.range));
+ queue.enqueue(PrintElementCall::new(&verbatim.element, args));
+ }
}
}
@@ -353,6 +362,7 @@ impl<'a> Printer<'a> {
FormatElement::Empty
| FormatElement::Space
| FormatElement::Indent { .. }
+ | FormatElement::Verbatim { .. }
| FormatElement::List { .. } => self.print_element(queue, element, args),
}
@@ -461,6 +471,7 @@ struct PrinterState<'a> {
line_width: usize,
// mappings: Mapping[];
line_suffixes: Vec<PrintElementCall<'a>>,
+ verbatim_markers: Vec<(String, TextRange)>,
}
impl<'a> PrinterState<'a> {
@@ -474,6 +485,7 @@ impl<'a> PrinterState<'a> {
line_width: self.line_width,
buffer_position: self.buffer.len(),
tokens_position: self.source_markers.len(),
+ verbatim_markers: self.verbatim_markers.len(),
}
}
@@ -486,6 +498,7 @@ impl<'a> PrinterState<'a> {
self.line_width = snapshot.line_width;
self.buffer.truncate(snapshot.buffer_position);
self.source_markers.truncate(snapshot.tokens_position);
+ self.verbatim_markers.truncate(snapshot.verbatim_markers);
}
}
@@ -498,6 +511,7 @@ struct PrinterStateSnapshot {
line_width: usize,
buffer_position: usize,
tokens_position: usize,
+ verbatim_markers: usize,
}
/// Stores arguments passed to `print_element` call, holding the state specific to printing an element.

MODIFY
crates/rome_formatter/tests/spec_test.rs
crates/rome_formatter/tests/spec_test.rs
use rome_core::App;
-use rome_formatter::{format, FormatOptions, IndentStyle};
+use rome_formatter::{format, FormatOptions, Formatted, IndentStyle};
use rome_path::RomePath;
use rslint_parser::{parse, Syntax};
use serde::{Deserialize, Serialize};
@@ -56,8 +56,18 @@ struct SnapshotContent {
}
impl SnapshotContent {
- pub fn add_output(&mut self, content: impl Into<String>, options: FormatOptions) {
- self.output.push((content.into(), options))
+ pub fn add_output(&mut self, formatted: Formatted, options: FormatOptions) {
+ let mut output: String = formatted.as_code().into();
+ if !formatted.verbatim().is_empty() {
+ output.push_str("\n\n");
+ output.push_str("## Unimplemented nodes/tokens");
+ output.push_str("\n\n");
+ for (text, range) in formatted.verbatim() {
+ let string = format!("{:?} => {:?}\n", text, range);
+ output.push_str(string.as_str());
+ }
+ }
+ self.output.push((output, options));
}
pub fn set_input(&mut self, content: impl Into<String>) {
@@ -137,7 +147,7 @@ pub fn run(spec_input_file: &str, _expected_file: &str, test_directory: &str, fi
// we ignore the error for now
let result = formatted_result.unwrap();
- snapshot_content.add_output(result.as_code(), FormatOptions::default());
+ snapshot_content.add_output(result, FormatOptions::default());
let test_directory = PathBuf::from(test_directory);
let options_path = test_directory.join("options.json");
@@ -151,7 +161,7 @@ pub fn run(spec_input_file: &str, _expected_file: &str, test_directory: &str, fi
for test_case in options.cases {
let format_options: FormatOptions = test_case.into();
let formatted_result = format(format_options, &root).unwrap();
- snapshot_content.add_output(formatted_result.as_code(), format_options);
+ snapshot_content.add_output(formatted_result, format_options);
}
}
}

MODIFY
crates/rome_formatter/tests/specs/ts/expression/type_expression.ts
crates/rome_formatter/tests/specs/ts/expression/type_expression.ts
@@ -37,7 +37,8 @@ type OptionsFlags
in
keyof
Type
- as string]?: boolean;
+ as string]// something
+ ?: boolean;
};
type OptionsFlag

MODIFY
crates/rome_formatter/tests/specs/ts/expression/type_expression.ts.snap
crates/rome_formatter/tests/specs/ts/expression/type_expression.ts.snap
---
source: crates/rome_formatter/tests/spec_test.rs
-assertion_line: 163
+assertion_line: 173
expression: type_expression.ts
---
@@ -44,7 +44,8 @@ type OptionsFlags
in
keyof
Type
- as string]?: boolean;
+ as string]// something
+ ?: boolean;
};
type OptionsFlag
@@ -117,7 +118,8 @@ type X = { a: string, b: symbol };
type Z = { a: string, b: symbol };
type OptionsFlags<Type> = {
- [Property in keyof Type as string]?: boolean;
+ [Property in keyof Type as string]
+ ?: boolean; // something
};
type OptionsFlag<A extends OptionsFlags<any> = any> = string;
@@ -138,3 +140,8 @@ type TupleD = [
...name: [string],
];
+
+## Unimplemented nodes/tokens
+
+"\n ?" => 698..708
+

feat(rome_formatter): track verbatim code inside snapshots (#2147)