feat(formater) CST based formatting for JSON
Hacky implementation that uses the rs-lint parser to parse
a JSON string to a CST.
MODIFY
Cargo.lock
Cargo.lock
@@ -867,8 +867,7 @@ checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
name = "rome-formatter"
version = "0.0.0"
dependencies = [
- "serde",
- "serde_json",
+ "rslint_parser",
]
[[package]]
@@ -1070,7 +1069,6 @@ version = "1.0.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8"
dependencies = [
- "indexmap",
"itoa",
"ryu",
"serde",

MODIFY
crates/formatter/Cargo.toml
crates/formatter/Cargo.toml
@@ -6,8 +6,7 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
-serde = { version = "1.0", features = ["derive"] }
-serde_json = { version = "1.0", features = ["preserve_order"] }
+rslint_parser = { path = "../rslint_parser" }
[dev-dependencies]

MODIFY
crates/formatter/src/format_json.rs
crates/formatter/src/format_json.rs
use crate::format_token::{GroupToken, LineToken};
-use crate::{format_token::FormatToken, FormatValue, ListToken};
-use serde_json::Value;
-
-impl FormatValue for Value {
- fn format(&self) -> FormatToken {
- match self {
- Value::String(string) => {
- FormatToken::string(format!("\"{}\"", escape_string(string)).as_str())
- }
- Value::Number(number) => {
- let number = number.as_f64().unwrap();
- FormatToken::f64(number)
- }
- Value::Bool(value) => FormatToken::from(value),
- Value::Object(value) => {
- let separator = FormatToken::concat(vec![
- FormatToken::string(","),
- FormatToken::Line(LineToken::soft_or_space()),
- ]);
-
- let properties_list: Vec<FormatToken> = value
- .iter()
- .map(|(key, value)| {
- FormatToken::concat(vec![
- FormatToken::string(format!("\"{}\":", escape_string(key)).as_str()),
+use crate::{format_token::FormatToken, format_tokens};
+use rslint_parser::ast::{
+ ArrayExpr, GroupingExpr, Literal, LiteralProp, ObjectExpr, ObjectProp, UnaryExpr,
+};
+use rslint_parser::{parse_text, AstNode, SyntaxKind, SyntaxNode, SyntaxToken};
+
+fn tokenize_token(token: SyntaxToken) -> FormatToken {
+ match token.kind() {
+ SyntaxKind::NULL_KW => FormatToken::string("null"),
+ SyntaxKind::TRUE_KW => FormatToken::string("true"),
+ SyntaxKind::FALSE_KW => FormatToken::string("false"),
+ SyntaxKind::STRING => FormatToken::string(token.text().as_str()),
+ SyntaxKind::NUMBER => FormatToken::string(token.text().as_str()),
+ SyntaxKind::MINUS => FormatToken::string("-"),
+ _ => panic!("Unsupported JSON token {:?}", token),
+ }
+}
+
+fn tokenize_node(node: SyntaxNode) -> FormatToken {
+ match node.kind() {
+ SyntaxKind::LITERAL => {
+ let literal = Literal::cast(node).unwrap();
+ tokenize_token(literal.token())
+ }
+ SyntaxKind::UNARY_EXPR => {
+ let expr = UnaryExpr::cast(node).unwrap();
+ format_tokens![
+ tokenize_token(expr.op_token().unwrap()),
+ tokenize_node(expr.expr().unwrap().syntax().clone())
+ ]
+ }
+
+ SyntaxKind::LITERAL_PROP => {
+ let prop = LiteralProp::cast(node).unwrap();
+ format_tokens![
+ tokenize_node(prop.key().unwrap().syntax().clone()),
+ ":",
FormatToken::Space,
- value.format(),
- ])
+ tokenize_node(prop.value().unwrap().syntax().clone()),
+ ]
+ }
+
+ SyntaxKind::OBJECT_EXPR => {
+ let object = ObjectExpr::cast(node).unwrap();
+
+ let separator = format_tokens![",", LineToken::soft_or_space(),];
+
+ let properties_list: Vec<FormatToken> = object
+ .props()
+ .map(|prop| match prop {
+ ObjectProp::LiteralProp(prop) => {
+ format_tokens!(tokenize_node(prop.syntax().clone()))
+ }
+ _ => panic!("Unsupported prop type {:?}", prop),
})
.collect();
- let properties = vec![
- FormatToken::Line(LineToken::soft()),
+ let properties = format_tokens![
+ LineToken::soft(),
FormatToken::join(separator, properties_list),
];
- FormatToken::Group(GroupToken::new(vec![
- FormatToken::string("{"),
+ FormatToken::Group(GroupToken::new(format_tokens![
+ "{",
FormatToken::indent(properties),
- FormatToken::Line(LineToken::soft()),
- FormatToken::string("}"),
+ LineToken::soft(),
+ "}",
]))
}
- Value::Null => FormatToken::string("null"),
- Value::Array(array) => {
- let separator = FormatToken::concat(vec![
- FormatToken::string(","),
- FormatToken::Line(LineToken::soft_or_space()),
- ]);
+ SyntaxKind::ARRAY_EXPR => {
+ let array = ArrayExpr::cast(node).unwrap();
+
+ let separator = format_tokens![",", LineToken::soft_or_space(),];
- let elements = vec![
- FormatToken::Line(LineToken::soft()),
- FormatToken::join(separator, array.iter().map(|element| element.format())),
+ let elements = format_tokens![
+ LineToken::soft(),
+ FormatToken::join(
+ separator,
+ array
+ .elements()
+ .map(|element| tokenize_node(element.syntax().clone())),
+ ),
];
- FormatToken::Group(GroupToken::new(vec![
- FormatToken::string("["),
+ FormatToken::Group(GroupToken::new(format_tokens![
+ "[",
FormatToken::indent(elements),
- FormatToken::Line(LineToken::soft()),
- FormatToken::string("]"),
+ LineToken::soft(),
+ "]",
]))
}
- }
+ _ => panic!("Unsupported JSON kind: {:?}", node.kind()),
}
}
-fn escape_string(string: &str) -> String {
- string
- .replace("\\", "\\\\")
- .replace('"', "\\\"")
- .replace('\r', "\\r")
- .replace('\t', "\\t")
- .replace('\n', "\\n")
-}
+pub fn tokenize_json(content: &str) -> FormatToken {
+ let script = parse_text(format!("({})", content).as_str(), 0);
-pub fn json_to_tokens(content: &str) -> FormatToken {
- let json: Value = serde_json::from_str(content).expect("cannot convert json to tokens");
+ // Unwrap the grouping to get to the JSON content. The grouping is only used as a trick to parse JSON
+ let json_content = GroupingExpr::cast(
+ script
+ .syntax()
+ .descendants()
+ .find(|e| e.kind() == SyntaxKind::GROUPING_EXPR)
+ .unwrap(),
+ )
+ .and_then(|grouping| grouping.inner())
+ .unwrap();
- FormatToken::from(ListToken::concat(vec![
- json.format(),
- FormatToken::from(LineToken::hard()),
- ]))
+ let tokenized_content = tokenize_node(json_content.syntax().clone());
+ format_tokens!(tokenized_content, LineToken::hard())
}
#[cfg(test)]
mod test {
- use crate::{FormatToken, ListToken};
+ use crate::format_tokens;
- use super::json_to_tokens;
+ use super::tokenize_json;
use crate::format_token::{GroupToken, LineToken};
#[test]
fn tokenize_number() {
- let result = json_to_tokens("6.45");
+ let result = tokenize_json("6.45");
- assert_eq!(
- FormatToken::List(ListToken::concat(vec![
- FormatToken::string("6.45"),
- FormatToken::Line(LineToken::hard())
- ])),
- result
- );
+ assert_eq!(format_tokens!["6.45", LineToken::hard()], result);
}
#[test]
fn tokenize_string() {
- let result = json_to_tokens(r#""foo""#);
+ let result = tokenize_json(r#""foo""#);
- assert_eq!(
- FormatToken::List(ListToken::concat(vec![
- FormatToken::string(r#""foo""#),
- FormatToken::Line(LineToken::hard())
- ])),
- result
- );
+ assert_eq!(format_tokens![r#""foo""#, LineToken::hard()], result);
}
#[test]
fn tokenize_boolean_false() {
- let result = json_to_tokens("false");
+ let result = tokenize_json("false");
- assert_eq!(
- FormatToken::List(ListToken::concat(vec![
- FormatToken::string("false"),
- FormatToken::Line(LineToken::hard())
- ])),
- result
- );
+ assert_eq!(format_tokens!["false", LineToken::hard()], result);
}
#[test]
fn tokenize_boolean_true() {
- let result = json_to_tokens("true");
+ let result = tokenize_json("true");
- assert_eq!(
- FormatToken::List(ListToken::concat(vec![
- FormatToken::string("true"),
- FormatToken::Line(LineToken::hard())
- ])),
- result
- );
+ assert_eq!(format_tokens!["true", LineToken::hard()], result);
}
#[test]
fn tokenize_boolean_null() {
- let result = json_to_tokens("null");
+ let result = tokenize_json("null");
- assert_eq!(
- FormatToken::List(ListToken::concat(vec![
- FormatToken::string("null"),
- FormatToken::Line(LineToken::hard())
- ])),
- result
- );
+ assert_eq!(format_tokens!["null", LineToken::hard()], result);
}
#[test]
fn tokenize_object() {
let input = r#"{ "foo": "bar", "num": 5 }"#;
- let expected = FormatToken::List(ListToken::concat(vec![
- FormatToken::Group(GroupToken::new(vec![
- FormatToken::string("{"),
- FormatToken::indent(FormatToken::concat(vec![
- FormatToken::Line(LineToken::soft()),
- FormatToken::string("\"foo\":"),
+ let expected = format_tokens![
+ GroupToken::new(format_tokens![
+ "{",
+ FormatToken::indent(format_tokens![
+ LineToken::soft(),
+ "\"foo\"",
+ ":",
FormatToken::Space,
- FormatToken::string("\"bar\""),
- FormatToken::string(","),
- FormatToken::Line(LineToken::soft_or_space()),
- FormatToken::string("\"num\":"),
+ "\"bar\"",
+ ",",
+ LineToken::soft_or_space(),
+ "\"num\"",
+ ":",
FormatToken::Space,
- FormatToken::string("5"),
- ])),
- FormatToken::Line(LineToken::soft()),
- FormatToken::string("}"),
- ])),
- FormatToken::Line(LineToken::hard()),
- ]));
+ "5",
+ ]),
+ LineToken::soft(),
+ "}",
+ ]),
+ LineToken::hard(),
+ ];
- let result = json_to_tokens(input);
+ let result = tokenize_json(input);
assert_eq!(expected, result);
}
@@ -186,26 +187,26 @@ mod test {
#[test]
fn tokenize_array() {
let input = r#"[ "foo", "bar", 5 ]"#;
- let expected = FormatToken::List(ListToken::concat(vec![
- FormatToken::Group(GroupToken::new(vec![
- FormatToken::string("["),
- FormatToken::indent(FormatToken::concat(vec![
- FormatToken::Line(LineToken::soft()),
- FormatToken::string("\"foo\""),
- FormatToken::string(","),
- FormatToken::Line(LineToken::soft_or_space()),
- FormatToken::string("\"bar\""),
- FormatToken::string(","),
- FormatToken::Line(LineToken::soft_or_space()),
- FormatToken::string("5"),
- ])),
- FormatToken::Line(LineToken::soft()),
- FormatToken::string("]"),
- ])),
- FormatToken::Line(LineToken::hard()),
- ]));
-
- let result = json_to_tokens(input);
+ let expected = format_tokens![
+ GroupToken::new(format_tokens![
+ "[",
+ FormatToken::indent(format_tokens![
+ LineToken::soft(),
+ "\"foo\"",
+ ",",
+ LineToken::soft_or_space(),
+ "\"bar\"",
+ ",",
+ LineToken::soft_or_space(),
+ "5",
+ ]),
+ LineToken::soft(),
+ "]",
+ ]),
+ LineToken::hard(),
+ ];
+
+ let result = tokenize_json(input);
assert_eq!(expected, result);
}

MODIFY
crates/formatter/src/format_tokens_macro.rs
crates/formatter/src/format_tokens_macro.rs
@@ -52,7 +52,7 @@ macro_rules! format_tokens {
};
( $( $token:expr ),+ $(,)?) => {{
- use $crate::{FormatToken, ListToken};
+ use $crate::{FormatToken};
FormatToken::concat(vec![
$(
FormatToken::from($token)

MODIFY
crates/formatter/src/lib.rs
crates/formatter/src/lib.rs
@@ -48,7 +48,7 @@ mod format_tokens_macro;
mod intersperse;
mod printer;
-use crate::format_json::json_to_tokens;
+use crate::format_json::tokenize_json;
use std::{fs::File, io::Read, path::PathBuf, str::FromStr};
pub use format_token::{
@@ -115,14 +115,14 @@ pub fn format(path: PathBuf, options: FormatOptions) {
file.read_to_string(&mut buffer)
.expect("cannot read the file to format");
- let tokens = json_to_tokens(buffer.as_str());
+ let tokens = tokenize_json(buffer.as_str());
let print_result = format_token(&tokens, options);
println!("{}", print_result.code());
}
pub fn format_str(content: &str, options: FormatOptions) -> PrintResult {
- let tokens = json_to_tokens(content);
+ let tokens = tokenize_json(content);
format_token(&tokens, options)
}

MODIFY
crates/formatter/src/printer.rs
crates/formatter/src/printer.rs
@@ -236,7 +236,7 @@ impl Printer {
let calls = self.print_token(token, args);
// If the line is too long, break the group
- if self.state.line_width > self.options.print_width {
+ if self.state.line_width > self.options.print_width as usize {
return Err(LineBreakRequiredError);
}
@@ -299,7 +299,7 @@ impl Printer {
self.state.generated_column += 1;
let char_width = if char == '\t' {
- self.options.tab_width as u16
+ self.options.tab_width as usize
} else {
1
};
@@ -319,9 +319,9 @@ struct PrinterState {
pending_indent: u16,
pending_spaces: u16,
generated_index: usize,
- generated_line: u16,
- generated_column: u16,
- line_width: u16,
+ generated_line: usize,
+ generated_column: usize,
+ line_width: usize,
// mappings: Mapping[];
// We'll need to clone the line suffixes tokens into the state.
// I guess that's fine. They're only used for comments and should, therefore, be very limited
@@ -360,9 +360,9 @@ struct PrinterStateSnapshot {
pending_indents: u16,
pending_spaces: u16,
generated_index: usize,
- generated_column: u16,
- generated_line: u16,
- line_width: u16,
+ generated_column: usize,
+ generated_line: usize,
+ line_width: usize,
buffer_position: usize,
}

MODIFY
crates/formatter/tests/specs/json/int1.expected.json
crates/formatter/tests/specs/json/int1.expected.json
null,
{
"integer": 1234567890,
- "real": -9876.54321,
+ "real": -9876.543210,
"": 23456789012,
"zero": 0,
"one": 1,
"quote": "\"",
"backslash": "\\",
"controls": "\n\r\t",
- "slash": "/ & /",
+ "slash": "/ & \/",
"alpha": "abcdefghijklmnopqrstuvwyz",
"ALPHA": "ABCDEFGHIJKLMNOPQRSTUVWYZ",
"digit": "0123456789",
" s p a c e d ": [1, 2, 3, 4, 5, 6, 7],
"compact": [1, 2, 3, 4, 5, 6, 7],
"jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}",
- "/\\\"\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?": "A key can be any string"
+ "\/\\\"\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?": "A key can be any string"
},
0.5,
98.6,

feat(formater) CST based formatting for JSON

Hacky implementation that uses the rs-lint parser to parse
a JSON string to a CST.