feat(formatter): Support different line endings
Adds support for different line endings to the formatter's printer.
Line feeds will be used internally that then get converted into whatever the users prefer when `Printing` the tokens.
MODIFY
Cargo.lock
Cargo.lock
@@ -214,6 +214,7 @@ version = "1.0.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8"
dependencies = [
+ "indexmap",
"itoa",
"ryu",
"serde",

MODIFY
crates/formatter/Cargo.toml
crates/formatter/Cargo.toml
@@ -7,7 +7,7 @@ edition = "2018"
[dependencies]
serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
+serde_json = { version = "1.0", features = ["preserve_order"] }
[dev-dependencies]

MODIFY
crates/formatter/src/format_token.rs
crates/formatter/src/format_token.rs
use crate::intersperse::Intersperse;
+use std::ops::Deref;
type Content = Box<FormatToken>;
pub type Tokens = Vec<FormatToken>;
@@ -17,8 +18,7 @@ pub enum FormatToken {
List(ListToken),
// TODO Revisit, structure is a bit weird
IfBreak(IfBreakToken),
- /// A string that will be printed as is.
- String(String),
+ String(StringToken),
}
#[derive(Debug, Clone, Eq, PartialEq)]
@@ -64,7 +64,7 @@ impl IndentToken {
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct ListToken {
- pub content: Vec<FormatToken>,
+ content: Vec<FormatToken>,
}
impl ListToken {
@@ -98,6 +98,14 @@ impl<T: Into<Vec<FormatToken>>> From<T> for ListToken {
}
}
+impl Deref for ListToken {
+ type Target = Vec<FormatToken>;
+
+ fn deref(&self) -> &Self::Target {
+ &self.content
+ }
+}
+
#[derive(Debug, Clone, Eq, PartialEq)]
pub struct IfBreakToken {
pub break_contents: Content,
@@ -153,6 +161,48 @@ impl GroupToken {
}
}
+/// A code fragment that gets written to the output. Newlines must be encoded with line feeds `\n`.
+/// The [Printer] takes care of converting the line feeds to the line ending specified in the options.
+///
+/// # Example
+/// ```
+/// use rome_formatter::{FormatToken, format_token, FormatOptions};
+/// let token = FormatToken::string(r#"["a", "b"]"#);
+/// let formatted = format_token(&token, FormatOptions::default());
+///
+/// assert_eq!(r#"["a", "b"]"#, formatted.code());
+/// ```
+/// Is not allowed to contain any new lines
+#[derive(Debug, PartialEq, Eq, Clone)]
+pub struct StringToken(String);
+
+impl StringToken {
+ pub fn new(content: &str) -> Self {
+ debug_assert!(!content.contains('\r'), "The content '{}' contains a carriage return '\\r' character but string tokens must only use line feeds '\\n' as line separator. Use '\\n' instead of '\\r' and '\\r\\n' to insert a line break in strings.", content);
+ Self(String::from(content))
+ }
+}
+
+impl From<&str> for StringToken {
+ fn from(content: &str) -> Self {
+ StringToken::new(content)
+ }
+}
+
+impl From<String> for StringToken {
+ fn from(content: String) -> Self {
+ StringToken::new(content.as_str())
+ }
+}
+
+impl Deref for StringToken {
+ type Target = String;
+
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
+}
+
impl<'a> FormatToken {
/// Stores lint a list a `Vec` of `FormatToken`
pub fn concat<T: Into<Tokens>>(tokens: T) -> FormatToken {
@@ -178,52 +228,52 @@ impl<'a> FormatToken {
/// Utility to tokenize a string
pub fn string<T: Into<&'a str>>(content: T) -> FormatToken {
- FormatToken::String(String::from(content.into()))
+ FormatToken::String(StringToken::new(content.into()))
}
/// Utility to tokenize a f64
pub fn f64<T: Into<f64>>(content: T) -> FormatToken {
- FormatToken::from(content.into())
+ FormatToken::string(content.into().to_string().as_str())
}
/// Utility to tokenize a u64
pub fn u64<T: Into<u64>>(content: T) -> FormatToken {
- FormatToken::from(content.into())
+ FormatToken::string(content.into().to_string().as_str())
}
/// Utility to tokenize a boolean
pub fn boolean<T: Into<bool>>(content: T) -> FormatToken {
- FormatToken::from(content.into())
+ FormatToken::string(content.into().to_string().as_str())
}
}
impl From<&str> for FormatToken {
fn from(value: &str) -> Self {
- FormatToken::String(String::from(value))
+ FormatToken::string(value)
}
}
impl From<u64> for FormatToken {
fn from(value: u64) -> Self {
- FormatToken::String(value.to_string())
+ FormatToken::u64(value)
}
}
impl From<f64> for FormatToken {
fn from(value: f64) -> Self {
- FormatToken::String(value.to_string())
+ FormatToken::f64(value)
}
}
impl From<&bool> for FormatToken {
fn from(value: &bool) -> Self {
- FormatToken::String(value.to_string())
+ FormatToken::boolean(*value)
}
}
impl From<bool> for FormatToken {
fn from(value: bool) -> Self {
- FormatToken::String(value.to_string())
+ FormatToken::boolean(value)
}
}

MODIFY
crates/formatter/src/lib.rs
crates/formatter/src/lib.rs
@@ -6,8 +6,9 @@ mod printer;
use crate::format_json::json_to_tokens;
use std::{fs::File, io::Read, path::PathBuf, str::FromStr};
-use crate::printer::Printer;
-pub use format_token::{FormatToken, LineMode};
+pub use format_token::{FormatToken, GroupToken, IfBreakToken, IndentToken, LineMode, ListToken};
+pub use printer::PrintResult;
+use printer::Printer;
/// This trait should implemented on each node/value that should have a formatted representation
pub trait FormatValue {
@@ -22,6 +23,12 @@ pub enum IndentStyle {
Space(u8),
}
+impl Default for IndentStyle {
+ fn default() -> Self {
+ IndentStyle::Tab
+ }
+}
+
impl FromStr for IndentStyle {
type Err = &'static str;
@@ -35,8 +42,7 @@ impl FromStr for IndentStyle {
}
}
-// TODO: evaluate to remove it
-#[derive(Debug)]
+#[derive(Debug, Default)]
pub struct FormatOptions {
/// The indent style
indent_style: IndentStyle,
@@ -61,9 +67,12 @@ pub fn format(path: PathBuf, options: FormatOptions) {
file.read_to_string(&mut buffer).unwrap();
let tokens = json_to_tokens(buffer.as_str());
-
- let printer = Printer::new(options);
- let print_result = printer.print(&tokens);
+ let print_result = format_token(&tokens, options);
println!("{}", print_result.code());
}
+
+pub fn format_token(token: &FormatToken, options: FormatOptions) -> PrintResult {
+ let printer = Printer::new(options);
+ printer.print(token)
+}

MODIFY
crates/formatter/src/printer.rs
crates/formatter/src/printer.rs
use crate::format_token::IfBreakToken;
-use crate::LineMode;
use crate::{FormatOptions, FormatToken, IndentStyle};
+use crate::{GroupToken, LineMode};
/// Options that affect how the [Printer] prints the format tokens
#[derive(Clone, Debug, Eq, PartialEq)]
@@ -11,6 +11,8 @@ pub struct PrinterOptions {
/// What's the max width of a line. Defaults to 80
pub print_width: u16,
+ pub line_ending: LineEnding,
+
/// The never ending question whatever to use spaces or tabs, and if spaces, how many spaces
/// to indent code.
///
@@ -37,12 +39,37 @@ impl From<FormatOptions> for PrinterOptions {
}
}
+#[allow(dead_code)]
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub enum LineEnding {
+ /// Line Feed only (\n), common on Linux and macOS as well as inside git repos
+ LineFeed,
+
+ /// Carriage Return + Line Feed characters (\r\n), common on Windows
+ CarriageReturnLineFeed,
+
+ /// Carriage Return character only (\r), used very rarely
+ CarriageReturn,
+}
+
+impl LineEnding {
+ #[inline]
+ pub const fn as_str(&self) -> &'static str {
+ match self {
+ LineEnding::LineFeed => "\n",
+ LineEnding::CarriageReturnLineFeed => "\r\n",
+ LineEnding::CarriageReturn => "\r",
+ }
+ }
+}
+
impl Default for PrinterOptions {
fn default() -> Self {
PrinterOptions {
tab_width: 2,
print_width: 80,
indent_string: String::from("\t"),
+ line_ending: LineEnding::LineFeed,
}
}
}
@@ -127,11 +154,14 @@ impl Printer {
vec![]
}
- FormatToken::Group(group) if !group.should_break => {
+ FormatToken::Group(GroupToken {
+ should_break: false,
+ content,
+ }) => {
match self.try_print_flat(token, args.clone()) {
Err(_) => {
// Flat printing didn't work, print with line breaks
- vec![PrintTokenCall::new(group.content.as_ref(), args)]
+ vec![PrintTokenCall::new(content.as_ref(), args)]
}
Ok(_) => vec![],
}
@@ -142,7 +172,6 @@ impl Printer {
}
FormatToken::List(list) => list
- .content
.iter()
.map(|t| PrintTokenCall::new(t, args.clone()))
.collect(),
@@ -228,9 +257,9 @@ impl Printer {
LineMode::Hard => return Err(LineBreakRequiredError {}),
}
}
- FormatToken::Group(group) if group.should_break => {
- return Err(LineBreakRequiredError {})
- }
+ FormatToken::Group(GroupToken {
+ should_break: true, ..
+ }) => return Err(LineBreakRequiredError {}),
FormatToken::Group(group) => vec![PrintTokenCall::new(group.content.as_ref(), args)],
@@ -251,14 +280,21 @@ impl Printer {
}
fn print_str(&mut self, content: &str) {
+ self.state.buffer.reserve(content.len());
+
for char in content.chars() {
+ if char == '\n' {
+ for char in self.options.line_ending.as_str().chars() {
self.state.generated_index += 1;
+ self.state.buffer.push(char);
+ }
- if char == '\n' {
self.state.generated_line += 1;
self.state.generated_column = 0;
self.state.line_width = 0;
} else {
+ self.state.buffer.push(char);
+ self.state.generated_index += 1;
self.state.generated_column += 1;
let char_width = if char == '\t' {
@@ -270,8 +306,6 @@ impl Printer {
self.state.line_width += char_width;
}
}
-
- self.state.buffer.push_str(content);
}
}
@@ -409,8 +443,8 @@ impl<'a> TokenCallQueue<'a> {
#[cfg(test)]
mod tests {
- use crate::format_token::{GroupToken, IfBreakToken, IndentToken, LineToken};
- use crate::printer::{PrintResult, Printer, PrinterOptions};
+ use crate::format_token::{GroupToken, IfBreakToken, IndentToken, LineToken, ListToken};
+ use crate::printer::{LineEnding, PrintResult, Printer, PrinterOptions};
use crate::FormatToken;
/// Prints the given token with the default printer options
@@ -480,13 +514,13 @@ a"#,
#[test]
fn it_breaks_a_group_if_a_string_contains_a_newline() {
let result = print_token(create_array_tokens(vec![
- FormatToken::string("`This string spans\ntwo lines`"),
+ FormatToken::string("`This is a string spanning\ntwo lines`"),
FormatToken::string("\"b\""),
]));
assert_eq!(
r#"[
- `This string spans
+ `This is a string spanning
two lines`,
"b",
]"#,
@@ -494,6 +528,32 @@ two lines`,
)
}
+ #[test]
+ fn it_converts_line_endings_in_strings() {
+ let options = PrinterOptions {
+ line_ending: LineEnding::CarriageReturnLineFeed,
+ ..PrinterOptions::default()
+ };
+
+ let program = ListToken::concat(vec![
+ FormatToken::string("function main() {"),
+ FormatToken::Indent(IndentToken::new(ListToken::concat(vec![
+ FormatToken::Line(LineToken::hard()),
+ FormatToken::string("let x = `This is a multiline\nstring`;"),
+ ]))),
+ FormatToken::Line(LineToken::hard()),
+ FormatToken::string("}"),
+ FormatToken::Line(LineToken::hard()),
+ ]);
+
+ let result = Printer::new(options).print(&FormatToken::from(program));
+
+ assert_eq!(
+ "function main() {\r\n\tlet x = `This is a multiline\r\nstring`;\r\n}\r\n",
+ result.code
+ );
+ }
+
#[test]
fn it_breaks_parent_groups_if_they_dont_fit_on_a_single_line() {
let result = print_token(create_array_tokens(vec![
@@ -528,6 +588,7 @@ two lines`,
indent_string: String::from("\t"),
tab_width: 4,
print_width: 19,
+ ..PrinterOptions::default()
});
let result = printer.print(&create_array_tokens(vec![

feat(formatter): Support different line endings (#1672)

Adds support for different line endings to the formatter's printer.
Line feeds will be used internally that then get converted into whatever the users prefer when `Printing` the tokens.