feature(xtask): git integration for formatter codegen
MODIFY
Cargo.lock
Cargo.lock
@@ -173,6 +173,9 @@ name = "cc"
version = "1.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee"
+dependencies = [
+ "jobserver",
+]
[[package]]
name = "cfg-if"
@@ -564,6 +567,21 @@ version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4"
+[[package]]
+name = "git2"
+version = "0.13.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6"
+dependencies = [
+ "bitflags",
+ "libc",
+ "libgit2-sys",
+ "log",
+ "openssl-probe",
+ "openssl-sys",
+ "url",
+]
+
[[package]]
name = "globset"
version = "0.4.8"
@@ -729,6 +747,15 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
+[[package]]
+name = "jobserver"
+version = "0.1.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "js-sys"
version = "0.3.56"
@@ -750,6 +777,46 @@ version = "0.2.113"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eef78b64d87775463c549fbd80e19249ef436ea3bf1de2a1eb7e717ec7fab1e9"
+[[package]]
+name = "libgit2-sys"
+version = "0.12.26+1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494"
+dependencies = [
+ "cc",
+ "libc",
+ "libssh2-sys",
+ "libz-sys",
+ "openssl-sys",
+ "pkg-config",
+]
+
+[[package]]
+name = "libssh2-sys"
+version = "0.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b094a36eb4b8b8c8a7b4b8ae43b2944502be3e59cd87687595cf6b0a71b3f4ca"
+dependencies = [
+ "cc",
+ "libc",
+ "libz-sys",
+ "openssl-sys",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
+name = "libz-sys"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de5435b8549c16d423ed0c03dbaafe57cf6c3344744f1242520d59c9d8ecec66"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
[[package]]
name = "linked-hash-map"
version = "0.5.4"
@@ -953,6 +1020,25 @@ version = "11.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
+[[package]]
+name = "openssl-probe"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.72"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb"
+dependencies = [
+ "autocfg",
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
[[package]]
name = "os_str_bytes"
version = "6.0.0"
@@ -1034,6 +1120,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+[[package]]
+name = "pkg-config"
+version = "0.3.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe"
+
[[package]]
name = "plotters"
version = "0.3.1"
@@ -1911,6 +2003,12 @@ dependencies = [
"serde",
]
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
[[package]]
name = "version_check"
version = "0.9.4"
@@ -2123,10 +2221,10 @@ name = "xtask_codegen"
version = "0.0.0"
dependencies = [
"anyhow",
+ "git2",
"pico-args",
"proc-macro2",
"quote",
- "syn",
"ungrammar",
"ureq",
"walkdir",

MODIFY
xtask/codegen/Cargo.toml
xtask/codegen/Cargo.toml
@@ -14,7 +14,4 @@ proc-macro2 = { version = "1.0.36", features = ["span-locations"] }
ungrammar = "1.14.9"
walkdir = "2.3.2"
ureq = "2.4.0"
-
-[dependencies.syn]
-version = "*"
-features = ["parsing", "printing", "full"]
+git2 = "0.13.25"

MODIFY
xtask/codegen/src/formatter.rs
xtask/codegen/src/formatter.rs
-#![allow(deprecated)]
-
use std::{
- cell::RefCell,
- cmp::Ordering,
- collections::{BTreeMap, BTreeSet},
- fs::{create_dir_all, read_dir, read_to_string, remove_file, File},
- io::{self, Write},
+ collections::{BTreeMap, BTreeSet, HashSet, VecDeque},
+ env,
+ fs::{create_dir_all, read_dir, remove_file, File},
+ io::Write,
path::{Path, PathBuf},
- rc::Rc,
};
+use git2::{Repository, Status, StatusOptions};
use proc_macro2::{Ident, Span};
use quote::quote;
-use syn::{parse_file, spanned::Spanned};
use xtask::project_root;
use crate::ast::load_ast;
-#[deprecated]
-struct FileStore {
- use_items: Vec<String>,
- impl_items: BTreeMap<String, String>,
+struct GitRepo {
+ repo: Repository,
+ allow_staged: bool,
+ staged: HashSet<PathBuf>,
+ dirty: HashSet<PathBuf>,
}
-#[deprecated]
-type FileSet = Rc<RefCell<FileStore>>;
+impl GitRepo {
+ fn open() -> Self {
+ let root = project_root();
+ let repo = Repository::discover(&root).expect("failed to open git repo");
-#[deprecated]
-#[derive(Default)]
-struct NodeIndex {
- entries: BTreeMap<String, NodeIndexEntry>,
- files: BTreeMap<PathBuf, FileSet>,
+ let mut allow_staged = false;
+ let mut allow_dirty = false;
+ for arg in env::args() {
+ match arg.as_str() {
+ "--allow-staged" => {
+ allow_staged = true;
+ }
+ "--allow-dirty" => {
+ allow_dirty = true;
+ }
+ _ => {}
}
-
-#[deprecated]
-struct NodeIndexEntry {
- use_items: Vec<String>,
- impl_item: String,
- file: FileSet,
}
-#[deprecated]
-fn load_source(index: &mut NodeIndex, path: &Path) {
- let code_str = read_to_string(path).unwrap();
- let code = parse_file(&code_str)
- .unwrap_or_else(|err| panic!("failed to parse {}: {:?}", path.display(), err));
+ let mut repo_opts = StatusOptions::new();
+ repo_opts.include_ignored(false);
- let mut use_items: Vec<String> = Vec::new();
- let mut impl_items: BTreeMap<String, String> = BTreeMap::new();
+ let statuses = repo
+ .statuses(Some(&mut repo_opts))
+ .expect("failed to read repository status");
- for item in code.items {
- match item {
- syn::Item::Use(use_item) => {
- use_items.push(slice_source(&code_str, &use_item));
- }
+ let mut staged = HashSet::new();
+ let mut dirty = HashSet::new();
- syn::Item::Impl(impl_item) => {
- if let Some((_, path, _)) = &impl_item.trait_ {
- if let Some(segment) = path.segments.last() {
- if segment.ident == "ToFormatElement" {
- if let syn::Type::Path(path) = &*impl_item.self_ty {
- if let Some(segment) = path.path.segments.last() {
- impl_items.insert(
- segment.ident.to_string(),
- slice_source(&code_str, &impl_item),
- );
+ for status in statuses.iter() {
+ if let Some(path) = status.path() {
+ match status.status() {
+ Status::CURRENT => (),
+ Status::INDEX_NEW
+ | Status::INDEX_MODIFIED
+ | Status::INDEX_DELETED
+ | Status::INDEX_RENAMED
+ | Status::INDEX_TYPECHANGE => {
+ if !allow_staged {
+ staged.insert(root.join(path));
}
}
+ _ => {
+ if !allow_dirty {
+ dirty.insert(root.join(path));
}
}
+ };
}
}
- _ => {}
+ drop(statuses);
+
+ Self {
+ repo,
+ allow_staged,
+ staged,
+ dirty,
}
}
- let file = Rc::new(RefCell::new(FileStore {
- use_items: use_items.clone(),
- impl_items: impl_items.clone(),
- }));
+ fn check_path(&self, path: &Path) {
+ if self.dirty.contains(path) {
+ panic!("Codegen would overwrite '{}' but it has uncommited changes. Commit the file to git, or pass --allow-dirty to the command to proceed anyway", path.display());
+ }
+ if self.staged.contains(path) {
+ panic!("Codegen would overwrite '{}' but it has uncommited changes. Commit the file to git, or pass --allow-staged to the command to proceed anyway", path.display());
+ }
+ }
- index.files.insert(path.into(), Rc::clone(&file));
+ fn stage_paths(&self, paths: &[PathBuf]) {
+ // Do not overwrite a version of the file
+ // that's potentially already staged
+ if self.allow_staged {
+ return;
+ }
- for (key, impl_item) in impl_items {
- index.entries.insert(
- key,
- NodeIndexEntry {
- use_items: use_items.clone(),
- impl_item,
- file: Rc::clone(&file),
- },
- );
+ let root = project_root();
+ self.repo
+ .index()
+ .expect("could not open index for git repository")
+ .update_all(
+ paths.iter().map(|path| {
+ path.strip_prefix(&root).unwrap_or_else(|err| {
+ panic!(
+ "path '{}' is not inside of project '{}': {}",
+ path.display(),
+ root.display(),
+ err,
+ )
+ })
+ }),
+ None,
+ )
+ .expect("failed to stage updated files");
}
}
-#[deprecated]
-fn slice_source(source: &str, node: &impl Spanned) -> String {
- let span = node.span();
- let start = span.start();
- let end = span.end();
-
- let mut buffer = String::new();
- for (line_index, line_content) in source.lines().enumerate() {
- let line_number = line_index + 1;
- let (offset, line_content) = match line_number.cmp(&start.line) {
- Ordering::Less => continue,
- Ordering::Equal => (start.column, &line_content[start.column..]),
- Ordering::Greater => (0, line_content),
- };
+struct ModuleIndex {
+ root: PathBuf,
+ modules: BTreeMap<PathBuf, BTreeSet<String>>,
+ unused_files: HashSet<PathBuf>,
+}
- let line_content = match line_number.cmp(&end.line) {
- Ordering::Less => line_content,
- Ordering::Equal => &line_content[..end.column - offset],
- Ordering::Greater => break,
- };
+impl ModuleIndex {
+ fn new(root: PathBuf) -> Self {
+ let mut unused_files = HashSet::new();
+ let mut queue = VecDeque::new();
- buffer.push_str(line_content);
- buffer.push('\n');
- }
+ queue.push_back(root.join("js"));
+ queue.push_back(root.join("ts"));
- buffer
-}
+ while let Some(dir) = queue.pop_front() {
+ let iter = read_dir(&dir)
+ .unwrap_or_else(|err| panic!("failed to read '{}': {}", dir.display(), err));
-#[deprecated]
-fn traverse_directory(index: &mut NodeIndex, path: &Path) -> io::Result<()> {
- for entry in read_dir(path)? {
- let entry = match entry {
- Ok(entry) => entry,
- Err(_) => continue,
- };
+ for entry in iter {
+ let entry = entry.expect("failed to read DirEntry");
- let kind = match entry.file_type() {
- Ok(kind) => kind,
- Err(_) => continue,
- };
+ let path = entry.path();
+ let file_type = entry.file_type().unwrap_or_else(|err| {
+ panic!("failed to read file type of '{}': {}", path.display(), err)
+ });
- if kind.is_file() {
- load_source(index, &entry.path());
+ if file_type.is_dir() {
+ queue.push_back(path);
continue;
}
- if kind.is_dir() {
- traverse_directory(index, &entry.path()).ok();
- continue;
+ if file_type.is_file() {
+ unused_files.insert(path);
}
}
-
- Ok(())
-}
-
-struct ModuleIndex {
- root: PathBuf,
- modules: BTreeMap<PathBuf, BTreeSet<String>>,
}
-impl ModuleIndex {
- fn new(root: PathBuf) -> Self {
Self {
root,
modules: BTreeMap::default(),
+ unused_files,
}
}
/// Add a new module to the index
- fn insert(&mut self, path: &Path) {
+ fn insert(&mut self, repo: &GitRepo, path: &Path) {
+ self.unused_files.remove(path);
+
// Walk up from the module file towards the root
let mut parent = path.parent();
let mut file_stem = path.file_stem();
while let (Some(path), Some(stem)) = (parent, file_stem) {
+ repo.check_path(&path.join("mod.rs"));
+
// Insert each module into its parent
let stem = stem.to_str().unwrap().to_owned();
self.modules.entry(path.into()).or_default().insert(stem);
@@ -186,7 +189,7 @@ impl ModuleIndex {
/// Create all the mod.rs files needed to import
/// all the modules in the index up to the root
- fn print(self) {
+ fn print(mut self, stage: &mut Vec<PathBuf>) {
for (path, imports) in self.modules {
let mut content = String::new();
@@ -208,6 +211,16 @@ impl ModuleIndex {
let path = path.join("mod.rs");
let mut file = File::create(&path).unwrap();
file.write_all(content.as_bytes()).unwrap();
+ drop(file);
+
+ self.unused_files.remove(&path);
+ stage.push(path);
+ }
+
+ for file in self.unused_files {
+ remove_file(&file)
+ .unwrap_or_else(|err| panic!("failed to delete '{}': {}", file.display(), err));
+ stage.push(file);
}
}
}
@@ -220,15 +233,7 @@ enum NodeKind {
}
pub fn generate_formatter() {
- let mut index = NodeIndex::default();
-
- if true {
- traverse_directory(
- &mut index,
- &project_root().join("crates/rome_formatter/src/old"),
- )
- .ok();
- }
+ let repo = GitRepo::open();
let ast = load_ast();
@@ -263,11 +268,13 @@ pub fn generate_formatter() {
)
}));
+ let mut stage = Vec::new();
+
// Create a default implementation for theses nodes only if
// the file doesn't already exist
for (kind, name) in names {
let path = name_to_path(&kind, &name);
- modules.insert(&path);
+ modules.insert(&repo, &path);
// Union nodes except for AnyFunction and AnyClass have a generated
// implementation, the codegen will always overwrite any existing file
@@ -279,19 +286,11 @@ pub fn generate_formatter() {
continue;
}
+ repo.check_path(&path);
+
let dir = path.parent().unwrap();
create_dir_all(dir).unwrap();
- let tokens = match index.entries.remove(&name) {
- Some(entry) if !allow_overwrite => {
- let use_items = entry.use_items;
- let impl_item = entry.impl_item;
-
- entry.file.borrow_mut().impl_items.remove(&name);
-
- format!("{}\n{}", use_items.join("\n"), impl_item)
- }
- _ => {
let id = Ident::new(&name, Span::call_site());
// Generate a default implementation of ToFormatElement using format_list on
@@ -345,32 +344,21 @@ pub fn generate_formatter() {
}
};
- if allow_overwrite {
+ let tokens = if allow_overwrite {
xtask::reformat(tokens).unwrap()
} else {
xtask::reformat_without_preamble(tokens).unwrap()
- }
- }
};
let mut file = File::create(&path).unwrap();
file.write_all(tokens.as_bytes()).unwrap();
- }
-
- for (path, entry) in index.files {
- let entry = entry.borrow();
- if entry.impl_items.is_empty() {
- remove_file(path).unwrap();
- } else {
- let impl_items: Vec<_> = entry.impl_items.values().cloned().collect();
- let tokens = format!("{}\n{}", entry.use_items.join("\n"), impl_items.join("\n"));
+ drop(file);
- let mut file = File::create(&path).unwrap();
- file.write_all(tokens.as_bytes()).unwrap();
- }
+ stage.push(path);
}
- modules.print();
+ modules.print(&mut stage);
+ repo.stage_paths(&stage);
}
enum NodeLanguage {

feature(xtask): git integration for formatter codegen (#2116)