ci: first draft of comapre results
MODIFY
.github/workflows/ci.yml
.github/workflows/ci.yml
@@ -74,25 +74,3 @@ jobs:
command: test
args: --workspace --verbose
- coverage:
- name: Test262 coverage
- runs-on: ${{ matrix.os }}
- strategy:
- fail-fast: false
- matrix:
- os: [windows-latest, macos-latest, ubuntu-latest]
- steps:
- - name: Checkout repository
- uses: actions/checkout@v2
- with:
- submodules: recursive
- - name: Install toolchain
- uses: actions-rs/toolchain@v1
- with:
- toolchain: 1.56.1
- profile: minimal
- - name: Cache
- uses: Swatinem/rust-cache@v1
- - name: Run Test262 suite
- continue-on-error: true
- run: cargo xtask coverage

ADD
.github/workflows/test262.yml
+name: CI
+on:
+ push:
+ branches:
+ - main
+ pull_request:
+ branches:
+ - main
+
+env:
+ RUST_LOG: info
+ RUST_BACKTRACE: 1
+
+jobs:
+ coverage:
+ name: Test262 coverage
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [windows-latest, macos-latest, ubuntu-latest]
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+ with:
+ submodules: recursive
+ - name: Install toolchain
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: 1.56.1
+ profile: minimal
+ - name: Cache
+ uses: Swatinem/rust-cache@v1
+ - name: Run Test262 suite
+ continue-on-error: true
+ run: cargo xtask coverage --json
+ - name: Rename the emitted file
+ run: |
+ mv base_results.json new_results.json
+ - name: Save test results
+ uses: actions/upload-artifact@v2
+ with:
+ name: new_results
+ path: new_results.json
+
+ compare:
+ needs: coverage
+ name: Compare coverage
+ runs-on: ${{ matrix.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ os: [windows-latest, macos-latest, ubuntu-latest]
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+ with:
+ submodules: recursive
+ # To restore once the main branch has the new command
+ # ref: main
+ - name: Install toolchain
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: 1.56.1
+ profile: minimal
+ - name: Cache
+ uses: Swatinem/rust-cache@v1
+ - name: Run Test262 suite on main branch
+ continue-on-error: true
+ run: cargo xtask coverage --json
+ - name: Download PRs test results
+ uses: actions/download-artifact@v2
+ with:
+ name: new_results
+ - name: Compare results on ${{ matrix.os }}
+ run: cargo xtask compare ./base_results.json ./new_results.json --markdown

ADD
xtask/src/compare/mod.rs
+use crate::{project_root, BASE_RESULT_FILE, NEW_RESULT_FILE};
+use std::path::PathBuf;
+
+mod results;
+
+pub fn run(base_result_path: Option<&str>, new_result_path: Option<&str>, markdown: bool) {
+ // resolve the path passed as argument, or retrieve the default one
+ let base_result_dir = if let Some(base_result_path) = base_result_path {
+ PathBuf::from(base_result_path)
+ } else {
+ project_root().join(BASE_RESULT_FILE)
+ };
+
+ // resolve the path passed as argument, or retrieve the default one
+ let new_result_dir = if let Some(new_result_path) = new_result_path {
+ PathBuf::from(new_result_path)
+ } else {
+ project_root().join(NEW_RESULT_FILE)
+ };
+
+ if !base_result_dir.exists() {
+ panic!(
+ "The path to the base results doesn't exist: {:?}",
+ base_result_dir
+ );
+ }
+
+ if !&new_result_dir.exists() {
+ panic!(
+ "The path to the new results doesn't exist: {:?}",
+ new_result_dir
+ );
+ }
+
+ results::emit_compare(
+ base_result_dir.as_path(),
+ new_result_dir.as_path(),
+ markdown,
+ );
+}

ADD
xtask/src/compare/results.rs
+use crate::coverage::files::{Outcome, TestResults};
+use ascii_table::{AsciiTable, Column};
+use colored::Colorize;
+use std::{fs::File, path::Path};
+
+pub fn emit_compare(base: &Path, new: &Path, markdown: bool) {
+ let base_results: TestResults =
+ serde_json::from_reader(File::open(base).expect("Can't read the file of the base results"))
+ .expect("Can't parse the JSON file of the base results");
+ let new_results: TestResults =
+ serde_json::from_reader(File::open(new).expect("Can't read the file of the new results"))
+ .expect("Can't parse the JSON file of the new results");
+
+ let base_total = base_results.summary.tests_ran as isize;
+ let new_total = new_results.summary.tests_ran as isize;
+ let total_diff = new_total - base_total;
+
+ let base_passed = base_results.summary.passed as isize;
+ let new_passed = new_results.summary.passed as isize;
+ let passed_diff = new_passed - base_passed;
+
+ let base_failed = base_results.summary.failed as isize;
+ let new_failed = new_results.summary.failed as isize;
+ let failed_diff = new_failed - base_failed;
+
+ let base_panics = base_results.summary.panics as isize;
+ let new_panics = new_results.summary.panics as isize;
+ let panics_diff = new_panics - base_panics;
+
+ let base_coverage = base_results.summary.coverage;
+ let new_coverage = new_results.summary.coverage;
+ let coverage_diff = new_coverage - base_coverage;
+
+ let report_diff = compare_diffs(&base_results, &new_results);
+
+ if markdown {
+ /// Generates a proper diff format, with some bold text if things change.
+ fn diff_format(diff: isize) -> String {
+ format!(
+ "{}{}{}{}",
+ if diff != 0 { "**" } else { "" },
+ if diff > 0 { "+" } else { "" },
+ diff,
+ if diff != 0 { "**" } else { "" }
+ )
+ }
+
+ println!("| Test result | `main` count | This PR count | Difference |");
+ println!("| :---------: | :----------: | :-----------: | :--------: |");
+
+ println!(
+ "| Total | {} | {} | {} |",
+ base_total,
+ new_total,
+ diff_format(total_diff)
+ );
+
+ println!(
+ "| Passed | {} | {} | {} |",
+ base_passed,
+ new_passed,
+ diff_format(passed_diff)
+ );
+
+ println!(
+ "| Failed | {} | {} | {} |",
+ base_failed,
+ new_failed,
+ diff_format(failed_diff)
+ );
+
+ println!(
+ "| Panics | {} | {} | {} |",
+ base_passed,
+ new_passed,
+ diff_format(panics_diff)
+ );
+
+ println!(
+ "| Coverage | {:.2}% | {:.2}% | {} |",
+ base_coverage,
+ new_coverage,
+ format!(
+ "{}{}{:.2}%{}",
+ if coverage_diff.abs() > f64::EPSILON {
+ "**"
+ } else {
+ ""
+ },
+ if coverage_diff > 0_f64 { "+" } else { "" },
+ coverage_diff,
+ if coverage_diff.abs() > f64::EPSILON {
+ "**"
+ } else {
+ ""
+ },
+ ),
+ );
+
+ if !report_diff.fixed.is_empty() {
+ println!();
+ println!(
+ "<details><summary><b>Fixed tests ({}):</b></summary>",
+ report_diff.fixed.len()
+ );
+ println!("\n```");
+ for test in report_diff.fixed {
+ println!("{}", test);
+ }
+ println!("```");
+ println!("</details>");
+ }
+
+ if !report_diff.failed.is_empty() {
+ println!();
+ println!(
+ "<details><summary><b>Failed tests ({}):</b></summary>",
+ report_diff.failed.len()
+ );
+ println!("\n```");
+ for test in report_diff.failed {
+ println!("{}", test);
+ }
+ println!("```");
+ println!("</details>");
+ }
+
+ if !report_diff.new_panics.is_empty() {
+ println!();
+ println!(
+ "<details><summary><b>New panics ({}):</b></summary>",
+ report_diff.new_panics.len()
+ );
+ println!("\n```");
+ for test in report_diff.new_panics {
+ println!("{}", test);
+ }
+ println!("```");
+ println!("</details>");
+ }
+
+ if !report_diff.panic_fixed.is_empty() {
+ println!();
+ println!(
+ "<details><summary><b>Panics fixed ({}):</b></summary>",
+ report_diff.panic_fixed.len()
+ );
+ println!("\n```");
+ for test in report_diff.panic_fixed {
+ println!("{}", test);
+ }
+ println!("```");
+ println!("</details>");
+ }
+ } else {
+ let mut table = AsciiTable::default();
+ let mut counter = 0usize;
+
+ let mut create_column = |name: colored::ColoredString| {
+ let column = Column {
+ header: name.to_string(),
+ align: ascii_table::Align::Center,
+ ..Column::default()
+ };
+ table.columns.insert(counter, column);
+ counter += 1;
+ };
+
+ println!("Test262 conformance changes:");
+
+ create_column("Tests result".into());
+ create_column("main branch".green());
+ create_column("PR".yellow());
+ create_column("Difference".cyan());
+
+ let passed_diff = base_passed - new_passed;
+ let failed_diff = base_failed - new_failed;
+ let panics_diff = base_panics - new_panics;
+ let passed_row: Vec<&dyn std::fmt::Display> =
+ vec![&"Passed", &base_passed, &new_passed, &passed_diff];
+ let failed_row: Vec<&dyn std::fmt::Display> =
+ vec![&"Failed", &base_failed, &new_failed, &failed_diff];
+ let panics_row: Vec<&dyn std::fmt::Display> =
+ vec![&"Panics", &base_panics, &new_panics, &panics_diff];
+ table.print(vec![passed_row, failed_row, panics_row]);
+ }
+}
+
+struct ReportDiff<'a> {
+ pub fixed: Vec<&'a str>,
+ pub failed: Vec<&'a str>,
+ pub new_panics: Vec<&'a str>,
+ pub panic_fixed: Vec<&'a str>,
+}
+
+impl<'a> ReportDiff<'a> {
+ pub fn new() -> Self {
+ Self {
+ fixed: vec![],
+ failed: vec![],
+ new_panics: vec![],
+ panic_fixed: vec![],
+ }
+ }
+}
+
+fn compare_diffs<'a>(
+ base_results: &'a TestResults,
+ new_results: &'a TestResults,
+) -> ReportDiff<'a> {
+ let mut report_diff = ReportDiff::new();
+ for base_result in &base_results.details {
+ let test_to_analyze = new_results
+ .details
+ .iter()
+ .find(|new_test| new_test.path.as_os_str().eq(base_result.path.as_os_str()));
+
+ if let Some(test_to_analyze) = test_to_analyze {
+ match (&base_result.outcome, &test_to_analyze.outcome) {
+ // we want to ignore cases where both results yield the same enum
+ // this means that their status hasn't changed, not worth tracking
+ (b, n) if b == n => {}
+ // the new result passed
+ (_, Outcome::Passed) => report_diff
+ .fixed
+ .push(test_to_analyze.path.as_os_str().to_str().unwrap()),
+ // an old test passed but now failed
+ (Outcome::Passed, Outcome::Failed) => report_diff
+ .failed
+ .push(test_to_analyze.path.as_os_str().to_str().unwrap()),
+ // an existing test now panics
+ (_, Outcome::Panicked) => report_diff
+ .new_panics
+ .push(test_to_analyze.path.as_os_str().to_str().unwrap()),
+ // a panic error is now fixed
+ (Outcome::Panicked, _) => report_diff
+ .panic_fixed
+ .push(test_to_analyze.path.as_os_str().to_str().unwrap()),
+ _ => {}
+ }
+ }
+ }
+
+ report_diff
+}

MODIFY
xtask/src/coverage/files.rs
xtask/src/coverage/files.rs
@@ -2,14 +2,17 @@ use colored::Colorize;
use indicatif::ProgressBar;
use regex::Regex;
use rslint_parser::ParserError;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
use std::any::Any;
use std::fs::read_to_string;
use std::io;
use std::path::PathBuf;
+use std::{fs::File, io::Write};
use walkdir::WalkDir;
use yastl::Pool;
+use crate::{project_root, BASE_RESULT_FILE};
+
const BASE_PATH: &str = "xtask/src/coverage/test262/test";
#[derive(Debug, Clone)]
@@ -152,10 +155,15 @@ pub fn get_test_files(query: Option<&str>, pool: &Pool) -> Vec<TestFile> {
files
}
-#[derive(Debug)]
+#[derive(Debug, Serialize, Deserialize)]
pub struct TestResult {
+ #[serde(skip)]
pub fail: Option<FailReason>,
+ #[serde(rename = "o")]
+ pub outcome: Outcome,
+ #[serde(rename = "h")]
pub path: PathBuf,
+ #[serde(skip)]
pub code: String,
}
@@ -165,3 +173,103 @@ pub enum FailReason {
IncorrectlyErrored(Vec<ParserError>),
ParserPanic(Box<dyn Any + Send + 'static>),
}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub enum Outcome {
+ Passed,
+ Failed,
+ Panicked,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct TestResults {
+ #[serde(rename = "s")]
+ pub summary: Summary,
+ #[serde(rename = "p")]
+ pub details: Vec<TestResult>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Summary {
+ #[serde(rename = "a")]
+ pub tests_ran: u32,
+ #[serde(rename = "pa")]
+ pub passed: u32,
+ #[serde(rename = "f")]
+ pub failed: u32,
+ #[serde(rename = "pc")]
+ pub panics: u32,
+ #[serde(rename = "c")]
+ pub coverage: f64,
+}
+
+impl Default for TestResults {
+ fn default() -> Self {
+ Self {
+ summary: Summary {
+ tests_ran: 0,
+ passed: 0,
+ failed: 0,
+ panics: 0,
+ coverage: 0.0,
+ },
+ details: vec![],
+ }
+ }
+}
+
+impl TestResults {
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ pub fn store_results(&mut self, results: Vec<TestResult>) {
+ self.details = results;
+ let passed = self.passed_tests() as u32;
+ let tests_ran = self.details.len();
+ let coverage = (passed as f64 / tests_ran as f64) * 100.0;
+ self.summary = Summary {
+ tests_ran: self.details.len() as u32,
+ passed,
+ failed: self.errored_tests() as u32,
+ panics: self.panicked_tests() as u32,
+ coverage,
+ };
+ }
+
+ pub fn panicked_tests(&self) -> usize {
+ self.details
+ .iter()
+ .filter(|res| matches!(res.fail, Some(FailReason::ParserPanic(_))))
+ .count()
+ }
+
+ pub fn errored_tests(&self) -> usize {
+ self.details
+ .iter()
+ .filter(|res| {
+ matches!(
+ res.fail,
+ Some(FailReason::IncorrectlyErrored(_)) | Some(FailReason::IncorrectlyPassed)
+ )
+ })
+ .count()
+ }
+
+ pub fn passed_tests(&self) -> usize {
+ self.details.iter().filter(|res| res.fail.is_none()).count()
+ }
+
+ /// Saves results into a JSON file inside the temporary directory of the OS
+ pub fn dump_to_json(&self) {
+ let json = serde_json::to_string(&self).unwrap();
+ let path = project_root().join(BASE_RESULT_FILE);
+
+ let mut file = File::create(&path).expect("Can't open the JSON file");
+
+ file.write_all(json.as_bytes())
+ .expect("Can't write in the JSON file");
+ println!();
+ println!("The test result report has been saved in: {:?}", &path);
+ }
+}

MODIFY
xtask/src/coverage/mod.rs
xtask/src/coverage/mod.rs
-mod files;
+pub mod files;
use ascii_table::{AsciiTable, Column};
use colored::Colorize;
@@ -8,7 +8,9 @@ use std::any::Any;
use std::path::PathBuf;
use yastl::Pool;
-pub fn run(query: Option<&str>, pool: Pool) {
+pub const TEST_JSON_PATH: &str = "xtask/src/base_results.json";
+
+pub fn run(query: Option<&str>, pool: Pool, json: bool) {
let files = get_test_files(query, &pool);
let num_ran = files.len();
@@ -21,6 +23,7 @@ pub fn run(query: Option<&str>, pool: Pool) {
std::panic::set_hook(Box::new(|_| {}));
let start_tests = std::time::Instant::now();
+ let mut test_results = TestResults::new();
let (tx, rx) = std::sync::mpsc::channel();
@@ -63,7 +66,7 @@ pub fn run(query: Option<&str>, pool: Pool) {
});
drop(tx);
- let res = rx.into_iter().collect::<Vec<_>>();
+ test_results.store_results(rx.into_iter().collect::<Vec<_>>());
let _ = std::panic::take_hook();
@@ -75,20 +78,10 @@ pub fn run(query: Option<&str>, pool: Pool) {
start_tests.elapsed().as_secs_f32()
);
- let panicked = res
- .iter()
- .filter(|res| matches!(res.fail, Some(FailReason::ParserPanic(_))))
- .count();
- let errored = res
- .iter()
- .filter(|res| {
- matches!(
- res.fail,
- Some(FailReason::IncorrectlyErrored(_)) | Some(FailReason::IncorrectlyPassed)
- )
- })
- .count();
- let passed = res.iter().filter(|res| res.fail.is_none()).count();
+ let panicked = test_results.summary.panics;
+ let errored = test_results.summary.failed;
+ let passed = test_results.summary.passed;
+ let coverage = format!("{:.2}", test_results.summary.coverage);
let mut table = AsciiTable::default();
@@ -108,13 +101,15 @@ pub fn run(query: Option<&str>, pool: Pool) {
create_column("Failed".red());
create_column("Panics".red());
create_column("Coverage".cyan());
-
- let coverage = (passed as f64 / num_ran as f64) * 100.0;
- let coverage = format!("{:.2}", coverage);
let numbers: Vec<&dyn std::fmt::Display> =
vec![&num_ran, &passed, &errored, &panicked, &coverage];
+
table.print(vec![numbers]);
+ if json {
+ test_results.dump_to_json();
+ }
+
if passed > 0 {
std::process::exit(1);
} else {
@@ -128,15 +123,33 @@ pub fn run_test_file(file: TestFile) -> TestResult {
if meta.flags.contains(&TestFlag::OnlyStrict) {
let (code, res) = exec_test(code, true, false);
let fail = passed(res, meta);
- TestResult { fail, path, code }
+ let outcome = extract_outcome(&fail);
+ TestResult {
+ fail,
+ path,
+ code,
+ outcome,
+ }
} else if meta.flags.contains(&TestFlag::NoStrict) || meta.flags.contains(&TestFlag::Raw) {
let (code, res) = exec_test(code, false, false);
let fail = passed(res, meta);
- TestResult { fail, path, code }
+ let outcome = extract_outcome(&fail);
+ TestResult {
+ fail,
+ path,
+ code,
+ outcome,
+ }
} else if meta.flags.contains(&TestFlag::Module) {
let (code, res) = exec_test(code, false, true);
let fail = passed(res, meta);
- TestResult { fail, path, code }
+ let outcome = extract_outcome(&fail);
+ TestResult {
+ fail,
+ path,
+ code,
+ outcome,
+ }
} else {
let (_, l) = exec_test(code.clone(), false, false);
let (code, r) = exec_test(code, true, false);
@@ -208,7 +221,14 @@ fn default_bar_style() -> indicatif::ProgressStyle {
fn merge_tests(code: String, l: ExecRes, r: ExecRes, meta: MetaData, path: PathBuf) -> TestResult {
let fail = passed(l, meta.clone()).or_else(|| passed(r, meta));
- TestResult { fail, path, code }
+ // outcome lowers down the reason of failing, being able to use serde over it
+ let outcome = extract_outcome(&fail);
+ TestResult {
+ fail,
+ path,
+ code,
+ outcome,
+ }
}
fn passed(res: ExecRes, meta: MetaData) -> Option<FailReason> {
@@ -227,6 +247,17 @@ fn passed(res: ExecRes, meta: MetaData) -> Option<FailReason> {
}
}
+fn extract_outcome(fail: &Option<FailReason>) -> Outcome {
+ if let Some(fail) = fail {
+ match fail {
+ FailReason::IncorrectlyPassed | FailReason::IncorrectlyErrored(_) => Outcome::Failed,
+ FailReason::ParserPanic(_) => Outcome::Panicked,
+ }
+ } else {
+ Outcome::Passed
+ }
+}
+
enum ExecRes {
Errors(Vec<ParserError>),
ParseCorrectly,

MODIFY
xtask/src/coverage/test262
xtask/src/coverage/test262
-Subproject commit 26f1f4567ee7e33163d961c867d689173cbb9065
+Subproject commit 19da3ca0757248f7595ee09d532bb83dd438f2b5

MODIFY
xtask/src/lib.rs
xtask/src/lib.rs
//! Codegen tools mostly used to generate ast and syntax definitions. Adapted from rust analyzer's codegen
pub mod codegen;
+pub mod compare;
pub mod coverage;
pub mod docgen;
pub mod glue;
@@ -17,6 +18,12 @@ use crate::{
pub use anyhow::{bail, Context as _, Result};
+// this is the filename of the results coming from `main` branch
+pub const BASE_RESULT_FILE: &str = "base_results.json";
+
+// this is the filename of the results coming from the current PR
+pub const NEW_RESULT_FILE: &str = "new_results.json";
+
pub fn project_root() -> PathBuf {
Path::new(
&env::var("CARGO_MANIFEST_DIR").unwrap_or_else(|_| env!("CARGO_MANIFEST_DIR").to_owned()),

MODIFY
xtask/src/main.rs
xtask/src/main.rs
use pico_args::Arguments;
use xtask::{
codegen::{self, Mode},
- coverage,
- // docgen,
+ compare, coverage,
glue::pushd,
- project_root,
- run_rustfmt,
- Result,
+ project_root, run_rustfmt, Result,
};
fn main() -> Result<()> {
@@ -14,7 +11,6 @@ fn main() -> Result<()> {
let mut args = Arguments::from_env();
let subcommand = args.subcommand()?.unwrap_or_default();
-
match subcommand.as_str() {
"codegen" => {
args.finish()?;
@@ -30,17 +26,28 @@ fn main() -> Result<()> {
args.finish()?;
run_rustfmt(Mode::Overwrite)
}
+ "compare" => {
+ let markdown = args.contains("--markdown");
+ let free = args.free()?;
+ let base_result_path = free.get(0).map(String::as_str);
+ let new_result_path = free.get(1).map(String::as_str);
+
+ compare::run(base_result_path, new_result_path, markdown);
+ Ok(())
+ }
// "docgen" => {
// args.finish()?;
// docgen::run();
// Ok(())
// }
"coverage" => {
+ let json = args.contains("--json");
+
let free = args.free()?;
let query = free.get(0).map(String::as_str);
let pool = yastl::ThreadConfig::new().stack_size(8 << 30);
- coverage::run(query, yastl::Pool::with_config(num_cpus::get(), pool));
+ coverage::run(query, yastl::Pool::with_config(num_cpus::get(), pool), json);
Ok(())
}
_ => {
@@ -49,13 +56,18 @@ fn main() -> Result<()> {
cargo xtask
Run custom build command.
USAGE:
- cargo xtask <SUBCOMMAND>
+ cargo xtask <SUBCOMMAND> [option]
SUBCOMMANDS:
format
codegen
syntax
docgen
- coverage"
+ coverage [--json]
+ compare [--markdown]
+OPTIONS
+ --markdown Emits supported output into markdown format. Supported by compare subcommand
+ --json Emits supported output into json format. Supported by coverage subcommand
+ "
);
Ok(())
}

ci: first draft of comapre results (#1779)