CI: Typescript Coverage
Changes the `cargo coverage` command to run all test suites if no `--language` filter is provided.
Extends the JSON output to include the results for each test suite and extends the comparison to print the results by test suite.
RENAME
.github/workflows/test262.yml
.github/workflows/parser_conformance.yml
# Test coverage job. It is run on pull request because it prints the results via comment
-name: Test262 coverage and comparison
+name: Parser conformance and comparison
on:
pull_request:
branches:
@@ -11,7 +11,7 @@ env:
jobs:
coverage:
- name: Test262 coverage
+ name: Parser conformance
runs-on: ${{ matrix.os }}
strategy:
@@ -47,21 +47,21 @@ jobs:
- name: Run Test262 suite
continue-on-error: true
- run: cargo coverage js --json > new_results.json
+ run: cargo coverage --json > new_results.json
- name: Checkout Main Branch
run: git checkout main
- name: Run Test262 suite on main branch
continue-on-error: true
- run: cargo coverage js --json > base_results.json
+ run: cargo coverage --json > base_results.json
- name: Compare results on ${{ matrix.os }}
if: github.event_name == 'pull_request'
id: comparison
shell: bash
run: |
- echo "### Test262 comparison coverage results on ${{ matrix.os }}" > output
+ echo "## Parser conformance results on ${{ matrix.os }}" > output
cargo coverage compare ./base_results.json ./new_results.json --markdown >> output
cat output
comment="$(cat output)"
@@ -81,7 +81,7 @@ jobs:
id: previous-comment
with:
issue-number: ${{ steps.pr-number.outputs.pr }}
- body-includes: Test262 comparison coverage results on ${{ matrix.os }}
+ body-includes: Parser conformance results on ${{ matrix.os }}
- name: Update existing comment
if: github.event_name == 'pull_request' && steps.previous-comment.outputs.comment-id

MODIFY
CONTRIBUTING.md
CONTRIBUTING.md
@@ -60,7 +60,7 @@ It's strongly advised to **run this command before committing new changes**.
#### `cargo coverage`
-This command will check and report parser coverage against different test suites.
+This command will check and report parser conformance against different test suites.
We currently target the [Official ECMAScript Conformance Test Suite](https://github.com/tc39/test262) and
the [Typescript Test Suite](https://github.com/microsoft/TypeScript/tree/main/tests)

MODIFY
xtask/coverage/src/compare.rs
xtask/coverage/src/compare.rs
-use std::path::PathBuf;
+use std::char::decode_utf16;
+use std::collections::HashMap;
+use std::fs::File;
+use std::io::Read;
+use std::path::{Path, PathBuf};
use xtask::project_root;
use crate::results::emit_compare;
+use crate::TestResults;
// this is the filename of the results coming from `main` branch
const BASE_RESULT_FILE: &str = "base_results.json";
@@ -42,9 +47,84 @@ pub fn coverage_compare(
);
}
- emit_compare(
- base_result_dir.as_path(),
- new_result_dir.as_path(),
- markdown,
- );
+ let base_results = read_test_results(base_result_dir.as_path(), "base");
+ let mut new_results = read_test_results(new_result_dir.as_path(), "new");
+
+ for (suite, base) in base_results.into_iter() {
+ let new_result = new_results.remove(&suite).unwrap_or_else(TestResults::new);
+
+ emit_compare(&base, &new_result, suite.as_str(), markdown);
+ }
+
+ for (suite, new) in new_results.drain() {
+ emit_compare(&TestResults::new(), &new, suite.as_str(), markdown);
+ }
+}
+
+fn read_test_results(path: &Path, name: &'static str) -> HashMap<String, TestResults> {
+ let mut file = File::open(path)
+ .unwrap_or_else(|err| panic!("Can't read the file of the {} results: {:?}", name, err));
+
+ let mut buffer = Vec::new();
+ file.read_to_end(&mut buffer)
+ .unwrap_or_else(|err| panic!("Can't read the file of the {} results: {:?}", name, err));
+
+ enum FileEncoding {
+ Unknown,
+ Utf8,
+ Utf16Le,
+ Utf16Be,
+ }
+
+ let mut encoding = FileEncoding::Unknown;
+ let mut content: &[u8] = &buffer;
+
+ // Read the BOM if present and skip it
+ let bom = content.get(0..3);
+ if let Some(&[0xef, 0xbb, 0xbf]) = bom {
+ content = &content[3..];
+ encoding = FileEncoding::Utf8;
+ } else if let Some(&[0xfe, 0xff, _]) = bom {
+ content = &content[2..];
+ encoding = FileEncoding::Utf16Be;
+ } else if let Some(&[0xff, 0xfe, _]) = bom {
+ content = &content[2..];
+ encoding = FileEncoding::Utf16Le;
+ }
+
+ if matches!(encoding, FileEncoding::Unknown | FileEncoding::Utf8) {
+ // Attempt to parse as UTF-8
+ let result = serde_json::from_slice(content);
+
+ if let FileEncoding::Utf8 = encoding {
+ // If the file is known to be UTF-8 unwrap the result
+ return result.unwrap_or_else(|err| {
+ panic!(
+ "Can't parse the JSON file of the {} results: {:?}",
+ name, err
+ )
+ });
+ } else if let Ok(result) = result {
+ // Otherwise only return if the parsing was successful
+ return result;
+ }
+ }
+
+ // If a UTF-16 BOM was found or an error was encountered, attempt to parse as UTF-16
+ let content_str = decode_utf16(content.chunks(2).map(|bytes| match encoding {
+ FileEncoding::Utf16Be => u16::from_be_bytes([bytes[0], bytes[1]]),
+ FileEncoding::Utf16Le => u16::from_le_bytes([bytes[0], bytes[1]]),
+ // If the encoding is unknown attempt to decode in native endianness
+ FileEncoding::Unknown => u16::from_ne_bytes([bytes[0], bytes[1]]),
+ FileEncoding::Utf8 => unreachable!(),
+ }))
+ .collect::<Result<String, _>>()
+ .unwrap_or_else(|err| panic!("Can't read the file of the {} results: {:?}", name, err));
+
+ serde_json::from_str(&content_str).unwrap_or_else(|err| {
+ panic!(
+ "Can't parse the JSON file of the {} results: {:?}",
+ name, err
+ )
+ })
}

MODIFY
xtask/coverage/src/lib.rs
xtask/coverage/src/lib.rs
@@ -7,7 +7,7 @@ pub mod typescript;
use crate::reporters::{
CliProgressReporter, DiagnosticsReporter, JsonReporter, MulticastTestReporter, RastReporter,
- SummaryReporter,
+ SummaryReporter, TestReporter,
};
use crate::runner::{run_test_suite, TestRunContext, TestSuite};
use crate::test262::Test262TestSuite;
@@ -40,7 +40,7 @@ pub struct TestResults {
pub details: Vec<TestResult>,
}
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct Summary {
#[serde(rename = "a")]
pub tests_ran: u32,
@@ -111,23 +111,16 @@ impl TestResults {
}
pub fn run(
- language: &str,
+ language: Option<&str>,
query: Option<&str>,
json: bool,
show_rast: bool,
show_diagnostics: bool,
) {
- let language = language.to_lowercase();
- let loader: Box<dyn TestSuite> = match language.as_str() {
- "javascript" | "js" => Box::new(Test262TestSuite),
- "typescript" | "ts" => Box::new(TypeScriptTestSuite),
- other => panic!("Unknown language: {}", other),
- };
-
let mut reporters = MulticastTestReporter::new(Box::new(CliProgressReporter::default()));
if json {
- reporters.add(Box::new(JsonReporter));
+ reporters.add(Box::new(JsonReporter::default()));
} else {
reporters.add(Box::new(SummaryReporter::default()));
}
@@ -140,7 +133,7 @@ pub fn run(
reporters.add(Box::new(DiagnosticsReporter));
}
- let context = TestRunContext {
+ let mut context = TestRunContext {
query: query.map(|s| s.to_string()),
reporter: &mut reporters,
pool: &yastl::Pool::with_config(
@@ -149,9 +142,29 @@ pub fn run(
),
};
- let results = run_test_suite(loader.as_ref(), context);
+ let mut ran_any_tests = false;
+ for test_suite in get_test_suites(language) {
+ let result = run_test_suite(test_suite.as_ref(), &mut context);
+ ran_any_tests = ran_any_tests || result.summary.tests_ran > 0
+ }
+
+ reporters.run_completed();
- if results.passed_tests() == 0 {
+ if !ran_any_tests {
std::process::exit(1);
}
}
+
+fn get_test_suites(language: Option<&str>) -> Vec<Box<dyn TestSuite>> {
+ language
+ .map(|language| {
+ let test_suite: Box<dyn TestSuite> = match language.to_lowercase().as_str() {
+ "js" | "javascript" => Box::new(Test262TestSuite),
+ "ts" | "typescript" => Box::new(TypeScriptTestSuite),
+ other => panic!("Unknown language: {}", other),
+ };
+
+ vec![test_suite]
+ })
+ .unwrap_or_else(|| vec![Box::new(Test262TestSuite), Box::new(TypeScriptTestSuite)])
+}

MODIFY
xtask/coverage/src/main.rs
xtask/coverage/src/main.rs
@@ -7,10 +7,9 @@ fn main() -> Result<()> {
let _d = pushd(project_root());
let mut args = Arguments::from_env();
- let command = args.subcommand()?.unwrap_or_default();
+ let sub_command = args.subcommand()?;
- match command.as_str() {
- "compare" => {
+ if sub_command.as_deref() == Some("compare") {
// on pr branch, run
// git checkout main
// cargo coverage js --json > base_results.json
@@ -22,16 +21,10 @@ fn main() -> Result<()> {
let base_result_path = free.get(0).map(String::as_str);
let new_result_path = free.get(1).map(String::as_str);
coverage_compare(base_result_path, new_result_path, markdown);
+ return Ok(());
}
- language @ ("js" | "javascript" | "ts" | "typescript") => {
- let json = args.contains("--json");
- let show_rast = args.contains("--show-rast");
- let show_diagnostics = args.contains("--show-diagnostics");
- let free = args.free()?;
- let query = free.get(0).map(String::as_str);
- run(language, query, json, show_rast, show_diagnostics);
- }
- _ => {
+
+ if args.contains("--help") {
eprintln!(
"\
cargo coverage
@@ -39,16 +32,30 @@ Run coverage command.
USAGE:
cargo coverage <SUBCOMMAND> [option]
SUBCOMMANDS:
- js Runs test262 tests
- ts Runs TypeScript tests
compare Compares output between two --json outputs
OPTIONS
--markdown Emits supported output into markdown format. Supported by compare subcommand
--json Emits supported output into json format. Supported by js subcommand
+ --language=[js|ts] Runs a specific test suite
+ --help Prints this help
"
);
+ return Ok(());
}
- }
+
+ let json = args.contains("--json");
+ let show_rast = args.contains("--show-rast");
+ let show_diagnostics = args.contains("--show-diagnostics");
+ let language: Option<String> = args.opt_value_from_str("--language").unwrap();
+ let free = args.free()?;
+ let query = free.get(0).map(String::as_str);
+ run(
+ language.as_deref(),
+ query,
+ json,
+ show_rast,
+ show_diagnostics,
+ );
Ok(())
}

MODIFY
xtask/coverage/src/reporters.rs
xtask/coverage/src/reporters.rs
use crate::runner::{TestRunOutcome, TestRunResult, TestSuite, TestSuiteInstance};
-use crate::TestResults;
+use crate::{Summary, TestResults};
use ascii_table::{AsciiTable, Column};
use colored::Colorize;
use indicatif::{ProgressBar, ProgressDrawTarget};
use rslint_errors::file::SimpleFile;
use rslint_errors::Emitter;
use rslint_parser::parse;
+use serde_json::Value;
+use std::collections::HashMap;
use std::time::Instant;
pub(crate) trait TestReporter: Send + Sync {
@@ -21,6 +23,9 @@ pub(crate) trait TestReporter: Send + Sync {
fn test_completed(&mut self, _result: &TestRunResult) {}
/// A test suite completed
fn test_suite_completed(&mut self, _suite: &TestSuiteInstance, _result: &TestResults) {}
+
+ /// Called when all test suites have completed
+ fn run_completed(&mut self) {}
}
pub(crate) struct CliProgressReporter {
@@ -166,12 +171,14 @@ fn print_detailed_test_result(pb: &ProgressBar, result: &TestRunResult) {
/// Reporter that prints a summary for each phase (tests loaded, test suite completed) to the console output
pub(crate) struct SummaryReporter {
start: Instant,
+ results: HashMap<String, Summary>,
}
impl Default for SummaryReporter {
fn default() -> Self {
Self {
start: Instant::now(),
+ results: HashMap::default(),
}
}
}
@@ -201,43 +208,86 @@ impl TestReporter for SummaryReporter {
self.start.elapsed().as_secs_f32()
);
- let panicked = results.summary.panics;
- let errored = results.summary.failed;
- let passed = results.summary.passed;
- let coverage = format!("{:.2}", results.summary.coverage);
-
- let total = panicked + errored + passed;
+ self.results
+ .insert(suite.name().to_string(), results.summary.clone());
+ }
+ fn run_completed(&mut self) {
let mut table = AsciiTable::default();
+ let results = std::mem::take(&mut self.results);
+ let has_multiple_test_suites = results.len() > 1;
+
+ if has_multiple_test_suites {
+ table.columns.insert(
+ 0,
+ Column {
+ header: "Test suite".into(),
+ align: ascii_table::Align::Left,
+ ..Column::default()
+ },
+ );
+ }
- let mut counter = 0usize;
- let mut create_column = |name: colored::ColoredString| {
+ let mut create_number_column = |name: colored::ColoredString| {
let column = Column {
header: name.to_string(),
- align: ascii_table::Align::Center,
+ align: ascii_table::Align::Right,
..Column::default()
};
- table.columns.insert(counter, column);
- counter += 1;
+ table.columns.insert(table.columns.len(), column);
};
- create_column("Tests ran".into());
- create_column("Passed".green());
- create_column("Failed".red());
- create_column("Panics".red());
- create_column("Coverage".cyan());
- let numbers: Vec<&dyn std::fmt::Display> =
- vec![&total, &passed, &errored, &panicked, &coverage];
- table.print(vec![numbers]);
+ create_number_column("Tests ran".into());
+ create_number_column("Passed".green());
+ create_number_column("Failed".red());
+ create_number_column("Panics".red());
+ create_number_column("Coverage".cyan());
+
+ let rows = results.into_iter().map(|(suite, summary)| {
+ let panicked = summary.panics;
+ let errored = summary.failed;
+ let passed = summary.passed;
+ let coverage = format!("{:.2}", summary.coverage);
+
+ let total = panicked + errored + passed;
+
+ let mut values = if has_multiple_test_suites {
+ vec![suite]
+ } else {
+ Vec::default()
+ };
+
+ values.extend([
+ total.to_string(),
+ passed.to_string(),
+ errored.to_string(),
+ panicked.to_string(),
+ coverage,
+ ]);
+
+ values
+ });
+
+ table.print(rows);
}
}
-pub(crate) struct JsonReporter;
+#[derive(Default)]
+pub(crate) struct JsonReporter {
+ results: HashMap<String, Value>,
+}
impl TestReporter for JsonReporter {
- fn test_suite_completed(&mut self, _suite: &TestSuiteInstance, result: &TestResults) {
- let json = serde_json::to_string(result).unwrap();
- println!("{}", json);
+ fn test_suite_completed(&mut self, suite: &TestSuiteInstance, result: &TestResults) {
+ self.results.insert(
+ suite.name().to_string(),
+ serde_json::to_value(result).unwrap(),
+ );
+ }
+
+ fn run_completed(&mut self) {
+ let results = std::mem::take(&mut self.results);
+ println!("{}", serde_json::to_string(&results).unwrap());
}
}
@@ -291,6 +341,12 @@ impl TestReporter for MulticastTestReporter {
reporter.test_suite_completed(suite, result);
}
}
+
+ fn run_completed(&mut self) {
+ for reporter in &mut self.0 {
+ reporter.run_completed();
+ }
+ }
}
/// Prints the rast output for all (not panicking) tests to the console

MODIFY
xtask/coverage/src/results.rs
xtask/coverage/src/results.rs
use ascii_table::{AsciiTable, Column};
use colored::Colorize;
-use std::char::decode_utf16;
use std::collections::{HashMap, HashSet};
-use std::fs::File;
-use std::io::Read;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
use crate::{Outcome, TestResult, TestResults};
-pub fn emit_compare(base: &Path, new: &Path, markdown: bool) {
- let base_results = read_test_results(base, "base");
- let new_results = read_test_results(new, "new");
-
+pub fn emit_compare(
+ base_results: &TestResults,
+ new_results: &TestResults,
+ test_suite: &str,
+ markdown: bool,
+) {
let base_total = base_results.summary.tests_ran as isize;
let new_total = new_results.summary.tests_ran as isize;
let total_diff = new_total - base_total;
@@ -32,7 +31,7 @@ pub fn emit_compare(base: &Path, new: &Path, markdown: bool) {
let new_coverage = new_results.summary.coverage;
let coverage_diff = new_coverage - base_coverage;
- let report_diff = compare_diffs(&base_results, &new_results);
+ let report_diff = compare_diffs(base_results, new_results);
if markdown {
/// Generates a proper diff format, with some bold text if things change.
@@ -74,6 +73,8 @@ pub fn emit_compare(base: &Path, new: &Path, markdown: bool) {
)
}
+ println!("\n### {}\n", test_suite);
+
println!("| Test result | `main` count | This PR count | Difference |");
println!("| :---------: | :----------: | :-----------: | :--------: |");
@@ -174,7 +175,7 @@ pub fn emit_compare(base: &Path, new: &Path, markdown: bool) {
counter += 1;
};
- println!("Test262 conformance changes:");
+ println!("{} conformance changes:", test_suite);
create_column("Tests result".into());
create_column("main branch".green());
@@ -194,74 +195,6 @@ pub fn emit_compare(base: &Path, new: &Path, markdown: bool) {
}
}
-fn read_test_results(path: &Path, name: &'static str) -> TestResults {
- let mut file = File::open(path)
- .unwrap_or_else(|err| panic!("Can't read the file of the {} results: {:?}", name, err));
-
- let mut buffer = Vec::new();
- file.read_to_end(&mut buffer)
- .unwrap_or_else(|err| panic!("Can't read the file of the {} results: {:?}", name, err));
-
- enum FileEncoding {
- Unknown,
- Utf8,
- Utf16Le,
- Utf16Be,
- }
-
- let mut encoding = FileEncoding::Unknown;
- let mut content: &[u8] = &buffer;
-
- // Read the BOM if present and skip it
- let bom = content.get(0..3);
- if let Some(&[0xef, 0xbb, 0xbf]) = bom {
- content = &content[3..];
- encoding = FileEncoding::Utf8;
- } else if let Some(&[0xfe, 0xff, _]) = bom {
- content = &content[2..];
- encoding = FileEncoding::Utf16Be;
- } else if let Some(&[0xff, 0xfe, _]) = bom {
- content = &content[2..];
- encoding = FileEncoding::Utf16Le;
- }
-
- if matches!(encoding, FileEncoding::Unknown | FileEncoding::Utf8) {
- // Attempt to parse as UTF-8
- let result = serde_json::from_slice(content);
-
- if let FileEncoding::Utf8 = encoding {
- // If the file is known to be UTF-8 unwrap the result
- return result.unwrap_or_else(|err| {
- panic!(
- "Can't parse the JSON file of the {} results: {:?}",
- name, err
- )
- });
- } else if let Ok(result) = result {
- // Otherwise only return if the parsing was successful
- return result;
- }
- }
-
- // If a UTF-16 BOM was found or an error was encountered, attempt to parse as UTF-16
- let content_str = decode_utf16(content.chunks(2).map(|bytes| match encoding {
- FileEncoding::Utf16Be => u16::from_be_bytes([bytes[0], bytes[1]]),
- FileEncoding::Utf16Le => u16::from_le_bytes([bytes[0], bytes[1]]),
- // If the encoding is unknown attempt to decode in native endianness
- FileEncoding::Unknown => u16::from_ne_bytes([bytes[0], bytes[1]]),
- FileEncoding::Utf8 => unreachable!(),
- }))
- .collect::<Result<String, _>>()
- .unwrap_or_else(|err| panic!("Can't read the file of the {} results: {:?}", name, err));
-
- serde_json::from_str(&content_str).unwrap_or_else(|err| {
- panic!(
- "Can't parse the JSON file of the {} results: {:?}",
- name, err
- )
- })
-}
-
struct ReportDiff<'a> {
pub regression: Vec<&'a TestResult>,
pub fixed: Vec<&'a TestResult>,

MODIFY
xtask/coverage/src/runner.rs
xtask/coverage/src/runner.rs
@@ -99,10 +99,10 @@ pub(crate) struct TestRunContext<'a> {
pub(crate) fn run_test_suite(
test_suite: &dyn TestSuite,
- mut context: TestRunContext,
+ context: &mut TestRunContext,
) -> TestResults {
context.reporter.test_suite_started(test_suite);
- let instance = load_tests(test_suite, &mut context);
+ let instance = load_tests(test_suite, context);
context.reporter.test_suite_run_started(&instance);
std::panic::set_hook(Box::new(|_| {}));

CI: Typescript Coverage (#2020)

Changes the `cargo coverage` command to run all test suites if no `--language` filter is provided.

Extends the JSON output to include the results for each test suite and extends the comparison to print the results by test suite.