feature(xtask): add encoding detection to compare
This adds a simple encoding detection logic to the loading of test results file that detects (and skips over) Unicode Byte Order Mark, and re-encodes content into before handing it to serde_json
MODIFY
xtask/src/compare/results.rs
xtask/src/compare/results.rs
use crate::coverage::files::{Outcome, TestResult, TestResults};
use ascii_table::{AsciiTable, Column};
use colored::Colorize;
+use std::char::decode_utf16;
use std::collections::{HashMap, HashSet};
use std::fs::File;
+use std::io::Read;
use std::path::{Path, PathBuf};
pub fn emit_compare(base: &Path, new: &Path, markdown: bool) {
- let base_results: TestResults =
- serde_json::from_reader(File::open(base).expect("Can't read the file of the base results"))
- .expect("Can't parse the JSON file of the base results");
- let new_results: TestResults =
- serde_json::from_reader(File::open(new).expect("Can't read the file of the new results"))
- .expect("Can't parse the JSON file of the new results");
+ let base_results = read_test_results(base, "base");
+ let new_results = read_test_results(new, "new");
let base_total = base_results.summary.tests_ran as isize;
let new_total = new_results.summary.tests_ran as isize;
@@ -195,6 +193,74 @@ pub fn emit_compare(base: &Path, new: &Path, markdown: bool) {
}
}
+fn read_test_results(path: &Path, name: &'static str) -> TestResults {
+ let mut file = File::open(path)
+ .unwrap_or_else(|err| panic!("Can't read the file of the {} results: {:?}", name, err));
+
+ let mut buffer = Vec::new();
+ file.read_to_end(&mut buffer)
+ .unwrap_or_else(|err| panic!("Can't read the file of the {} results: {:?}", name, err));
+
+ enum FileEncoding {
+ Unknown,
+ Utf8,
+ Utf16Le,
+ Utf16Be,
+ }
+
+ let mut encoding = FileEncoding::Unknown;
+ let mut content: &[u8] = &buffer;
+
+ // Read the BOM if present and skip it
+ let bom = content.get(0..3);
+ if let Some(&[0xef, 0xbb, 0xbf]) = bom {
+ content = &content[3..];
+ encoding = FileEncoding::Utf8;
+ } else if let Some(&[0xfe, 0xff, _]) = bom {
+ content = &content[2..];
+ encoding = FileEncoding::Utf16Be;
+ } else if let Some(&[0xff, 0xfe, _]) = bom {
+ content = &content[2..];
+ encoding = FileEncoding::Utf16Le;
+ }
+
+ if matches!(encoding, FileEncoding::Unknown | FileEncoding::Utf8) {
+ // Attempt to parse as UTF-8
+ let result = serde_json::from_slice(content);
+
+ if let FileEncoding::Utf8 = encoding {
+ // If the file is known to be UTF-8 unwrap the result
+ return result.unwrap_or_else(|err| {
+ panic!(
+ "Can't parse the JSON file of the {} results: {:?}",
+ name, err
+ )
+ });
+ } else if let Ok(result) = result {
+ // Otherwise only return if the parsing was successful
+ return result;
+ }
+ }
+
+ // If a UTF-16 BOM was found or an error was encountered, attempt to parse as UTF-16
+ let content_str = decode_utf16(content.chunks(2).map(|bytes| match encoding {
+ FileEncoding::Utf16Be => u16::from_be_bytes([bytes[0], bytes[1]]),
+ FileEncoding::Utf16Le => u16::from_le_bytes([bytes[0], bytes[1]]),
+ // If the encoding is unknown attempt to decode in native endianness
+ FileEncoding::Unknown => u16::from_ne_bytes([bytes[0], bytes[1]]),
+ FileEncoding::Utf8 => unreachable!(),
+ }))
+ .collect::<Result<String, _>>()
+ .unwrap_or_else(|err| panic!("Can't read the file of the {} results: {:?}", name, err));
+
+ serde_json::from_str(&content_str).unwrap_or_else(|err| {
+ panic!(
+ "Can't parse the JSON file of the {} results: {:?}",
+ name, err
+ )
+ })
+}
+
struct ReportDiff<'a> {
pub regression: Vec<&'a TestResult>,
pub fixed: Vec<&'a TestResult>,

feature(xtask): add encoding detection to compare (#1907)

This adds a simple encoding detection logic to the loading of test results file that detects (and skips over) Unicode Byte Order Mark, and re-encodes UTF-16 content into UTF-8 before handing it to serde_json