diff --git a/lib/spectests/tests/excludes.txt b/lib/spectests/tests/excludes.txt index 628027693..45310f2e8 100644 --- a/lib/spectests/tests/excludes.txt +++ b/lib/spectests/tests/excludes.txt @@ -12,10 +12,12 @@ # Star line allows skipping an entire wast file # clif:skip:simd.wast:* # -# Excludes can also contain platform +# Excludes can also contain target family # clif:skip:data.wast:172:windows # clif:skip:data.wast:172:unix # +# Or target arch +# singlepass:skip:atomic.wast:*:*:aarch64 # Cranelift clif:skip:atomic.wast:* # Threads not implemented diff --git a/lib/spectests/tests/spectest.rs b/lib/spectests/tests/spectest.rs index 299971ee1..9bb6f631c 100644 --- a/lib/spectests/tests/spectest.rs +++ b/lib/spectests/tests/spectest.rs @@ -18,6 +18,7 @@ mod tests { // TODO Files could be run with multiple threads // TODO Allow running WAST &str directly (E.g. for use outside of spectests) + use std::collections::HashSet; use std::sync::{Arc, Mutex}; struct SpecFailure { @@ -46,15 +47,14 @@ mod tests { pub fn add_failure( &mut self, failure: SpecFailure, - testkey: &str, - excludes: &HashMap, + _testkey: &str, + excludes: &Vec, + line: u64, ) { - if excludes.contains_key(testkey) { - self.allowed_failure += 1; - return; - } - let platform_key = format!("{}:{}", testkey, get_platform()); - if excludes.contains_key(&platform_key) { + if excludes + .iter() + .any(|e| e.line_matches(line) && e.exclude_kind == ExcludeKind::Fail) + { self.allowed_failure += 1; return; } @@ -104,15 +104,113 @@ mod tests { } #[cfg(unix)] - fn get_platform() -> &'static str { + fn get_target_family() -> &'static str { "unix" } #[cfg(windows)] - fn get_platform() -> &'static str { + fn get_target_family() -> &'static str { "windows" } + fn get_target_arch() -> &'static str { + if cfg!(target_arch = "x86_64") { + "x86_64" + } else if cfg!(target_arch = "aarch64") { + "aarch64" + } else if cfg!(target_arch = "x86") { + "x86" + } else if cfg!(target_arch = "mips") { + "mips" + } else if cfg!(target_arch = "powerpc") { + "powerpc" + } else if cfg!(target_arch = "powerpc64") { + "powerpc64" + } else if cfg!(target_arch = "arm") { + "arm" + } else { + panic!("unknown target arch") + } + } + + // clif:skip:data.wast:172:unix:x86 + #[allow(dead_code)] + struct Exclude { + backend: Option, + exclude_kind: ExcludeKind, + file: String, + line: Option, + target_family: Option, + target_arch: Option, + } + + impl Exclude { + fn line_matches(&self, value: u64) -> bool { + self.line.is_none() || self.line.unwrap() == value + } + + fn line_exact_match(&self, value: u64) -> bool { + self.line.is_some() && self.line.unwrap() == value + } + + fn matches_backend(&self, value: &str) -> bool { + self.backend.is_none() || self.backend.as_ref().unwrap() == value + } + + fn matches_target_family(&self, value: &str) -> bool { + self.target_family.is_none() || self.target_family.as_ref().unwrap() == value + } + + fn matches_target_arch(&self, value: &str) -> bool { + self.target_arch.is_none() || self.target_arch.as_ref().unwrap() == value + } + + fn from( + backend: &str, + exclude_kind: &str, + file: &str, + line: &str, + target_family: &str, + target_arch: &str, + ) -> Exclude { + let backend: Option = match backend { + "*" => None, + "clif" => Some("clif".to_string()), + "singlepass" => Some("singlepass".to_string()), + "llvm" => Some("llvm".to_string()), + _ => panic!("backend {:?} not recognized", backend), + }; + let exclude_kind = match exclude_kind { + "skip" => ExcludeKind::Skip, + "fail" => ExcludeKind::Fail, + _ => panic!("exclude kind {:?} not recognized", exclude_kind), + }; + let line = match line { + "*" => None, + _ => Some( + line.parse::() + .expect(&format!("expected * or int: {:?}", line)), + ), + }; + let target_family = match target_family { + "*" => None, + _ => Some(target_family.to_string()), + }; + let target_arch = match target_arch { + "*" => None, + _ => Some(target_arch.to_string()), + }; + Exclude { + backend, + exclude_kind, + file: file.to_string(), + line, + target_family, + target_arch, + } + } + } + #[cfg(not(any(feature = "llvm", feature = "clif", feature = "singlepass")))] fn get_compiler_name() -> &'static str { panic!("compiler not specified, activate a compiler via features"); @@ -160,7 +258,8 @@ mod tests { fn parse_and_run( path: &PathBuf, - excludes: &HashMap, + file_excludes: &HashSet, + excludes: &HashMap>, ) -> Result { let mut test_report = TestReport { failures: vec![], @@ -171,15 +270,9 @@ mod tests { let filename = path.file_name().unwrap().to_str().unwrap(); let source = fs::read(&path).unwrap(); - let backend = get_compiler_name(); - let platform = get_platform(); - let star_key = format!("{}:{}:*", backend, filename); - let platform_star_key = format!("{}:{}:*:{}", backend, filename, platform); - if (excludes.contains_key(&star_key) && *excludes.get(&star_key).unwrap() == Exclude::Skip) - || (excludes.contains_key(&platform_star_key) - && *excludes.get(&platform_star_key).unwrap() == Exclude::Skip) - { + // Entire file is excluded by line * and skip + if file_excludes.contains(filename) { return Ok(test_report); } @@ -198,21 +291,27 @@ mod tests { let mut registered_modules: HashMap>> = HashMap::new(); // + let empty_excludes = vec![]; + let excludes = if excludes.contains_key(filename) { + excludes.get(filename).unwrap() + } else { + &empty_excludes + }; + + let backend = get_compiler_name(); while let Some(Command { kind, line }) = parser.next().map_err(|e| format!("Parse err: {:?}", e))? { let test_key = format!("{}:{}:{}", backend, filename, line); - let test_platform_key = format!("{}:{}:{}:{}", backend, filename, line, platform); // Use this line to debug which test is running println!("Running test: {}", test_key); - if (excludes.contains_key(&test_key) - && *excludes.get(&test_key).unwrap() == Exclude::Skip) - || (excludes.contains_key(&test_platform_key) - && *excludes.get(&test_platform_key).unwrap() == Exclude::Skip) + // Skip tests that match this line + if excludes + .iter() + .any(|e| e.line_exact_match(line) && e.exclude_kind == ExcludeKind::Skip) { - // println!("Skipping test: {}", test_key); continue; } @@ -251,6 +350,7 @@ mod tests { }, &test_key, excludes, + line, ); instance = None; } @@ -290,6 +390,7 @@ mod tests { }, &test_key, excludes, + line, ); } else { let call_result = maybe_call_result.unwrap(); @@ -304,6 +405,7 @@ mod tests { }, &test_key, excludes, + line, ); } Ok(values) => { @@ -320,7 +422,7 @@ mod tests { "result {:?} ({:?}) does not match expected {:?} ({:?})", v, to_hex(v.clone()), expected_value, to_hex(expected_value.clone()) ), - }, &test_key, excludes); + }, &test_key, excludes, line); } else { test_report.count_passed(); } @@ -350,6 +452,7 @@ mod tests { }, &test_key, excludes, + line, ); } else { let export: Export = maybe_call_result.unwrap(); @@ -373,6 +476,7 @@ mod tests { }, &test_key, excludes, + line, ); } } @@ -386,6 +490,7 @@ mod tests { }, &test_key, excludes, + line, ); } } @@ -416,6 +521,7 @@ mod tests { }, &test_key, excludes, + line, ); } else { let call_result = maybe_call_result.unwrap(); @@ -430,6 +536,7 @@ mod tests { }, &test_key, excludes, + line, ); } Ok(values) => { @@ -453,6 +560,7 @@ mod tests { }, &test_key, excludes, + line, ); } } @@ -484,6 +592,7 @@ mod tests { }, &test_key, excludes, + line, ); } else { let call_result = maybe_call_result.unwrap(); @@ -498,6 +607,7 @@ mod tests { }, &test_key, excludes, + line, ); } Ok(values) => { @@ -521,6 +631,7 @@ mod tests { }, &test_key, excludes, + line, ); } } @@ -552,6 +663,7 @@ mod tests { }, &test_key, excludes, + line, ); } else { let call_result = maybe_call_result.unwrap(); @@ -569,6 +681,7 @@ mod tests { }, &test_key, excludes, + line, ); } CallError::Runtime(r) => { @@ -590,6 +703,7 @@ mod tests { }, &test_key, excludes, + line, ); } } @@ -606,6 +720,7 @@ mod tests { }, &test_key, excludes, + line, ); } } @@ -649,6 +764,7 @@ mod tests { }, &test_key, excludes, + line, ); } } @@ -662,6 +778,7 @@ mod tests { }, &test_key, excludes, + line, ); } } @@ -704,6 +821,7 @@ mod tests { }, &test_key, excludes, + line, ); } } @@ -717,6 +835,7 @@ mod tests { }, &test_key, excludes, + line, ); } } @@ -755,6 +874,7 @@ mod tests { }, &test_key, excludes, + line, ); } }; @@ -786,6 +906,7 @@ mod tests { }, &test_key, excludes, + line, ); } else { let call_result = maybe_call_result.unwrap(); @@ -807,6 +928,7 @@ mod tests { }, &test_key, excludes, + line, ); } } @@ -845,6 +967,7 @@ mod tests { }, &test_key, excludes, + line, ); } Ok(result) => match result { @@ -860,6 +983,7 @@ mod tests { }, &test_key, excludes, + line, ); } Err(e) => match e { @@ -876,6 +1000,7 @@ mod tests { }, &test_key, excludes, + line, ); } }, @@ -909,6 +1034,7 @@ mod tests { }, &test_key, excludes, + line, ); } } @@ -934,6 +1060,7 @@ mod tests { }, &test_key, excludes, + line, ); } else { let call_result = maybe_call_result.unwrap(); @@ -948,6 +1075,7 @@ mod tests { }, &test_key, excludes, + line, ); } Ok(_values) => { @@ -1105,7 +1233,7 @@ mod tests { } #[derive(Debug, Copy, Clone, PartialEq, Eq)] - enum Exclude { + enum ExcludeKind { Skip, Fail, } @@ -1115,13 +1243,18 @@ mod tests { use std::io::{BufRead, BufReader}; /// Reads the excludes.txt file into a hash map - fn read_excludes() -> HashMap { + fn read_excludes() -> (HashMap>, HashSet) { let mut excludes_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); excludes_path.push("tests"); excludes_path.push("excludes.txt"); let input = File::open(excludes_path).unwrap(); let buffered = BufReader::new(input); let mut result = HashMap::new(); + let mut file_excludes = HashSet::new(); + let current_backend = get_compiler_name(); + let current_target_family = get_target_family(); + let current_target_arch = get_target_arch(); + for line in buffered.lines() { let mut line = line.unwrap(); if line.trim().is_empty() || line.starts_with("#") { @@ -1136,26 +1269,53 @@ mod tests { // ::: let split: Vec<&str> = line.trim().split(':').collect(); - let kind = match *split.get(1).unwrap() { - "skip" => Exclude::Skip, - "fail" => Exclude::Fail, - _ => panic!("unknown exclude kind"), + let file = *split.get(2).unwrap(); + let exclude = match split.len() { + 0..=3 => panic!("expected at least 4 exclude conditions"), + 4 => Exclude::from( + *split.get(0).unwrap(), + *split.get(1).unwrap(), + *split.get(2).unwrap(), + *split.get(3).unwrap(), + "*", + "*", + ), + 5 => Exclude::from( + *split.get(0).unwrap(), + *split.get(1).unwrap(), + *split.get(2).unwrap(), + *split.get(3).unwrap(), + *split.get(4).unwrap(), + "*", + ), + 6 => Exclude::from( + *split.get(0).unwrap(), + *split.get(1).unwrap(), + *split.get(2).unwrap(), + *split.get(3).unwrap(), + *split.get(4).unwrap(), + *split.get(5).unwrap(), + ), + _ => panic!("too many exclude conditions {}", split.len()), }; - let has_platform = split.len() > 4; - let backend = split.get(0).unwrap(); - let testfile = split.get(2).unwrap(); - let line = split.get(3).unwrap(); - let key = if has_platform { - let platform = split.get(4).unwrap(); - format!("{}:{}:{}:{}", backend, testfile, line, platform) - } else { - format!("{}:{}:{}", backend, testfile, line) - }; - result.insert(key, kind); + if exclude.matches_backend(current_backend) + && exclude.matches_target_family(current_target_family) + && exclude.matches_target_arch(current_target_arch) + { + // Skip the whole file for line * and skip + if exclude.line.is_none() && exclude.exclude_kind == ExcludeKind::Skip { + file_excludes.insert(file.to_string()); + } + + if !result.contains_key(file) { + result.insert(file.to_string(), vec![]); + } + result.get_mut(file).unwrap().push(exclude); + } } } - result + (result, file_excludes) } #[test] @@ -1163,7 +1323,7 @@ mod tests { let mut success = true; let mut test_reports = vec![]; - let excludes = read_excludes(); + let (excludes, file_excludes) = read_excludes(); let mut glob_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); glob_path.push("spectests"); @@ -1173,7 +1333,7 @@ mod tests { for entry in glob(glob_str).expect("Failed to read glob pattern") { match entry { Ok(wast_path) => { - let result = parse_and_run(&wast_path, &excludes); + let result = parse_and_run(&wast_path, &file_excludes, &excludes); match result { Ok(test_report) => { if test_report.has_failures() {