diff --git a/.gitignore b/.gitignore index b7e8e8fa1575..e1e73a4b5316 100644 --- a/.gitignore +++ b/.gitignore @@ -76,3 +76,6 @@ __pycache__/ src/test/rustdoc-gui/src/**.lock # Before adding new lines, see the comment at the top. +/src/test/ref +/src/tools/dashboard/Cargo.lock +/src/tools/dashboard/target diff --git a/Cargo.toml b/Cargo.toml index 4c00a7dc99ea..142ca97225cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,7 @@ exclude = [ "src/tools/x", # stdarch has its own Cargo workspace "library/stdarch", + "src/tools/dashboard", ] [profile.release.package.compiler_builtins] diff --git a/scripts/display-dashboard.sh b/scripts/display-dashboard.sh new file mode 100755 index 000000000000..f90d1a65a921 --- /dev/null +++ b/scripts/display-dashboard.sh @@ -0,0 +1,10 @@ +#!/bin/bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 OR MIT + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +RUST_DIR=$SCRIPT_DIR/.. +export PATH=$SCRIPT_DIR:$PATH + +cargo build --manifest-path src/tools/dashboard/Cargo.toml +cargo run --manifest-path src/tools/dashboard/Cargo.toml diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index c71faeb1c9bd..4b4df8557b47 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -461,6 +461,7 @@ impl<'a> Builder<'a> { test::SMACK, test::CargoRMC, test::Expected, + test::Ref, // Run run-make last, since these won't pass without make on Windows test::RunMake, ), diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index 8b2bb5e53b98..27816460396a 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -1203,6 +1203,8 @@ default_test!(CargoRMC { path: "src/test/cargo-rmc", mode: "cargo-rmc", suite: " default_test!(Expected { path: "src/test/expected", mode: "expected", suite: "expected" }); +default_test!(Ref { path: "src/test/ref", mode: "rmc", suite: "ref" }); + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] struct Compiletest { compiler: Compiler, diff --git a/src/tools/dashboard/Cargo.toml b/src/tools/dashboard/Cargo.toml new file mode 100644 index 000000000000..4e4ee120afc2 --- /dev/null +++ b/src/tools/dashboard/Cargo.toml @@ -0,0 +1,10 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 OR MIT + +[package] +name = "dashboard" +version = "0.1.0" +edition = "2018" + +[dependencies] +pulldown-cmark = { version = "0.8.0", default-features = false } diff --git a/src/tools/dashboard/print.sh b/src/tools/dashboard/print.sh new file mode 100755 index 000000000000..f803f1c5b2f4 --- /dev/null +++ b/src/tools/dashboard/print.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 OR MIT + +# `rustdoc` treats this script as `rustc` and sends code extracted from markdown +# files to stdin of this script. Instead of compiling the code, this scripts +# simply copies the contents of stdin to the location where `rustdoc` caches the +# "compiled" output. + +FILE="$6" +BASE=`basename "$FILE"` +mkdir -p "$BASE" +cp "/dev/stdin" "$FILE" diff --git a/src/tools/dashboard/src/dashboard.rs b/src/tools/dashboard/src/dashboard.rs new file mode 100644 index 000000000000..9e6aaf6c72f3 --- /dev/null +++ b/src/tools/dashboard/src/dashboard.rs @@ -0,0 +1,95 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT +//! Data structures representing the dashboard and their utilities. + +use std::fmt::{Display, Formatter, Result, Write}; + +/// This data structure holds the results of running a test or a suite. +#[derive(Clone, Debug)] +pub struct Node { + pub name: String, + pub num_pass: u32, + pub num_fail: u32, +} + +impl Node { + /// Creates a new test [`Node`]. + pub fn new(name: String, num_pass: u32, num_fail: u32) -> Node { + Node { name, num_pass, num_fail } + } +} + +/// Tree data structure representing a confidence dashboard. `children` +/// represent sub-tests and sub-suites of the current test suite. This tree +/// structure allows us to collect and display a summary for test results in an +/// organized manner. +#[derive(Clone, Debug)] +pub struct Tree { + pub data: Node, + pub children: Vec, +} + +impl Tree { + /// Creates a new [`Tree`] representing a dashboard or a part of it. + pub fn new(data: Node, children: Vec) -> Tree { + Tree { data, children } + } + + /// Merges two trees, if their root have equal node names, and returns the + /// merged tree. + pub fn merge(mut l: Tree, r: Tree) -> Option { + if l.data.name != r.data.name { + return None; + } + // For each subtree of `r`... + for cnr in r.children { + // Look for a subtree of `l` with an equal root node name. + let index = l.children.iter().position(|cnl| cnl.data.name == cnr.data.name); + if let Some(index) = index { + // If you find one, merge it with `r`'s subtree. + let cnl = l.children.remove(index); + l.children.insert(index, Tree::merge(cnl, cnr)?); + } else { + // Otherwise, `r`'s subtree is new. So, add it to `l`'s + // list of subtrees. + l.children.push(cnr); + } + } + Some(Tree::new( + Node::new( + l.data.name, + l.data.num_pass + r.data.num_pass, + l.data.num_fail + r.data.num_fail, + ), + l.children, + )) + } + + /// A helper format function that indents each level of the tree. + fn fmt_aux(&self, p: usize, f: &mut Formatter<'_>) -> Result { + // Do not print line numbers. + if self.children.len() == 0 { + return Ok(()); + } + // Write `p` spaces into the formatter. + f.write_fmt(format_args!("{:1$}", "", p))?; + f.write_str(&self.data.name)?; + if self.data.num_pass > 0 { + f.write_fmt(format_args!(" ✔️ {}", self.data.num_pass))?; + } + if self.data.num_fail > 0 { + f.write_fmt(format_args!(" ❌ {}", self.data.num_fail))?; + } + f.write_char('\n')?; + for cn in &self.children { + cn.fmt_aux(p + 2, f)?; + } + Ok(()) + } +} + +impl Display for Tree { + fn fmt(&self, f: &mut Formatter<'_>) -> Result { + self.fmt_aux(0, f) + } +} diff --git a/src/tools/dashboard/src/main.rs b/src/tools/dashboard/src/main.rs new file mode 100644 index 000000000000..a8d8243d03e9 --- /dev/null +++ b/src/tools/dashboard/src/main.rs @@ -0,0 +1,9 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +mod dashboard; +mod reference; + +fn main() { + reference::display_reference_dashboard(); +} diff --git a/src/tools/dashboard/src/reference.rs b/src/tools/dashboard/src/reference.rs new file mode 100644 index 000000000000..75e7ff7e9ae3 --- /dev/null +++ b/src/tools/dashboard/src/reference.rs @@ -0,0 +1,279 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 OR MIT +//! Utilities to extract examples from +//! [The Rust Reference](https://doc.rust-lang.org/nightly/reference), +//! run them through RMC, and display their results. + +use crate::dashboard; +use pulldown_cmark::{Parser, Tag}; +use std::{ + collections::HashMap, + env, fs, + io::{BufRead, BufReader}, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +/// Parses the chapter/section hierarchy in the markdown file specified by +/// `summary_path` and returns a mapping from markdown files containing rust +/// code to corresponding directories where the extracted rust code should +/// reside. +fn parse_hierarchy(summary_path: &Path) -> HashMap { + let start = "# The Rust Reference\n\n[Introduction](introduction.md)"; + let summary = fs::read_to_string(summary_path).unwrap(); + assert!(summary.starts_with(start), "Error: The start of the summary file changed."); + // Skip the title and introduction. + let n = Parser::new(start).count(); + let parser = Parser::new(&summary).skip(n); + // Set "ref" as the root of the hierarchical path. + let mut hierarchy = PathBuf::from("ref"); + let mut map = HashMap::new(); + // Introduction is a especial case, so handle it separately. + map.insert(PathBuf::from("introduction.md"), hierarchy.join("Introduction")); + for event in parser { + match event { + pulldown_cmark::Event::End(Tag::Item) => { + // Pop the current chapter/section from the hierarchy once + // we are done processing it and its subsections. + hierarchy.pop(); + } + pulldown_cmark::Event::End(Tag::Link(_, path, _)) => { + // At the start of the link tag, the hierarchy does not yet + // contain the title of the current chapter/section. So, we wait + // for the end of the link tag before adding the path and + // hierarchy of the current chapter/section to the map. + map.insert(path.split('/').collect(), hierarchy.clone()); + } + pulldown_cmark::Event::Text(text) => { + // Add the current chapter/section title to the hierarchy. + hierarchy.push(text.to_string()); + } + _ => (), + } + } + map +} + +/// Extracts examples from the given relative `paths` in the `book_dir` and +/// saves them in `gen_dir`. +fn extract_examples(paths: Vec<&PathBuf>, book_dir: &Path, gen_dir: &Path) { + for path in paths { + let mut cmd = Command::new("rustdoc"); + cmd.args([ + "+nightly", + "--test", + "-Z", + "unstable-options", + book_dir.join(path).to_str().unwrap(), + "--test-builder", + &["src", "tools", "dashboard", "print.sh"] + .iter() + .collect::() + .to_str() + .unwrap(), + "--persist-doctests", + gen_dir.to_str().unwrap(), + "--no-run", + ]); + cmd.stdout(Stdio::null()); + cmd.spawn().unwrap().wait().unwrap(); + } +} + +/// Copies the extracted rust code in `from_dir` to `src/test` following the +/// hierarchy specified by `map`. +fn organize_examples(map: &HashMap, book_dir: &Path, from_dir: &Path) { + // The names of the extracted examples generated by `rustdoc` have the + // format `__` where occurrences of '/', '-', and + // '.' in are replaced by '_'. This transformation is not injective, + // so we cannot map those names back to the original markdown file path. + // Instead, we apply the same transformation on the keys of `map` in the for + // loop below and lookup in those modified keys. + let mut modified_map = HashMap::new(); + for (path, hierarchy) in map.iter() { + modified_map.insert( + book_dir.join(path).to_str().unwrap().replace(&['\\', '/', '-', '.'][..], "_"), + hierarchy.clone(), + ); + } + for dir in from_dir.read_dir().unwrap() { + let dir = dir.unwrap().path(); + // Some directories do not contain tests because the markdown file + // instructs `rustdoc` to ignore those tests. + if let Some(example) = dir.read_dir().unwrap().next() { + let example = example.unwrap().path(); + copy(&example, &modified_map); + } + } +} + +/// Copy the file specified by `from` to the corresponding location specified by +/// `map`. +fn copy(from: &Path, map: &HashMap) { + // The path specified by `from` has the form: + // `src/tools/dashboard/target/ref/__/rust_out` + // We copy the file in this path to a new path of the form: + // `src/test//.rs + // where `map[] == `. We omit because all tests have + // the same number, 0. + // Extract `__`. + let key_line_test = from.parent().unwrap().file_name().unwrap().to_str().unwrap(); + // Extract and from `key_line_test` to get and + // construct destination path. + let splits: Vec<_> = key_line_test.rsplitn(3, '_').collect(); + let key = splits[2]; + let line = splits[1]; + let val = &map[key]; + let name = &format!("{}.rs", line); + let to = Path::new("src").join("test").join(val).join(name); + fs::create_dir_all(to.parent().unwrap()).unwrap(); + fs::copy(&from, &to).unwrap(); +} + +/// Pre-processes the tests in the specified `paths` before running them with +/// `compiletest`. +fn preprocess_examples(_paths: Vec<&PathBuf>) { + // For now, we will only pre-process the tests that cause infinite loops. + // TODO: properly implement this step (see issue #324). + let loop_tests: [PathBuf; 4] = [ + ["src", "test", "ref", "Appendices", "Glossary", "263.rs"].iter().collect(), + ["src", "test", "ref", "Linkage", "190.rs"].iter().collect(), + [ + "src", + "test", + "ref", + "Statements and expressions", + "Expressions", + "Loop expressions", + "133.rs", + ] + .iter() + .collect(), + [ + "src", + "test", + "ref", + "Statements and expressions", + "Expressions", + "Method call expressions", + "10.rs", + ] + .iter() + .collect(), + ]; + + for test in loop_tests { + let code = fs::read_to_string(&test).unwrap(); + let code = format!("// cbmc-flags: --unwind 1 --unwinding-assertions\n{}", code); + fs::write(&test, code).unwrap(); + } +} + +/// Runs `compiletest` on the `suite` and logs the results to `log_path`. +fn run_examples(suite: &str, log_path: &Path) { + // Before executing this program, `cargo` populates the environment with + // build configs. `x.py` respects those configs, causing a recompilation + // of `rustc`. This is not a desired behavior, so we remove those configs. + let filtered_env: HashMap = env::vars() + .filter(|&(ref k, _)| { + !(k.contains("CARGO") || k.contains("LD_LIBRARY_PATH") || k.contains("RUST")) + }) + .collect(); + let mut cmd = Command::new([".", "x.py"].iter().collect::()); + cmd.args([ + "test", + suite, + "-i", + "--stage", + "1", + "--test-args", + "--logfile", + "--test-args", + log_path.to_str().unwrap(), + ]); + cmd.env_clear().envs(filtered_env); + cmd.stdout(Stdio::null()); + + cmd.spawn().unwrap().wait().unwrap(); +} + +/// Creates a new [`Tree`] from `path`, and a test `result`. +fn tree_from_path(mut path: Vec, result: bool) -> dashboard::Tree { + assert!(path.len() > 0, "Error: `path` must contain at least 1 element."); + let mut tree = dashboard::Tree::new( + dashboard::Node::new( + path.pop().unwrap(), + if result { 1 } else { 0 }, + if result { 0 } else { 1 }, + ), + vec![], + ); + for _ in 0..path.len() { + tree = dashboard::Tree::new( + dashboard::Node::new(path.pop().unwrap(), tree.data.num_pass, tree.data.num_fail), + vec![tree], + ); + } + tree +} + +/// Parses and generates a dashboard from the log output of `compiletest` in +/// `path`. +fn parse_log(path: &Path) -> dashboard::Tree { + let file = fs::File::open(path).unwrap(); + let reader = BufReader::new(file); + let mut tests = dashboard::Tree::new(dashboard::Node::new(String::from("ref"), 0, 0), vec![]); + for line in reader.lines() { + let (ns, l) = parse_log_line(&line.unwrap()); + tests = dashboard::Tree::merge(tests, tree_from_path(ns, l)).unwrap(); + } + tests +} + +/// Parses a line in the log output of `compiletest` and returns a pair containing +/// the path to a test and its result. +fn parse_log_line(line: &str) -> (Vec, bool) { + // Each line has the format ` [rmc] `. Extract and + // . + let splits: Vec<_> = line.split(" [rmc] ").map(String::from).collect(); + let l = if splits[0].as_str() == "ok" { true } else { false }; + let mut ns: Vec<_> = splits[1].split(&['/', '.'][..]).map(String::from).collect(); + // Remove unnecessary `.rs` suffix. + ns.pop(); + (ns, l) +} + +/// Display the dashboard in the terminal. +fn display_dashboard(dashboard: dashboard::Tree) { + println!( + "# of tests: {}\t✔️ {}\t❌ {}", + dashboard.data.num_pass + dashboard.data.num_fail, + dashboard.data.num_pass, + dashboard.data.num_fail + ); + println!("{}", dashboard); +} + +/// Extracts examples from The Rust Reference, run them through RMC, and +/// displays their results in a terminal dashboard. +pub fn display_reference_dashboard() { + let summary_path: PathBuf = ["src", "doc", "reference", "src", "SUMMARY.md"].iter().collect(); + let ref_dir: PathBuf = ["src", "doc", "reference", "src"].iter().collect(); + let gen_dir: PathBuf = ["src", "tools", "dashboard", "target", "ref"].iter().collect(); + let log_path: PathBuf = ["src", "tools", "dashboard", "target", "ref.log"].iter().collect(); + // Parse the chapter/section hierarchy from the table of contents in The + // Rust Reference. + let map = parse_hierarchy(&summary_path); + // Extract examples from The Rust Reference. + extract_examples(map.keys().collect(), &ref_dir, &gen_dir); + // Reorganize those examples following the The Rust Reference hierarchy. + organize_examples(&map, &ref_dir, &gen_dir); + // Pre-process the examples before running them through `compiletest`. + preprocess_examples(map.values().collect()); + // Run `compiletest` on the reference examples. + run_examples("ref", &log_path); + // Parse `compiletest` log file. + let dashboard = parse_log(&log_path); + // Display the reference dashboard. + display_dashboard(dashboard); +}