Add support for headless testing

This commit adds support to the `wasm-bindgen-test-runner` binary to
perform headless testing via browsers. The previous commit introduced a
local server to serve up files and run tests in a browser, and this
commit adds support for executing that in an automated fashion.

The general idea here is that each browser has a binary that implements
the WebDriver specification. These binaries (typically `foodriver` for
the browser "Foo") are interfaced with using HTTP and JSON messages. The
implementation was simple enough and the crates.io support was lacking
enough that a small implementation of the WebDriver protocol was added
directly to this crate.

Currently Firefox (`geckodriver`), Chrome (`chromedriver`), and Safari
(`safaridriver`) are supported for running tests. The test harness will
recognize env vars like `GECKODRIVER=foo` to specifically use one or
otherwise detects the first driver in `PATH`. Eventually we may wish to
automatically download a driver if one isn't found, but that isn't
implemented yet.

Headless testing is turned on with the `CI=1` env var currently to be
amenable with things like Travis and AppVeyor, but this may wish to grow
an explicit option to run headless tests in the future.

crates/cli/src/bin/wasm-bindgen-test-runner/headless.rs

@@ -0,0 +1,444 @@
+use std::cell::{Cell, RefCell};
+use std::env;
+use std::io::{self, Read};
+use std::net::{SocketAddr, TcpListener, TcpStream};
+use std::path::{PathBuf, Path};
+use std::process::{Command, Stdio};
+use std::thread;
+use std::time::{Instant, Duration};
+
+use curl::easy::Easy;
+use failure::{ResultExt, Error};
+use serde::{Serialize, Deserialize};
+use serde_json;
+
+use shell::Shell;
+
+/// Execute a headless browser tests against a server running on `server`
+/// address.
+pub fn run(server: &SocketAddr, shell: &Shell) -> Result<(), Error> {
+    let (driver, args) = Driver::find()?;
+    println!("Running headless tests in {} with `{}`",
+             driver.browser(),
+             driver.path().display());
+
+    // Allow tests to run in parallel (in theory) by finding any open port
+    // available for our driver. We can't bind the port for the driver, but
+    // hopefully the OS gives this invocation unique ports across processes
+    let driver_addr = TcpListener::bind("127.0.0.1:0")?.local_addr()?;
+
+    // Spawn the driver binary, collecting its stdout/stderr in separate
+    // threads. We'll print this output later.
+    shell.status("Spawning Geckodriver...");
+    let mut cmd = Command::new(&driver.path());
+    cmd.args(&args)
+        .arg(format!("--port={}", driver_addr.port().to_string()))
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .stdin(Stdio::null());
+    let mut child = cmd.spawn()
+        .context(format!("failed to spawn {:?} binary", driver.path()))?;
+    let mut stdout = child.stdout.take().unwrap();
+    let mut stderr = child.stderr.take().unwrap();
+    let mut stdout = Some(thread::spawn(move || read(&mut stdout)));
+    let mut stderr = Some(thread::spawn(move || read(&mut stderr)));
+    let print_driver_stdio = Cell::new(true);
+    let _f = OnDrop(|| {
+        child.kill().unwrap();
+        let status = child.wait().unwrap();
+        if !print_driver_stdio.get() {
+            return
+        }
+
+        shell.clear();
+        println!("driver status: {}", status);
+
+        let stdout = stdout.take().unwrap().join().unwrap().unwrap();
+        if stdout.len() > 0 {
+            println!("driver stdout:\n{}", tab(&String::from_utf8_lossy(&stdout)));
+        }
+        let stderr = stderr.take().unwrap().join().unwrap().unwrap();
+        if stderr.len() > 0 {
+            println!("driver stderr:\n{}", tab(&String::from_utf8_lossy(&stderr)));
+        }
+    });
+
+    // Wait for the driver to come online and bind its port before we try to
+    // connect to it.
+    let start = Instant::now();
+    let max = Duration::new(5, 0);
+    let mut bound = false;
+    while start.elapsed() < max {
+        if TcpStream::connect(&driver_addr).is_ok() {
+            bound = true;
+            break
+        }
+        thread::sleep(Duration::from_millis(100));
+    }
+    if !bound {
+        bail!("driver failed to bind port during startup")
+    }
+
+    let client = Client {
+        handle: RefCell::new(Easy::new()),
+        driver_addr,
+    };
+    shell.status("Starting new webdriver session...");
+    // Allocate a new session with the webdriver protocol, and once we've done
+    // so schedule the browser to get closed with a call to `close_window`.
+    let id = client.new_session(&driver)?;
+    let _f = OnDrop(|| {
+        if let Err(e) = client.close_window(&id) {
+            warn!("failed to close window {:?}", e);
+        }
+    });
+
+    // Visit our local server to open up the page that runs tests, and then get
+    // some handles to objects on the page which we'll be scraping output from.
+    let url = format!("http://{}", server);
+    shell.status(&format!("Visiting {}...", url));
+    client.goto(&id, &url)?;
+    shell.status("Loading page elements...");
+    let output = client.element(&id, "#output")?;
+    let logs = client.element(&id, "#console_log")?;
+    let errors = client.element(&id, "#console_error")?;
+
+    // At this point we need to wait for the test to finish before we can take a
+    // look at what happened. There appears to be no great way to do this with
+    // the webdriver protocol today (in terms of synchronization), so for now we
+    // just go with a loop.
+    //
+    // We periodically check the page to see if the output contains a known
+    // string to only be printed when tests have finished running.
+    //
+    // TODO: harness failures aren't well handled here, they always force a
+    //       timeout. These sorts of failures could be "you typo'd the path to a
+    //       local script" which is pretty bad to time out for, we should detect
+    //       this on the page and look for such output here, printing diagnostic
+    //       information.
+    shell.status("Waiting for test to finish...");
+    let start = Instant::now();
+    let max = Duration::new(20, 0);
+    while start.elapsed() < max {
+        if client.text(&id, &output)?.contains("test result: ") {
+            break
+        }
+        thread::sleep(Duration::from_millis(100));
+    }
+    shell.clear();
+
+    // Tests have now finished or have timed out. At this point we need to print
+    // what happened on the console. Currently we just do this by scraping the
+    // output of various fields and printing them out, hopefully providing
+    // enough diagnostic info to see what went wrong (if anything).
+    let output = client.text(&id, &output)?;
+    let logs = client.text(&id, &logs)?;
+    let errors = client.text(&id, &errors)?;
+
+    if output.contains("test result: ") {
+        println!("{}", output);
+
+        // If the tests harness finished (either successfully or unsuccessfully)
+        // then in theory all the info needed to debug the failure is in its own
+        // output, so we shouldn't need the driver logs to get printed.
+        print_driver_stdio.set(false);
+    } else {
+        println!("failed to detect test as having been run");
+        if output.len() > 0 {
+            println!("output div contained:\n{}", tab(&output));
+        }
+    }
+    if logs.len() > 0 {
+        println!("console.log div contained:\n{}", tab(&logs));
+    }
+    if errors.len() > 0 {
+        println!("console.log div contained:\n{}", tab(&errors));
+    }
+
+    Ok(())
+}
+
+enum Driver {
+    Gecko(PathBuf),
+    Safari(PathBuf),
+    Chrome(PathBuf),
+}
+
+impl Driver {
+    fn find() -> Result<(Driver, Vec<String>), Error> {
+        let env_args = |name: &str| {
+            env::var(format!("{}_ARGS", name.to_uppercase()))
+                .unwrap_or_default()
+                .split_whitespace()
+                .map(|s| s.to_string())
+                .collect::<Vec<_>>()
+        };
+
+        let drivers = [
+            ("geckodriver", Driver::Gecko as fn(PathBuf) -> Driver),
+            ("safaridriver", Driver::Safari as fn(PathBuf) -> Driver),
+            ("chromedriver", Driver::Chrome as fn(PathBuf) -> Driver),
+        ];
+
+        // First up, if env vars like GECKODRIVER are present, use those to
+        // allow forcing usage of a particular driver.
+        for (driver, ctor) in drivers.iter() {
+            let env = driver.to_uppercase();
+            let path = match env::var_os(&env) {
+                Some(path) => path,
+                None => continue,
+            };
+            return Ok((ctor(path.into()), env_args(driver)))
+        }
+
+        // Next, check PATH. If we can find any supported driver, use that by
+        // default.
+        for path in env::split_paths(&env::var_os("PATH").unwrap_or_default()) {
+            let found = drivers
+                .iter()
+                .find(|(name, _)| {
+                    path.join(name)
+                        .with_extension(env::consts::EXE_EXTENSION)
+                        .exists()
+                });
+            let (name, ctor) = match found {
+                Some(p) => p,
+                None => continue,
+            };
+            return Ok((ctor(name.into()), env_args(name)))
+        }
+
+        // TODO: download an appropriate driver? How to know which one to
+        //       download?
+
+        bail!("failed to find a suitable WebDriver binary to drive headless \
+               testing")
+    }
+
+    fn path(&self) -> &Path {
+        match self {
+            Driver::Gecko(path) => path,
+            Driver::Safari(path) => path,
+            Driver::Chrome(path) => path,
+        }
+    }
+
+    fn browser(&self) -> &str {
+        match self {
+            Driver::Gecko(_) => "Firefox",
+            Driver::Safari(_) => "Safari",
+            Driver::Chrome(_) => "Chrome",
+        }
+    }
+}
+
+struct Client {
+    handle: RefCell<Easy>,
+    driver_addr: SocketAddr,
+}
+
+enum Method<'a> {
+    Get,
+    Post(&'a str),
+    Delete,
+}
+
+impl Client {
+    fn new_session(&self, driver: &Driver) -> Result<String, Error> {
+        match driver {
+            Driver::Gecko(_) => {
+                #[derive(Deserialize)]
+                struct Response {
+                    value: ResponseValue,
+                }
+
+                #[derive(Deserialize)]
+                struct ResponseValue {
+                    #[serde(rename = "sessionId")]
+                    session_id: String,
+                }
+                let request = json!({
+                    "capabilities": {
+                        "alwaysMatch": {
+                            "moz:firefoxOptions": {
+                                "args": ["-headless"],
+                            }
+                        }
+                    }
+                });
+                let x: Response = self.post("/session", &request)?;
+                Ok(x.value.session_id)
+            }
+            Driver::Safari(_) => {
+                #[derive(Deserialize)]
+                struct Response {
+                    #[serde(rename = "sessionId")]
+                    session_id: String,
+                }
+                let request = json!({
+                    "desiredCapabilities": {
+                    }
+                });
+                let x: Response = self.post("/session", &request)?;
+                Ok(x.session_id)
+            }
+            Driver::Chrome(_) => {
+                #[derive(Deserialize)]
+                struct Response {
+                    #[serde(rename = "sessionId")]
+                    session_id: String,
+                }
+                let request = json!({
+                    "desiredCapabilities": {
+                    }
+                });
+                let x: Response = self.post("/session", &request)?;
+                Ok(x.session_id)
+            }
+        }
+    }
+
+    fn close_window(&self, id: &str) -> Result<(), Error> {
+        #[derive(Deserialize)]
+        struct Response {
+        }
+        let x: Response = self.delete(&format!("/session/{}/window", id))?;
+        drop(x);
+        Ok(())
+    }
+
+    fn goto(&self, id: &str, url: &str) -> Result<(), Error> {
+        #[derive(Serialize)]
+        struct Request {
+            url: String,
+        }
+        #[derive(Deserialize)]
+        struct Response {
+        }
+
+        let request = Request {
+            url: url.to_string(),
+        };
+        let x: Response = self.post(&format!("/session/{}/url", id), &request)?;
+        drop(x);
+        Ok(())
+    }
+
+    fn element(&self, id: &str, selector: &str) -> Result<String, Error> {
+        #[derive(Serialize)]
+        struct Request {
+            using: String,
+            value: String,
+        }
+        #[derive(Deserialize)]
+        struct Response {
+            value: Reference,
+        }
+        #[derive(Deserialize)]
+        struct Reference {
+            #[serde(rename = "element-6066-11e4-a52e-4f735466cecf")]
+            gecko_reference: Option<String>,
+            #[serde(rename = "ELEMENT")]
+            safari_reference: Option<String>,
+        }
+
+        let request = Request {
+            using: "css selector".to_string(),
+            value: selector.to_string(),
+        };
+        let x: Response = self.post(&format!("/session/{}/element", id), &request)?;
+        Ok(x.value.gecko_reference
+           .or(x.value.safari_reference)
+           .ok_or(format_err!("failed to find element reference in response"))?)
+
+    }
+
+    fn text(&self, id: &str, element: &str) -> Result<String, Error> {
+        #[derive(Deserialize)]
+        struct Response {
+            value: String,
+        }
+        let x: Response = self.get(&format!("/session/{}/element/{}/text", id, element))?;
+        Ok(x.value)
+    }
+
+    fn get<U>(&self, path: &str) -> Result<U, Error>
+        where U: for<'a> Deserialize<'a>,
+    {
+        debug!("GET {}", path);
+        let result = self.doit(path, Method::Get)?;
+        Ok(serde_json::from_str(&result)?)
+    }
+
+    fn post<T, U>(&self, path: &str, data: &T) -> Result<U, Error>
+        where T: Serialize,
+              U: for<'a> Deserialize<'a>,
+    {
+        let input = serde_json::to_string(data)?;
+        debug!("POST {} {}", path, input);
+        let result = self.doit(path, Method::Post(&input))?;
+        Ok(serde_json::from_str(&result)?)
+    }
+
+    fn delete<U>(&self, path: &str) -> Result<U, Error>
+        where U: for<'a> Deserialize<'a>,
+    {
+        debug!("DELETE {}", path);
+        let result = self.doit(path, Method::Delete)?;
+        Ok(serde_json::from_str(&result)?)
+    }
+
+    fn doit(&self, path: &str, method: Method) -> Result<String, Error> {
+        let url = format!("http://{}{}", self.driver_addr, path);
+        let mut handle = self.handle.borrow_mut();
+        handle.reset();
+        handle.url(&url)?;
+        match method {
+            Method::Post(data) => {
+                handle.post(true)?;
+                handle.post_fields_copy(data.as_bytes())?;
+            }
+            Method::Delete => handle.custom_request("DELETE")?,
+            Method::Get => handle.get(true)?,
+        }
+        let mut result = Vec::new();
+        {
+            let mut t = handle.transfer();
+            t.write_function(|buf| {
+                result.extend_from_slice(buf);
+                Ok(buf.len())
+            })?;
+            t.perform()?
+        }
+        let result = String::from_utf8_lossy(&result);
+        if handle.response_code()? != 200 {
+            bail!("non-200 response code: {}\n{}", handle.response_code()?, result);
+        }
+        debug!("got: {}", result);
+        Ok(result.into_owned())
+    }
+}
+
+fn read<R: Read>(r: &mut R) -> io::Result<Vec<u8>> {
+    let mut dst = Vec::new();
+    r.read_to_end(&mut dst)?;
+    Ok(dst)
+}
+
+fn tab(s: &str) -> String {
+    let mut result = String::new();
+    for line in s.lines() {
+        result.push_str("    ");
+        result.push_str(line);
+        result.push_str("\n");
+    }
+    return result;
+}
+
+struct OnDrop<F: FnMut()>(F);
+
+impl<F: FnMut()> Drop for OnDrop<F> {
+    fn drop(&mut self) {
+        (self.0)();
+    }
+}

crates/cli/src/bin/wasm-bindgen-test-runner/index-headless.html

@@ -0,0 +1,37 @@
+<html>
+  <head>
+    <meta content="text/html;charset=utf-8" http-equiv="Content-Type"/>
+  </head>
+  <body>
+    <pre id='output'>Loading scripts...</pre>
+    <pre id='console_log'></pre>
+    <pre id='console_error'></pre>
+    <script>
+      const orig_console_log = function(...args) {
+        const logs = document.getElementById('console_log');
+        for (let msg of args) {
+          logs.innerHTML += `${msg}\n`;
+        }
+      };
+      const orig_console_error = function(...args) {
+        const logs = document.getElementById('console_error');
+        for (let msg of args) {
+          logs.innerHTML += `${msg}\n`;
+        }
+      };
+      console.log = function() {
+        if (window.global_cx)
+          window.global_cx.console_log(orig_console_log, arguments);
+        else
+          orig_console_log.apply(this, arguments);
+      };
+      console.error = function() {
+        if (window.global_cx)
+          window.global_cx.console_error(orig_console_error, arguments);
+        else
+          orig_console_error.apply(this, arguments);
+      };
+    </script>
+    <script src='run.js' type=module></script>
+  </body>
+</html>

crates/cli/src/bin/wasm-bindgen-test-runner/main.rs

@@ -1,23 +1,35 @@
+extern crate curl;
+extern crate env_logger;
 #[macro_use]
 extern crate failure;
+#[macro_use]
+extern crate log;
 extern crate parity_wasm;
 extern crate rouille;
+extern crate serde;
+#[macro_use]
+extern crate serde_derive;
+#[macro_use]
+extern crate serde_json;
 extern crate wasm_bindgen_cli_support;
 
 use std::env;
 use std::fs;
-use std::io::{self, Write};
 use std::path::PathBuf;
 use std::process;
+use std::thread;
 
 use failure::{ResultExt, Error};
-use parity_wasm::elements::{Module, Deserialize};
+use parity_wasm::elements::{Module, Deserialize, Section};
 use wasm_bindgen_cli_support::Bindgen;
 
+mod headless;
 mod node;
 mod server;
+mod shell;
 
 fn main() {
+    env_logger::init();
     let err = match rmain() {
         Ok(()) => return,
         Err(e) => e,
@@ -31,6 +43,7 @@ fn main() {
 
 fn rmain() -> Result<(), Error> {
     let mut args = env::args_os().skip(1);
+    let shell = shell::Shell::new();
 
     // Currently no flags are supported, and assume there's only one argument
     // which is the wasm file to test. This'll want to improve over time!
@@ -61,10 +74,6 @@ fn rmain() -> Result<(), Error> {
     // Collect all tests that the test harness is supposed to run. We assume
     // that any exported function with the prefix `__wbg_test` is a test we need
     // to execute.
-    //
-    // Note that we're collecting *JS objects* that represent the functions to
-    // execute, and then those objects are passed into wasm for it to execute
-    // when it sees fit.
     let wasm = fs::read(&wasm_file_to_test)
         .context("failed to read wasm file")?;
     let wasm = Module::deserialize(&mut &wasm[..])
@@ -78,18 +87,34 @@ fn rmain() -> Result<(), Error> {
             tests.push(export.field().to_string());
         }
     }
+
+    // Right now there's a bug where if no tests are present then the
+    // `wasm-bindgen-test` runtime support isn't linked in, so just bail out
+    // early saying everything is ok.
     if tests.len() == 0 {
         println!("no tests to run!");
         return Ok(())
     }
 
-    let node = true;
+    // Figure out if this tests is supposed to execute in node.js or a browser.
+    // That's done on a per-test-binary basis with the
+    // `wasm_bindgen_test_configure` macro, which emits a custom section for us
+    // to read later on.
+    let mut node = true;
+    for section in wasm.sections() {
+        let custom = match section {
+            Section::Custom(section) => section,
+            _ => continue,
+        };
+        if custom.name() != "__wasm_bindgen_test_unstable" {
+            continue
+        }
+        node = !custom.payload().contains(&0x01);
+    }
+    let headless = env::var("CI").is_ok();
 
-    print!("Executing bindgen ...\r");
-    io::stdout().flush()?;
-
-    // For now unconditionally generate wasm-bindgen code tailored for node.js,
-    // but eventually we'll want more options here for browsers!
+    // Make the generated bindings available for the tests to execute against.
+    shell.status("Executing bindgen...");
     let mut b = Bindgen::new();
     b.debug(true)
         .nodejs(node)
@@ -97,13 +122,37 @@ fn rmain() -> Result<(), Error> {
         .keep_debug(false)
         .generate(&tmpdir)
         .context("executing `wasm-bindgen` over the wasm file")?;
+    shell.clear();
 
-    print!("                     \r");
-    io::stdout().flush()?;
-
+    // If we're executing in node.js, that module will take it from here.
     if node {
         return node::execute(&module, &tmpdir, &args.collect::<Vec<_>>(), &tests)
     }
 
-    server::spawn(&module, &tmpdir, &args.collect::<Vec<_>>(), &tests)
+    // Otherwise we're executing in a browser. Spawn a server which serves up
+    // the local generated files over an HTTP server.
+    let srv = server::spawn(
+        &if headless {
+            "127.0.0.1:0".parse().unwrap()
+        } else {
+            "127.0.0.1:8000".parse().unwrap()
+        },
+        headless,
+        &module,
+        &tmpdir,
+        &args.collect::<Vec<_>>(),
+        &tests,
+    )?;
+    let addr = srv.server_addr();
+
+    // TODO: eventually we should provide the ability to exit at some point
+    // (gracefully) here, but for now this just runs forever.
+    if !headless {
+        println!("Running server on http://{}", addr);
+        return Ok(srv.run())
+    }
+
+    thread::spawn(|| srv.run());
+    headless::run(&addr, &shell)?;
+    Ok(())
 }

crates/cli/src/bin/wasm-bindgen-test-runner/node.rs

@@ -66,7 +66,6 @@ pub fn execute(module: &str, tmpdir: &Path, args: &[OsString], tests: &[String])
     for test in tests {
         js_to_execute.push_str(&format!("tests.push('{}')\n", test));
     }
-
     // And as a final addendum, exit with a nonzero code if any tests fail.
     js_to_execute.push_str("
         main(tests)

crates/cli/src/bin/wasm-bindgen-test-runner/server.rs

@@ -1,14 +1,20 @@
 use std::ffi::OsString;
 use std::path::Path;
 use std::fs;
+use std::net::SocketAddr;
 
 use failure::{ResultExt, Error};
-use rouille::{self, Response, Request};
+use rouille::{self, Response, Request, Server};
 use wasm_bindgen_cli_support::wasm2es6js::Config;
 
-pub fn spawn(module: &str, tmpdir: &Path, args: &[OsString], tests: &[String])
-    -> Result<(), Error>
-{
+pub fn spawn(
+    addr: &SocketAddr,
+    headless: bool,
+    module: &str,
+    tmpdir: &Path,
+    args: &[OsString],
+    tests: &[String],
+) -> Result<Server<impl Fn(&Request) -> Response + Send + Sync>, Error> {
     let mut js_to_execute = format!(r#"
         import {{ Context }} from './{0}';
         import * as wasm from './{0}_bg';
@@ -55,12 +61,16 @@ pub fn spawn(module: &str, tmpdir: &Path, args: &[OsString], tests: &[String])
         .context("failed to write JS file")?;
 
     // For now, always run forever on this port. We may update this later!
-    println!("Listening on port 8000");
     let tmpdir = tmpdir.to_path_buf();
-    rouille::start_server("localhost:8000", move |request| {
+    let srv = Server::new(addr, move |request| {
         // The root path gets our canned `index.html`
         if request.url() == "/" {
-            return Response::from_data("text/html", include_str!("index.html"));
+            let s = if headless {
+                include_str!("index-headless.html")
+            } else {
+                include_str!("index.html")
+            };
+            return Response::from_data("text/html", s)
         }
 
         // Otherwise we need to find the asset here. It may either be in our
@@ -74,7 +84,8 @@ pub fn spawn(module: &str, tmpdir: &Path, args: &[OsString], tests: &[String])
         // header?)
         response.headers.retain(|(k, _)| k != "Cache-Control");
         return response
-    });
+    }).map_err(|e| format_err!("{}", e))?;
+    return Ok(srv);
 
     fn try_asset(request: &Request, dir: &Path) -> Response {
         let response = rouille::match_assets(request, dir);

crates/cli/src/bin/wasm-bindgen-test-runner/shell.rs

@@ -0,0 +1,32 @@
+const WIDTH: usize = 50;
+
+use std::io::{self, Write};
+
+pub struct Shell {
+}
+
+impl Shell {
+    pub fn new() -> Shell {
+        Shell {}
+    }
+
+    pub fn status(&self, s: &str) {
+        let s = if s.len() > WIDTH {
+            &s[..WIDTH]
+        } else {
+            s
+        };
+        print!("{:<1$}\r", s, WIDTH);
+        io::stdout().flush().unwrap();
+    }
+
+    pub fn clear(&self) {
+        self.status("");
+    }
+}
+
+impl Drop for Shell {
+    fn drop(&mut self) {
+        self.clear();
+    }
+}