util/flashrom_tester: Upstream E2E testing framework

The following is a E2E tester for a specific chip/chipset
combo. The tester itself is completely self-contained and
allows the user to specify which tests they wish to preform.
Supported tests include:

 - chip-name
 - read
 - write
 - erase
 - wp-locking

Change-Id: Ic2905a76cad90b1546b9328d668bf8abbf8aed44
Signed-off-by: Edward O'Callaghan <quasisec@google.com>
Reviewed-on: https://review.coreboot.org/c/flashrom/+/38951
Tested-by: build bot (Jenkins) <no-reply@coreboot.org>
Reviewed-by: David Hendricks <david.hendricks@gmail.com>

1 files changed, 636 insertions, 0 deletions

diff --git a/util/flashrom_tester/src/tester.rs b/util/flashrom_tester/src/tester.rs
new file mode 100644
index 00000000..fbef2016
--- /dev/null
+++ b/util/flashrom_tester/src/tester.rs
@@ -0,0 +1,636 @@
+//
+// Copyright 2019, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+//
+// Alternatively, this software may be distributed under the terms of the
+// GNU General Public License ("GPL") version 2 as published by the Free
+// Software Foundation.
+//
+
+use super::rand_util;
+use super::types;
+use super::utils::{self, LayoutSizes};
+use flashrom::{FlashChip, Flashrom, FlashromCmd};
+use serde_json::json;
+use std::mem::MaybeUninit;
+use std::sync::Mutex;
+
+// type-signature comes from the return type of lib.rs workers.
+type TestError = Box<dyn std::error::Error>;
+pub type TestResult = Result<(), TestError>;
+
+pub struct TestEnv<'a> {
+    chip_type: FlashChip,
+    /// Flashrom instantiation information.
+    ///
+    /// Where possible, prefer to use methods on the TestEnv rather than delegating
+    /// to the raw flashrom functions.
+    pub cmd: &'a FlashromCmd,
+    layout: LayoutSizes,
+
+    pub wp: WriteProtectState<'a, 'static>,
+    /// The path to a file containing the flash contents at test start.
+    // TODO(pmarheine) migrate this to a PathBuf for clarity
+    original_flash_contents: String,
+    /// The path to a file containing flash-sized random data
+    // TODO(pmarheine) make this a PathBuf too
+    random_data: String,
+}
+
+impl<'a> TestEnv<'a> {
+    pub fn create(chip_type: FlashChip, cmd: &'a FlashromCmd) -> Result<Self, String> {
+        let rom_sz = cmd.get_size()?;
+        let out = TestEnv {
+            chip_type: chip_type,
+            cmd: cmd,
+            layout: utils::get_layout_sizes(rom_sz)?,
+            wp: WriteProtectState::from_hardware(cmd)?,
+            original_flash_contents: "/tmp/flashrom_tester_golden.bin".into(),
+            random_data: "/tmp/random_content.bin".into(),
+        };
+
+        info!("Stashing golden image for verification/recovery on completion");
+        flashrom::read(&out.cmd, &out.original_flash_contents)?;
+        flashrom::verify(&out.cmd, &out.original_flash_contents)?;
+
+        info!("Generating random flash-sized data");
+        rand_util::gen_rand_testdata(&out.random_data, rom_sz as usize)
+            .map_err(|io_err| format!("I/O error writing random data file: {:#}", io_err))?;
+
+        Ok(out)
+    }
+
+    pub fn run_test<T: TestCase>(&mut self, test: T) -> TestResult {
+        let use_dut_control = self.chip_type == FlashChip::SERVO;
+        if use_dut_control && flashrom::dut_ctrl_toggle_wp(false).is_err() {
+            error!("failed to dispatch dut_ctrl_toggle_wp()!");
+        }
+
+        let name = test.get_name();
+        info!("Beginning test: {}", name);
+        let out = test.run(self);
+        info!("Completed test: {}; result {:?}", name, out);
+
+        if use_dut_control && flashrom::dut_ctrl_toggle_wp(true).is_err() {
+            error!("failed to dispatch dut_ctrl_toggle_wp()!");
+        }
+        out
+    }
+
+    pub fn chip_type(&self) -> FlashChip {
+        // This field is not public because it should be immutable to tests,
+        // so this getter enforces that it is copied.
+        self.chip_type
+    }
+
+    /// Return the path to a file that contains random data and is the same size
+    /// as the flash chip.
+    pub fn random_data_file(&self) -> &str {
+        &self.random_data
+    }
+
+    pub fn layout(&self) -> &LayoutSizes {
+        &self.layout
+    }
+
+    /// Return true if the current Flash contents are the same as the golden image
+    /// that was present at the start of testing.
+    pub fn is_golden(&self) -> bool {
+        flashrom::verify(&self.cmd, &self.original_flash_contents).is_ok()
+    }
+
+    /// Do whatever is necessary to make the current Flash contents the same as they
+    /// were at the start of testing.
+    pub fn ensure_golden(&mut self) -> Result<(), String> {
+        self.wp.set_hw(false)?.set_sw(false)?;
+        flashrom::write(&self.cmd, &self.original_flash_contents)
+    }
+
+    /// Attempt to erase the flash.
+    pub fn erase(&self) -> Result<(), String> {
+        flashrom::erase(self.cmd)
+    }
+
+    /// Verify that the current Flash contents are the same as the file at the given
+    /// path.
+    ///
+    /// Returns Err if they are not the same.
+    pub fn verify(&self, contents_path: &str) -> Result<(), String> {
+        flashrom::verify(self.cmd, contents_path)
+    }
+}
+
+impl Drop for TestEnv<'_> {
+    fn drop(&mut self) {
+        info!("Verifying flash remains unmodified");
+        if !self.is_golden() {
+            warn!("ROM seems to be in a different state at finish; restoring original");
+            if let Err(e) = self.ensure_golden() {
+                error!("Failed to write back golden image: {:?}", e);
+            }
+        }
+    }
+}
+
+/// RAII handle for setting write protect in either hardware or software.
+///
+/// Given an instance, the state of either write protect can be modified by calling
+/// `set` or `push`. When it goes out of scope, the write protects will be returned
+/// to the state they had then it was created.
+///
+/// The lifetime `'p` on this struct is the parent state it derives from; `'static`
+/// implies it is derived from hardware, while anything else is part of a stack
+/// created by `push`ing states. An initial state is always static, and the stack
+/// forms a lifetime chain `'static -> 'p -> 'p1 -> ... -> 'pn`.
+pub struct WriteProtectState<'a, 'p> {
+    /// The parent state this derives from.
+    ///
+    /// If it's a root (gotten via `from_hardware`), then this is Hardware and the
+    /// liveness flag will be reset on drop.
+    initial: InitialState<'p>,
+    // Tuples are (hardware, software)
+    current: (bool, bool),
+    cmd: &'a FlashromCmd,
+}
+
+enum InitialState<'p> {
+    Hardware(bool, bool),
+    Previous(&'p WriteProtectState<'p, 'p>),
+}
+
+impl InitialState<'_> {
+    fn get_target(&self) -> (bool, bool) {
+        match self {
+            InitialState::Hardware(hw, sw) => (*hw, *sw),
+            InitialState::Previous(s) => s.current,
+        }
+    }
+}
+
+impl<'a> WriteProtectState<'a, 'static> {
+    /// Initialize a state from the current state of the hardware.
+    ///
+    /// Panics if there is already a live state derived from hardware. In such a situation the
+    /// new state must be derived from the live one, or the live one must be dropped first.
+    pub fn from_hardware(cmd: &'a FlashromCmd) -> Result<Self, String> {
+        let mut lock = Self::get_liveness_lock()
+            .lock()
+            .expect("Somebody panicked during WriteProtectState init from hardware");
+        if *lock {
+            drop(lock); // Don't poison the lock
+            panic!("Attempted to create a new WriteProtectState when one is already live");
+        }
+
+        let hw = Self::get_hw(cmd)?;
+        let sw = Self::get_sw(cmd)?;
+        info!("Initial hardware write protect: HW={} SW={}", hw, sw);
+
+        *lock = true;
+        Ok(WriteProtectState {
+            initial: InitialState::Hardware(hw, sw),
+            current: (hw, sw),
+            cmd,
+        })
+    }
+
+    /// Get the actual hardware write protect state.
+    fn get_hw(cmd: &FlashromCmd) -> Result<bool, String> {
+        if cmd.fc.can_control_hw_wp() {
+            super::utils::get_hardware_wp()
+        } else {
+            Ok(false)
+        }
+    }
+
+    /// Get the actual software write protect state.
+    fn get_sw(cmd: &FlashromCmd) -> Result<bool, String> {
+        flashrom::wp_status(cmd, true)
+    }
+}
+
+impl<'a, 'p> WriteProtectState<'a, 'p> {
+    /// Return true if the current programmer supports setting the hardware
+    /// write protect.
+    ///
+    /// If false, calls to set_hw() will do nothing.
+    pub fn can_control_hw_wp(&self) -> bool {
+        self.cmd.fc.can_control_hw_wp()
+    }
+
+    /// Set the software write protect.
+    pub fn set_sw(&mut self, enable: bool) -> Result<&mut Self, String> {
+        info!("request={}, current={}", enable, self.current.1);
+        if self.current.1 != enable {
+            flashrom::wp_toggle(self.cmd, /* en= */ enable)?;
+            self.current.1 = enable;
+        }
+        Ok(self)
+    }
+
+    /// Set the hardware write protect.
+    pub fn set_hw(&mut self, enable: bool) -> Result<&mut Self, String> {
+        if self.current.0 != enable {
+            if self.can_control_hw_wp() {
+                super::utils::toggle_hw_wp(/* dis= */ !enable)?;
+                self.current.0 = enable;
+            } else if enable {
+                info!(
+                    "Ignoring attempt to enable hardware WP with {:?} programmer",
+                    self.cmd.fc
+                );
+            }
+        }
+        Ok(self)
+    }
+
+    /// Stack a new write protect state on top of the current one.
+    ///
+    /// This is useful if you need to temporarily make a change to write protection:
+    ///
+    /// ```no_run
+    /// # fn main() -> Result<(), String> {
+    /// # let cmd: flashrom::FlashromCmd = unimplemented!();
+    /// let wp = flashrom_tester::tester::WriteProtectState::from_hardware(&cmd)?;
+    /// {
+    ///     let mut wp = wp.push();
+    ///     wp.set_sw(false)?;
+    ///     // Do something with software write protect disabled
+    /// }
+    /// // Now software write protect returns to its original state, even if
+    /// // set_sw() failed.
+    /// # Ok(())
+    /// # }
+    /// ```
+    ///
+    /// This returns a new state which restores the original when it is dropped- the new state
+    /// refers to the old, so the compiler enforces that states are disposed of in the reverse
+    /// order of their creation and correctly restore the original state.
+    pub fn push<'p1>(&'p1 self) -> WriteProtectState<'a, 'p1> {
+        WriteProtectState {
+            initial: InitialState::Previous(self),
+            current: self.current,
+            cmd: self.cmd,
+        }
+    }
+
+    fn get_liveness_lock() -> &'static Mutex<bool> {
+        static INIT: std::sync::Once = std::sync::Once::new();
+        /// Value becomes true when there is a live WriteProtectState derived `from_hardware`,
+        /// blocking duplicate initialization.
+        ///
+        /// This is required because hardware access is not synchronized; it's possible to leave the
+        /// hardware in an unintended state by creating a state handle from it, modifying the state,
+        /// creating another handle from the hardware then dropping the first handle- then on drop
+        /// of the second handle it will restore the state to the modified one rather than the initial.
+        ///
+        /// This flag ensures that a duplicate root state cannot be created.
+        ///
+        /// This is a Mutex<bool> rather than AtomicBool because acquiring the flag needs to perform
+        /// several operations that may themselves fail- acquisitions must be fully synchronized.
+        static mut LIVE_FROM_HARDWARE: MaybeUninit<Mutex<bool>> = MaybeUninit::uninit();
+
+        unsafe {
+            INIT.call_once(|| {
+                LIVE_FROM_HARDWARE.as_mut_ptr().write(Mutex::new(false));
+            });
+            &*LIVE_FROM_HARDWARE.as_ptr()
+        }
+    }
+
+    /// Reset the hardware to what it was when this state was created, reporting errors.
+    ///
+    /// This behaves exactly like allowing a state to go out of scope, but it can return
+    /// errors from that process rather than panicking.
+    pub fn close(mut self) -> Result<(), String> {
+        unsafe {
+            let out = self.drop_internal();
+            // We just ran drop, don't do it again
+            std::mem::forget(self);
+            out
+        }
+    }
+
+    /// Internal Drop impl.
+    ///
+    /// This is unsafe because it effectively consumes self when clearing the
+    /// liveness lock. Callers must be able to guarantee that self will be forgotten
+    /// if the state was constructed from hardware in order to uphold the liveness
+    /// invariant (that only a single state constructed from hardware exists at any
+    /// time).
+    unsafe fn drop_internal(&mut self) -> Result<(), String> {
+        let lock = match self.initial {
+            InitialState::Hardware(_, _) => Some(
+                Self::get_liveness_lock()
+                    .lock()
+                    .expect("Somebody panicked during WriteProtectState drop from hardware"),
+            ),
+            _ => None,
+        };
+        let (hw, sw) = self.initial.get_target();
+
+        fn enable_str(enable: bool) -> &'static str {
+            if enable {
+                "en"
+            } else {
+                "dis"
+            }
+        }
+
+        // Toggle both protects back to their initial states.
+        // Software first because we can't change it once hardware is enabled.
+        if sw != self.current.1 {
+            // Is the hw wp currently enabled?
+            if self.current.0 {
+                super::utils::toggle_hw_wp(/* dis= */ true).map_err(|e| {
+                    format!(
+                        "Failed to {}able hardware write protect: {}",
+                        enable_str(false),
+                        e
+                    )
+                })?;
+            }
+            flashrom::wp_toggle(self.cmd, /* en= */ sw).map_err(|e| {
+                format!(
+                    "Failed to {}able software write protect: {}",
+                    enable_str(sw),
+                    e
+                )
+            })?;
+        }
+
+        assert!(
+            self.cmd.fc.can_control_hw_wp() || (!self.current.0 && !hw),
+            "HW WP must be disabled if it cannot be controlled"
+        );
+        if hw != self.current.0 {
+            super::utils::toggle_hw_wp(/* dis= */ !hw).map_err(|e| {
+                format!(
+                    "Failed to {}able hardware write protect: {}",
+                    enable_str(hw),
+                    e
+                )
+            })?;
+        }
+
+        if let Some(mut lock) = lock {
+            // Initial state was constructed via from_hardware, now we can clear the liveness
+            // lock since reset is complete.
+            *lock = false;
+        }
+        Ok(())
+    }
+}
+
+impl<'a, 'p> Drop for WriteProtectState<'a, 'p> {
+    /// Sets both write protects to the state they had when this state was created.
+    ///
+    /// Panics on error because there is no mechanism to report errors in Drop.
+    fn drop(&mut self) {
+        unsafe { self.drop_internal() }.expect("Error while dropping WriteProtectState")
+    }
+}
+
+pub trait TestCase {
+    fn get_name(&self) -> &str;
+    fn expected_result(&self) -> TestConclusion;
+    fn run(&self, env: &mut TestEnv) -> TestResult;
+}
+
+impl<S: AsRef<str>, F: Fn(&mut TestEnv) -> TestResult> TestCase for (S, F) {
+    fn get_name(&self) -> &str {
+        self.0.as_ref()
+    }
+
+    fn expected_result(&self) -> TestConclusion {
+        TestConclusion::Pass
+    }
+
+    fn run(&self, env: &mut TestEnv) -> TestResult {
+        (self.1)(env)
+    }
+}
+
+impl<T: TestCase + ?Sized> TestCase for &T {
+    fn get_name(&self) -> &str {
+        (*self).get_name()
+    }
+
+    fn expected_result(&self) -> TestConclusion {
+        (*self).expected_result()
+    }
+
+    fn run(&self, env: &mut TestEnv) -> TestResult {
+        (*self).run(env)
+    }
+}
+
+#[allow(dead_code)]
+#[derive(Copy, Clone, PartialEq, Debug)]
+pub enum TestConclusion {
+    Pass,
+    Fail,
+    UnexpectedPass,
+    UnexpectedFail,
+}
+
+pub struct ReportMetaData {
+    pub chip_name: String,
+    pub os_release: String,
+    pub system_info: String,
+    pub bios_info: String,
+}
+
+fn decode_test_result(res: TestResult, con: TestConclusion) -> (TestConclusion, Option<TestError>) {
+    use TestConclusion::*;
+
+    match (res, con) {
+        (Ok(_), Fail) => (UnexpectedPass, None),
+        (Err(e), Pass) => (UnexpectedFail, Some(e)),
+        _ => (Pass, None),
+    }
+}
+
+pub fn run_all_tests<T, TS>(
+    chip: FlashChip,
+    cmd: &FlashromCmd,
+    ts: TS,
+) -> Vec<(String, (TestConclusion, Option<TestError>))>
+where
+    T: TestCase + Copy,
+    TS: IntoIterator<Item = T>,
+{
+    let mut env = TestEnv::create(chip, cmd).expect("Failed to set up test environment");
+
+    let mut results = Vec::new();
+    for t in ts {
+        let result = decode_test_result(env.run_test(t), t.expected_result());
+        results.push((t.get_name().into(), result));
+    }
+    results
+}
+
+#[derive(Debug, PartialEq, Clone, Copy)]
+pub enum OutputFormat {
+    Pretty,
+    Json,
+}
+
+impl std::str::FromStr for OutputFormat {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        use OutputFormat::*;
+
+        if s.eq_ignore_ascii_case("pretty") {
+            Ok(Pretty)
+        } else if s.eq_ignore_ascii_case("json") {
+            Ok(Json)
+        } else {
+            Err(())
+        }
+    }
+}
+
+pub fn collate_all_test_runs(
+    truns: &[(String, (TestConclusion, Option<TestError>))],
+    meta_data: ReportMetaData,
+    format: OutputFormat,
+) {
+    match format {
+        OutputFormat::Pretty => {
+            println!();
+            println!("  =============================");
+            println!("  =====  AVL qual RESULTS  ====");
+            println!("  =============================");
+            println!();
+            println!("  %---------------------------%");
+            println!("   os release: {}", meta_data.os_release);
+            println!("   chip name: {}", meta_data.chip_name);
+            println!("   system info: \n{}", meta_data.system_info);
+            println!("   bios info: \n{}", meta_data.bios_info);
+            println!("  %---------------------------%");
+            println!();
+
+            for trun in truns.iter() {
+                let (name, (result, error)) = trun;
+                if *result != TestConclusion::Pass {
+                    println!(
+                        " {} {}",
+                        style!(format!(" <+> {} test:", name), types::BOLD),
+                        style_dbg!(result, types::RED)
+                    );
+                    match error {
+                        None => {}
+                        Some(e) => info!(" - {} failure details:\n{}", name, e.to_string()),
+                    };
+                } else {
+                    println!(
+                        " {} {}",
+                        style!(format!(" <+> {} test:", name), types::BOLD),
+                        style_dbg!(result, types::GREEN)
+                    );
+                }
+            }
+            println!();
+        }
+        OutputFormat::Json => {
+            use serde_json::{Map, Value};
+
+            let mut all_pass = true;
+            let mut tests = Map::<String, Value>::new();
+            for (name, (result, error)) in truns {
+                let passed = *result == TestConclusion::Pass;
+                all_pass &= passed;
+
+                let error = match error {
+                    Some(e) => Value::String(format!("{:#?}", e)),
+                    None => Value::Null,
+                };
+
+                assert!(
+                    !tests.contains_key(name),
+                    "Found multiple tests named {:?}",
+                    name
+                );
+                tests.insert(
+                    name.into(),
+                    json!({
+                        "pass": passed,
+                        "error": error,
+                    }),
+                );
+            }
+
+            let json = json!({
+                "pass": all_pass,
+                "metadata": {
+                    "os_release": meta_data.os_release,
+                    "chip_name": meta_data.chip_name,
+                    "system_info": meta_data.system_info,
+                    "bios_info": meta_data.bios_info,
+                },
+                "tests": tests,
+            });
+            println!("{:#}", json);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    #[test]
+    fn decode_test_result() {
+        use super::decode_test_result;
+        use super::TestConclusion::*;
+
+        let (result, err) = decode_test_result(Ok(()), Pass);
+        assert_eq!(result, Pass);
+        assert!(err.is_none());
+
+        let (result, err) = decode_test_result(Ok(()), Fail);
+        assert_eq!(result, UnexpectedPass);
+        assert!(err.is_none());
+
+        let (result, err) = decode_test_result(Err("broken".into()), Pass);
+        assert_eq!(result, UnexpectedFail);
+        assert!(err.is_some());
+
+        let (result, err) = decode_test_result(Err("broken".into()), Fail);
+        assert_eq!(result, Pass);
+        assert!(err.is_none());
+    }
+
+    #[test]
+    fn output_format_round_trip() {
+        use super::OutputFormat::{self, *};
+
+        assert_eq!(format!("{:?}", Pretty).parse::<OutputFormat>(), Ok(Pretty));
+        assert_eq!(format!("{:?}", Json).parse::<OutputFormat>(), Ok(Json));
+    }
+}