Merge pull request #418 from timstobal/recomplicate-initial

Fix re-complication of initial state

Author: rexmas

proptest-state-machine/CHANGELOG.md

@@ -9,3 +9,4 @@
 - Removed the limit of number of transitions that can be deleted in shrinking that depended on the number the of transitions given to `prop_state_machine!` or `ReferenceStateMachine::sequential_strategy`.
 - Fixed state-machine macro's inability to handle missing config
 - Fixed logging of state machine transitions to be enabled when verbose config is >= 1. The "std" feature is added to proptest-state-machine as a default feature that allows to switch the logging off in non-std env.
+- Fixed an issue where after simplification of the initial state causes the test to succeed, the initial state would not be re-complicated - causing the test to report a succeeding input as the simplest failing input.

proptest-state-machine/proptest-regressions/strategy.txt

@@ -0,0 +1,8 @@
+# Seeds for failure cases proptest has generated in the past. It is
+# automatically read and these particular cases re-run before any
+# novel cases are generated.
+#
+# It is recommended to check this file in to source control so that
+# everyone who runs the test benefits from these saved cases.
+cc 2e5fb0b2e70b08bd8a567bf4d9ae66ee2478d832d6bdfb9670f5b98203e78178 # shrinks to seed = [120, 233, 214, 148, 193, 171, 178, 64, 157, 62, 78, 165, 215, 79, 177, 175, 171, 202, 51, 93, 79, 238, 39, 104, 174, 79, 152, 255, 45, 174, 27, 168]
+cc 39927f23e5f67ac32c5219c226c49d87e3e3c995a73cd969d72fbcdf52ac895b # shrinks to seed = [0, 10, 244, 19, 249, 125, 161, 150, 61, 56, 77, 245, 12, 228, 187, 180, 148, 61, 17, 32, 189, 118, 70, 47, 147, 210, 94, 127, 210, 23, 128, 75]

proptest-state-machine/src/strategy.rs

@@ -357,6 +357,7 @@ impl<
                     // Store the valid initial state
                     self.last_valid_initial_state =
                         self.initial_state.current();
+                    self.last_shrink = Some(self.shrink);
                     return true;
                 } else {
                     // If the shrink is not acceptable, clear it out
@@ -533,7 +534,6 @@ impl<
                 false
             }
             Some(InitialState) => {
-                self.last_shrink = None;
                 if self.initial_state.complicate()
                     && self.check_acceptable(None)
                 {
@@ -752,4 +752,114 @@ mod test {
             }
         }
     }
+
+    /// A tests that verifies that the strategy finds a simplest failing case, and
+    /// that this simplest failing case is ultimately reported by the test runner,
+    /// as opposed to reporting input that actually passes the test.
+    ///
+    /// This module defines a state machine test that is designed to fail.
+    /// The reference state machine consists of a lower bound the acceptable value
+    /// of a transition. And the test fails if an unacceptably low transition
+    /// value is observed, given the reference state's limit.
+    ///
+    /// This intentionally-failing state machine test is then run inside a proptest
+    /// to verify that it reports a simplest failing input when it fails.
+    mod find_simplest_failure {
+        use proptest::prelude::*;
+        use proptest::strategy::BoxedStrategy;
+        use proptest::test_runner::TestRng;
+        use proptest::{
+            collection,
+            strategy::Strategy,
+            test_runner::{Config, TestError, TestRunner},
+        };
+
+        use crate::{ReferenceStateMachine, StateMachineTest};
+
+        const MIN_TRANSITION: u32 = 10;
+        const MAX_TRANSITION: u32 = 20;
+
+        const MIN_LIMIT: u32 = 2;
+        const MAX_LIMIT: u32 = 50;
+
+        #[derive(Debug, Default, Clone)]
+        struct FailIfLessThan(u32);
+        impl ReferenceStateMachine for FailIfLessThan {
+            type State = Self;
+            type Transition = u32;
+
+            fn init_state() -> BoxedStrategy<Self> {
+                (MIN_LIMIT..MAX_LIMIT).prop_map(FailIfLessThan).boxed()
+            }
+
+            fn transitions(_: &Self::State) -> BoxedStrategy<u32> {
+                (MIN_TRANSITION..MAX_TRANSITION).boxed()
+            }
+
+            fn apply(state: Self::State, _: &Self::Transition) -> Self::State {
+                state
+            }
+        }
+
+        /// Defines a test that is intended to fail, so that we can inspect the
+        /// failing input.
+        struct FailIfLessThanTest;
+        impl StateMachineTest for FailIfLessThanTest {
+            type SystemUnderTest = ();
+            type Reference = FailIfLessThan;
+
+            fn init_test(ref_state: &FailIfLessThan) {
+                println!();
+                println!("starting {ref_state:?}");
+            }
+
+            fn apply(
+                (): Self::SystemUnderTest,
+                ref_state: &FailIfLessThan,
+                transition: u32,
+            ) -> Self::SystemUnderTest {
+                // Fail on any transition that is less than the ref state's limit.
+                let FailIfLessThan(limit) = ref_state;
+                println!("{transition} < {}?", limit);
+                if transition < ref_state.0 {
+                    panic!("{transition} < {}", limit);
+                }
+            }
+        }
+
+        proptest! {
+            #[test]
+            fn test_returns_simplest_failure(
+                seed in collection::vec(any::<u8>(), 32).no_shrink()) {
+
+                // We need to explicitly run create a runner so that we can
+                // inspect the output, and determine if it does return an input that
+                // should fail, and is minimal.
+                let mut runner = TestRunner::new_with_rng(
+                    Config::default(), TestRng::from_seed(Default::default(), &seed));
+                let result = runner.run(
+                    &FailIfLessThan::sequential_strategy(10..50_usize),
+                    |(ref_state, transitions)| {
+                        Ok(FailIfLessThanTest::test_sequential(
+                            Default::default(),
+                            ref_state,
+                            transitions,
+                        ))
+                    },
+                );
+                if let Err(TestError::Fail(
+                    _,
+                    (FailIfLessThan(limit), transitions),
+                )) = result
+                {
+                    assert_eq!(transitions.len(), 1, "The minimal failing case should be ");
+                    assert_eq!(limit, MIN_TRANSITION + 1);
+                    assert!(transitions[0] < limit);
+                } else {
+                    prop_assume!(false,
+                        "If the state machine doesn't fail as intended, we need a case that fails.");
+                }
+            }
+        }
+    }
 }