Add optional SEED=n command line option to Makefile, and -S n command line option to test scripts, for deterministic regression tests.

13 files changed, 161 insertions, 16 deletions

diff --git a/tests/asicworld/run-test.sh b/tests/asicworld/run-test.sh
index 24983f1a5..d5708c456 100755
--- a/tests/asicworld/run-test.sh
+++ b/tests/asicworld/run-test.sh
@@ -1,2 +1,14 @@
 #!/bin/bash
-exec ${MAKE:-make} -f ../tools/autotest.mk EXTRA_FLAGS="-e" *.v
+
+OPTIND=1
+seed=""    # default to no seed specified
+while getopts "S:" opt
+do
+    case "$opt" in
+	S) arg="${OPTARG#"${OPTARG%%[![:space:]]*}"}" # remove leading space
+	   seed="SEED=$arg" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
+exec ${MAKE:-make} -f ../tools/autotest.mk $seed EXTRA_FLAGS="-e" *.v
diff --git a/tests/bram/generate.py b/tests/bram/generate.py
index 05a7ed027..cab81255c 100644
--- a/tests/bram/generate.py
+++ b/tests/bram/generate.py
@@ -1,11 +1,11 @@
 #!/usr/bin/env python3
 
+import argparse
 import os
 import sys
 import random
 
 debug_mode = False
-seed = (int(os.times()[4]*100) + os.getpid()) % 900000 + 100000
 
 def create_bram(dsc_f, sim_f, ref_f, tb_f, k1, k2, or_next):
     while True:
@@ -243,10 +243,23 @@ def create_bram(dsc_f, sim_f, ref_f, tb_f, k1, k2, or_next):
     print("  end", file=tb_f)
     print("endmodule", file=tb_f)
 
-print("Rng seed: %d" % seed)
+parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-S', '--seed',  type = int, help = 'seed for PRNG')
+parser.add_argument('-c', '--count', type = int, default = 5, help = 'number of test cases to generate')
+parser.add_argument('-d', '--debug', action='store_true')
+args = parser.parse_args()
+
+debug_mode = args.debug
+
+if args.seed is not None:
+    seed = args.seed
+else:
+    seed = (int(os.times()[4]*100) + os.getpid()) % 900000 + 100000
+
+print("PRNG seed: %d" % seed)
 random.seed(seed)
 
-for k1 in range(5):
+for k1 in range(args.count):
     dsc_f = open("temp/brams_%02d.txt" % k1, "w")
     sim_f = open("temp/brams_%02d.v" % k1, "w")
     ref_f = open("temp/brams_%02d_ref.v" % k1, "w")
diff --git a/tests/bram/run-test.sh b/tests/bram/run-test.sh
index f0bf0131e..a30416787 100755
--- a/tests/bram/run-test.sh
+++ b/tests/bram/run-test.sh
@@ -4,11 +4,26 @@
 # MAKE="make -j8" time bash -c 'for ((i=0; i<100; i++)); do echo "-- $i --"; bash run-test.sh || exit 1; done'
 
 set -e
+
+OPTIND=1
+count=5
+seed=""    # default to no seed specified
+debug=""
+while getopts "c:S:" opt
+do
+    case "$opt" in
+	c) count="$OPTARG" ;;
+	d) debug="-d" ;;
+	S) seed="-S $OPTARG" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
 rm -rf temp
 mkdir -p temp
 
 echo "generating tests.."
-python3 generate.py
+python3 generate.py $debug -c $count $seed
 
 {
 	echo -n "all:"
diff --git a/tests/fsm/generate.py b/tests/fsm/generate.py
index 8757d4741..c8eda0cd1 100644
--- a/tests/fsm/generate.py
+++ b/tests/fsm/generate.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import argparse
 import sys
 import random
 from contextlib import contextmanager
@@ -30,7 +31,16 @@ def random_expr(variables):
         return "%d'd%s" % (bits, random.randint(0, 2**bits-1))
     raise AssertionError
 
-for idx in range(50):
+parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-S', '--seed',  type = int, help = 'seed for PRNG')
+parser.add_argument('-c', '--count', type = int, default = 50, help = 'number of test cases to generate')
+args = parser.parse_args()
+
+if args.seed is not None:
+    print("PRNG seed: %d" % args.seed)
+    random.seed(args.seed)
+
+for idx in range(args.count):
     with open('temp/uut_%05d.v' % idx, 'w') as f:
         with redirect_stdout(f):
             rst2 = random.choice([False, True])
diff --git a/tests/fsm/run-test.sh b/tests/fsm/run-test.sh
index 423892334..cf506470d 100755
--- a/tests/fsm/run-test.sh
+++ b/tests/fsm/run-test.sh
@@ -5,10 +5,22 @@
 
 set -e
 
+OPTIND=1
+count=100
+seed=""    # default to no seed specified
+while getopts "c:S:" opt
+do
+    case "$opt" in
+	c) count="$OPTARG" ;;
+	S) seed="-S $OPTARG" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
 rm -rf temp
 mkdir -p temp
 echo "generating tests.."
-python3 generate.py
+python3 generate.py -c $count $seed
 
 {
 	all_targets="all_targets:"
diff --git a/tests/hana/run-test.sh b/tests/hana/run-test.sh
index fb766eec9..878c80b3f 100755
--- a/tests/hana/run-test.sh
+++ b/tests/hana/run-test.sh
@@ -1,2 +1,14 @@
 #!/bin/bash
-exec ${MAKE:-make} -f ../tools/autotest.mk EXTRA_FLAGS="-l hana_vlib.v -n 300 -e" test_*.v
+
+OPTIND=1
+seed=""    # default to no seed specified
+while getopts "S:" opt
+do
+    case "$opt" in
+	S) arg="${OPTARG#"${OPTARG%%[![:space:]]*}"}" # remove leading space
+	   seed="SEED=$arg" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
+exec ${MAKE:-make} -f ../tools/autotest.mk $seed EXTRA_FLAGS="-l hana_vlib.v -n 300 -e" test_*.v
diff --git a/tests/memories/run-test.sh b/tests/memories/run-test.sh
index c3b196188..734a96682 100755
--- a/tests/memories/run-test.sh
+++ b/tests/memories/run-test.sh
@@ -1,7 +1,18 @@
 #!/bin/bash
 
 set -e
-bash ../tools/autotest.sh -G *.v
+
+OPTIND=1
+seed=""    # default to no seed specified
+while getopts "S:" opt
+do
+    case "$opt" in
+	S) seed="-S $OPTARG" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
+bash ../tools/autotest.sh $seed -G *.v
 
 for f in `egrep -l 'expect-(wr|rd)-ports' *.v`; do
 	echo -n "Testing expectations for $f .."
diff --git a/tests/realmath/generate.py b/tests/realmath/generate.py
index 19d01c7c6..2bedf38e4 100644
--- a/tests/realmath/generate.py
+++ b/tests/realmath/generate.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import argparse
 import sys
 import random
 from contextlib import contextmanager
@@ -36,7 +37,16 @@ def random_expression(depth = 3, maxparam = 0):
         return op + '(' + recursion() + ', ' + recursion() + ')'
     raise
 
-for idx in range(100):
+parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-S', '--seed',  type = int, help = 'seed for PRNG')
+parser.add_argument('-c', '--count', type = int, default = 100, help = 'number of test cases to generate')
+args = parser.parse_args()
+
+if args.seed is not None:
+    print("PRNG seed: %d" % args.seed)
+    random.seed(args.seed)
+
+for idx in range(args.count):
     with open('temp/uut_%05d.v' % idx, 'w') as f:
         with redirect_stdout(f):
             print('module uut_%05d(output [63:0] %s);\n' % (idx, ', '.join(['y%02d' % i for i in range(100)])))
diff --git a/tests/realmath/run-test.sh b/tests/realmath/run-test.sh
index f1ec5476b..e1a36c694 100755
--- a/tests/realmath/run-test.sh
+++ b/tests/realmath/run-test.sh
@@ -1,14 +1,26 @@
 #!/bin/bash
 set -e
 
+OPTIND=1
+count=100
+seed=""    # default to no seed specified
+while getopts "c:S:" opt
+do
+    case "$opt" in
+	c) count="$OPTARG" ;;
+	S) seed="-S $OPTARG" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
 rm -rf temp
 mkdir -p temp
 echo "generating tests.."
-python3 generate.py
+python3 generate.py -c $count $seed
 
 cd temp
 echo "running tests.."
-for ((i = 0; i < 100; i++)); do
+for ((i = 0; i < $count; i++)); do
 	echo -n "[$i]"
 	idx=$( printf "%05d" $i )
 	../../../yosys -qq uut_${idx}.ys
diff --git a/tests/share/generate.py b/tests/share/generate.py
index 01a19a8d9..7e87bd648 100644
--- a/tests/share/generate.py
+++ b/tests/share/generate.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import argparse
 import sys
 import random
 from contextlib import contextmanager
@@ -21,7 +22,16 @@ def maybe_plus_x(expr):
     else:
         return expr
 
-for idx in range(100):
+parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter)
+parser.add_argument('-S', '--seed',  type = int, help = 'seed for PRNG')
+parser.add_argument('-c', '--count', type = int, default = 100, help = 'number of test cases to generate')
+args = parser.parse_args()
+
+if args.seed is not None:
+    print("PRNG seed: %d" % args.seed)
+    random.seed(args.seed)
+
+for idx in range(args.count):
     with open('temp/uut_%05d.v' % idx, 'w') as f:
         with redirect_stdout(f):
             if random.choice(['bin', 'uni']) == 'bin':
diff --git a/tests/share/run-test.sh b/tests/share/run-test.sh
index 18dbbc279..1bcd8e423 100755
--- a/tests/share/run-test.sh
+++ b/tests/share/run-test.sh
@@ -5,10 +5,22 @@
 
 set -e
 
+OPTIND=1
+count=100
+seed=""    # default to no seed specified
+while getopts "c:S:" opt
+do
+    case "$opt" in
+	c) count="$OPTARG" ;;
+	S) seed="-S $OPTARG" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
 rm -rf temp
 mkdir -p temp
 echo "generating tests.."
-python3 generate.py
+python3 generate.py -c $count $seed
 
 echo "running tests.."
 for i in $( ls temp/*.ys | sed 's,[^0-9],,g; s,^0*\(.\),\1,g;' ); do
diff --git a/tests/simple/run-test.sh b/tests/simple/run-test.sh
index 6531d51ae..aaa1cf940 100755
--- a/tests/simple/run-test.sh
+++ b/tests/simple/run-test.sh
@@ -1,9 +1,20 @@
 #!/bin/bash
 
+OPTIND=1
+seed=""    # default to no seed specified
+while getopts "S:" opt
+do
+    case "$opt" in
+	S) arg="${OPTARG#"${OPTARG%%[![:space:]]*}"}" # remove leading space
+	   seed="SEED=$arg" ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
 # check for Icarus Verilog
 if ! which iverilog > /dev/null ; then
   echo "$0: Error: Icarus Verilog 'iverilog' not found."
   exit 1
 fi
 
-exec ${MAKE:-make} -f ../tools/autotest.mk *.v
+exec ${MAKE:-make} -f ../tools/autotest.mk $seed *.v
diff --git a/tests/tools/autotest.mk b/tests/tools/autotest.mk
index f65002cef..c68678929 100644
--- a/tests/tools/autotest.mk
+++ b/tests/tools/autotest.mk
@@ -1,8 +1,13 @@
 
 EXTRA_FLAGS=
+SEED=
+
+ifneq ($(strip $(SEED)),)
+SEEDOPT=-S$(SEED)
+endif
 
 $(MAKECMDGOALS):
-	@$(basename $(MAKEFILE_LIST)).sh -G -j $(EXTRA_FLAGS) $@
+	@$(basename $(MAKEFILE_LIST)).sh -G -j $(SEEDOPT) $(EXTRA_FLAGS) $@
 
 .PHONY: $(MAKECMDGOALS)