From f4240cc8a4545e4d2e2f926a72aa911d5373ab95 Mon Sep 17 00:00:00 2001 From: Eric Smith Date: Thu, 15 Sep 2016 02:00:29 -0600 Subject: Add optional SEED=n command line option to Makefile, and -S n command line option to test scripts, for deterministic regression tests. --- tests/asicworld/run-test.sh | 14 +++++++++++++- tests/bram/generate.py | 19 ++++++++++++++++--- tests/bram/run-test.sh | 17 ++++++++++++++++- tests/fsm/generate.py | 12 +++++++++++- tests/fsm/run-test.sh | 14 +++++++++++++- tests/hana/run-test.sh | 14 +++++++++++++- tests/memories/run-test.sh | 13 ++++++++++++- tests/realmath/generate.py | 12 +++++++++++- tests/realmath/run-test.sh | 16 ++++++++++++++-- tests/share/generate.py | 12 +++++++++++- tests/share/run-test.sh | 14 +++++++++++++- tests/simple/run-test.sh | 13 ++++++++++++- tests/tools/autotest.mk | 7 ++++++- 13 files changed, 161 insertions(+), 16 deletions(-) (limited to 'tests') diff --git a/tests/asicworld/run-test.sh b/tests/asicworld/run-test.sh index 24983f1a5..d5708c456 100755 --- a/tests/asicworld/run-test.sh +++ b/tests/asicworld/run-test.sh @@ -1,2 +1,14 @@ #!/bin/bash -exec ${MAKE:-make} -f ../tools/autotest.mk EXTRA_FLAGS="-e" *.v + +OPTIND=1 +seed="" # default to no seed specified +while getopts "S:" opt +do + case "$opt" in + S) arg="${OPTARG#"${OPTARG%%[![:space:]]*}"}" # remove leading space + seed="SEED=$arg" ;; + esac +done +shift "$((OPTIND-1))" + +exec ${MAKE:-make} -f ../tools/autotest.mk $seed EXTRA_FLAGS="-e" *.v diff --git a/tests/bram/generate.py b/tests/bram/generate.py index 05a7ed027..cab81255c 100644 --- a/tests/bram/generate.py +++ b/tests/bram/generate.py @@ -1,11 +1,11 @@ #!/usr/bin/env python3 +import argparse import os import sys import random debug_mode = False -seed = (int(os.times()[4]*100) + os.getpid()) % 900000 + 100000 def create_bram(dsc_f, sim_f, ref_f, tb_f, k1, k2, or_next): while True: @@ -243,10 +243,23 @@ def create_bram(dsc_f, sim_f, ref_f, tb_f, k1, k2, or_next): print(" end", file=tb_f) print("endmodule", file=tb_f) -print("Rng seed: %d" % seed) +parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('-S', '--seed', type = int, help = 'seed for PRNG') +parser.add_argument('-c', '--count', type = int, default = 5, help = 'number of test cases to generate') +parser.add_argument('-d', '--debug', action='store_true') +args = parser.parse_args() + +debug_mode = args.debug + +if args.seed is not None: + seed = args.seed +else: + seed = (int(os.times()[4]*100) + os.getpid()) % 900000 + 100000 + +print("PRNG seed: %d" % seed) random.seed(seed) -for k1 in range(5): +for k1 in range(args.count): dsc_f = open("temp/brams_%02d.txt" % k1, "w") sim_f = open("temp/brams_%02d.v" % k1, "w") ref_f = open("temp/brams_%02d_ref.v" % k1, "w") diff --git a/tests/bram/run-test.sh b/tests/bram/run-test.sh index f0bf0131e..a30416787 100755 --- a/tests/bram/run-test.sh +++ b/tests/bram/run-test.sh @@ -4,11 +4,26 @@ # MAKE="make -j8" time bash -c 'for ((i=0; i<100; i++)); do echo "-- $i --"; bash run-test.sh || exit 1; done' set -e + +OPTIND=1 +count=5 +seed="" # default to no seed specified +debug="" +while getopts "c:S:" opt +do + case "$opt" in + c) count="$OPTARG" ;; + d) debug="-d" ;; + S) seed="-S $OPTARG" ;; + esac +done +shift "$((OPTIND-1))" + rm -rf temp mkdir -p temp echo "generating tests.." -python3 generate.py +python3 generate.py $debug -c $count $seed { echo -n "all:" diff --git a/tests/fsm/generate.py b/tests/fsm/generate.py index 8757d4741..c8eda0cd1 100644 --- a/tests/fsm/generate.py +++ b/tests/fsm/generate.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse import sys import random from contextlib import contextmanager @@ -30,7 +31,16 @@ def random_expr(variables): return "%d'd%s" % (bits, random.randint(0, 2**bits-1)) raise AssertionError -for idx in range(50): +parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('-S', '--seed', type = int, help = 'seed for PRNG') +parser.add_argument('-c', '--count', type = int, default = 50, help = 'number of test cases to generate') +args = parser.parse_args() + +if args.seed is not None: + print("PRNG seed: %d" % args.seed) + random.seed(args.seed) + +for idx in range(args.count): with open('temp/uut_%05d.v' % idx, 'w') as f: with redirect_stdout(f): rst2 = random.choice([False, True]) diff --git a/tests/fsm/run-test.sh b/tests/fsm/run-test.sh index 423892334..cf506470d 100755 --- a/tests/fsm/run-test.sh +++ b/tests/fsm/run-test.sh @@ -5,10 +5,22 @@ set -e +OPTIND=1 +count=100 +seed="" # default to no seed specified +while getopts "c:S:" opt +do + case "$opt" in + c) count="$OPTARG" ;; + S) seed="-S $OPTARG" ;; + esac +done +shift "$((OPTIND-1))" + rm -rf temp mkdir -p temp echo "generating tests.." -python3 generate.py +python3 generate.py -c $count $seed { all_targets="all_targets:" diff --git a/tests/hana/run-test.sh b/tests/hana/run-test.sh index fb766eec9..878c80b3f 100755 --- a/tests/hana/run-test.sh +++ b/tests/hana/run-test.sh @@ -1,2 +1,14 @@ #!/bin/bash -exec ${MAKE:-make} -f ../tools/autotest.mk EXTRA_FLAGS="-l hana_vlib.v -n 300 -e" test_*.v + +OPTIND=1 +seed="" # default to no seed specified +while getopts "S:" opt +do + case "$opt" in + S) arg="${OPTARG#"${OPTARG%%[![:space:]]*}"}" # remove leading space + seed="SEED=$arg" ;; + esac +done +shift "$((OPTIND-1))" + +exec ${MAKE:-make} -f ../tools/autotest.mk $seed EXTRA_FLAGS="-l hana_vlib.v -n 300 -e" test_*.v diff --git a/tests/memories/run-test.sh b/tests/memories/run-test.sh index c3b196188..734a96682 100755 --- a/tests/memories/run-test.sh +++ b/tests/memories/run-test.sh @@ -1,7 +1,18 @@ #!/bin/bash set -e -bash ../tools/autotest.sh -G *.v + +OPTIND=1 +seed="" # default to no seed specified +while getopts "S:" opt +do + case "$opt" in + S) seed="-S $OPTARG" ;; + esac +done +shift "$((OPTIND-1))" + +bash ../tools/autotest.sh $seed -G *.v for f in `egrep -l 'expect-(wr|rd)-ports' *.v`; do echo -n "Testing expectations for $f .." diff --git a/tests/realmath/generate.py b/tests/realmath/generate.py index 19d01c7c6..2bedf38e4 100644 --- a/tests/realmath/generate.py +++ b/tests/realmath/generate.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse import sys import random from contextlib import contextmanager @@ -36,7 +37,16 @@ def random_expression(depth = 3, maxparam = 0): return op + '(' + recursion() + ', ' + recursion() + ')' raise -for idx in range(100): +parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('-S', '--seed', type = int, help = 'seed for PRNG') +parser.add_argument('-c', '--count', type = int, default = 100, help = 'number of test cases to generate') +args = parser.parse_args() + +if args.seed is not None: + print("PRNG seed: %d" % args.seed) + random.seed(args.seed) + +for idx in range(args.count): with open('temp/uut_%05d.v' % idx, 'w') as f: with redirect_stdout(f): print('module uut_%05d(output [63:0] %s);\n' % (idx, ', '.join(['y%02d' % i for i in range(100)]))) diff --git a/tests/realmath/run-test.sh b/tests/realmath/run-test.sh index f1ec5476b..e1a36c694 100755 --- a/tests/realmath/run-test.sh +++ b/tests/realmath/run-test.sh @@ -1,14 +1,26 @@ #!/bin/bash set -e +OPTIND=1 +count=100 +seed="" # default to no seed specified +while getopts "c:S:" opt +do + case "$opt" in + c) count="$OPTARG" ;; + S) seed="-S $OPTARG" ;; + esac +done +shift "$((OPTIND-1))" + rm -rf temp mkdir -p temp echo "generating tests.." -python3 generate.py +python3 generate.py -c $count $seed cd temp echo "running tests.." -for ((i = 0; i < 100; i++)); do +for ((i = 0; i < $count; i++)); do echo -n "[$i]" idx=$( printf "%05d" $i ) ../../../yosys -qq uut_${idx}.ys diff --git a/tests/share/generate.py b/tests/share/generate.py index 01a19a8d9..7e87bd648 100644 --- a/tests/share/generate.py +++ b/tests/share/generate.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import argparse import sys import random from contextlib import contextmanager @@ -21,7 +22,16 @@ def maybe_plus_x(expr): else: return expr -for idx in range(100): +parser = argparse.ArgumentParser(formatter_class = argparse.ArgumentDefaultsHelpFormatter) +parser.add_argument('-S', '--seed', type = int, help = 'seed for PRNG') +parser.add_argument('-c', '--count', type = int, default = 100, help = 'number of test cases to generate') +args = parser.parse_args() + +if args.seed is not None: + print("PRNG seed: %d" % args.seed) + random.seed(args.seed) + +for idx in range(args.count): with open('temp/uut_%05d.v' % idx, 'w') as f: with redirect_stdout(f): if random.choice(['bin', 'uni']) == 'bin': diff --git a/tests/share/run-test.sh b/tests/share/run-test.sh index 18dbbc279..1bcd8e423 100755 --- a/tests/share/run-test.sh +++ b/tests/share/run-test.sh @@ -5,10 +5,22 @@ set -e +OPTIND=1 +count=100 +seed="" # default to no seed specified +while getopts "c:S:" opt +do + case "$opt" in + c) count="$OPTARG" ;; + S) seed="-S $OPTARG" ;; + esac +done +shift "$((OPTIND-1))" + rm -rf temp mkdir -p temp echo "generating tests.." -python3 generate.py +python3 generate.py -c $count $seed echo "running tests.." for i in $( ls temp/*.ys | sed 's,[^0-9],,g; s,^0*\(.\),\1,g;' ); do diff --git a/tests/simple/run-test.sh b/tests/simple/run-test.sh index 6531d51ae..aaa1cf940 100755 --- a/tests/simple/run-test.sh +++ b/tests/simple/run-test.sh @@ -1,9 +1,20 @@ #!/bin/bash +OPTIND=1 +seed="" # default to no seed specified +while getopts "S:" opt +do + case "$opt" in + S) arg="${OPTARG#"${OPTARG%%[![:space:]]*}"}" # remove leading space + seed="SEED=$arg" ;; + esac +done +shift "$((OPTIND-1))" + # check for Icarus Verilog if ! which iverilog > /dev/null ; then echo "$0: Error: Icarus Verilog 'iverilog' not found." exit 1 fi -exec ${MAKE:-make} -f ../tools/autotest.mk *.v +exec ${MAKE:-make} -f ../tools/autotest.mk $seed *.v diff --git a/tests/tools/autotest.mk b/tests/tools/autotest.mk index f65002cef..c68678929 100644 --- a/tests/tools/autotest.mk +++ b/tests/tools/autotest.mk @@ -1,8 +1,13 @@ EXTRA_FLAGS= +SEED= + +ifneq ($(strip $(SEED)),) +SEEDOPT=-S$(SEED) +endif $(MAKECMDGOALS): - @$(basename $(MAKEFILE_LIST)).sh -G -j $(EXTRA_FLAGS) $@ + @$(basename $(MAKEFILE_LIST)).sh -G -j $(SEEDOPT) $(EXTRA_FLAGS) $@ .PHONY: $(MAKECMDGOALS) -- cgit v1.2.3