From 3eb1db03773c4f70090751e4388d38d56f74e932 Mon Sep 17 00:00:00 2001 From: Jouni Malinen Date: Tue, 23 Dec 2014 22:25:29 +0200 Subject: [PATCH] tests: Retry failed cases automatically in parallel VM run parallel-vm.py is now retrying failed cases once at the end of the run. If all the failed test cases passed on the second attempt, that is noted in the summary output. Results are also indicated as the exit value from the run: 0 = all cases passed on first run, 1 = some cases failed once, but everything passed after one retry, 2 = some cases failed did not succeed at all. Signed-off-by: Jouni Malinen --- tests/hwsim/vm/parallel-vm.py | 140 ++++++++++++++++++++++++++++------ 1 file changed, 117 insertions(+), 23 deletions(-) diff --git a/tests/hwsim/vm/parallel-vm.py b/tests/hwsim/vm/parallel-vm.py index e23ea80e6..97fb2c2ff 100755 --- a/tests/hwsim/vm/parallel-vm.py +++ b/tests/hwsim/vm/parallel-vm.py @@ -33,6 +33,7 @@ def show_progress(scr): global dir global timestamp global tests + global first_run_failures total_tests = len(tests) @@ -45,10 +46,73 @@ def show_progress(scr): scr.addstr(num_servers + 1, 20, "TOTAL={} STARTED=0 PASS=0 FAIL=0 SKIP=0".format(total_tests)) scr.refresh() + completed_first_pass = False + rerun_tests = [] + while True: running = False + first_running = False updated = False + for i in range(0, num_servers): + if completed_first_pass: + continue + if vm[i]['first_run_done']: + continue + if not vm[i]['proc']: + continue + if vm[i]['proc'].poll() is not None: + vm[i]['proc'] = None + scr.move(i + 1, 10) + scr.clrtoeol() + log = '{}/{}.srv.{}/console'.format(dir, timestamp, i + 1) + with open(log, 'r') as f: + if "Kernel panic" in f.read(): + scr.addstr("kernel panic") + else: + scr.addstr("unexpected exit") + updated = True + continue + + running = True + first_running = True + try: + err = vm[i]['proc'].stderr.read() + vm[i]['err'] += err + except: + pass + + try: + out = vm[i]['proc'].stdout.read() + vm[i]['out'] += out + if "READY" in out or "PASS" in out or "FAIL" in out or "SKIP" in out: + scr.move(i + 1, 10) + scr.clrtoeol() + updated = True + if not tests: + vm[i]['first_run_done'] = True + scr.addstr("completed first round") + continue + else: + name = tests.pop(0) + vm[i]['proc'].stdin.write(name + '\n') + scr.addstr(name) + except: + pass + + if not first_running and not completed_first_pass: + if tests: + raise Exception("Unexpected test cases remaining from first round") + completed_first_pass = True + (started, passed, failed, skipped) = get_results() + for f in failed: + name = f.split(' ')[1] + rerun_tests.append(name) + first_run_failures.append(name) + + for i in range(num_servers): + if not completed_first_pass: + continue if not vm[i]['proc']: continue if vm[i]['proc'].poll() is not None: @@ -72,28 +136,28 @@ def show_progress(scr): pass try: - out = vm[i]['proc'].stdout.read() - if "READY" in out or "PASS" in out or "FAIL" in out or "SKIP" in out: - if not tests: - vm[i]['proc'].stdin.write('\n') - else: - name = tests.pop(0) - vm[i]['proc'].stdin.write(name + '\n') - except: - continue - #print("VM {}: '{}'".format(i, out)) - vm[i]['out'] += out - lines = vm[i]['out'].splitlines() - last = [ l for l in lines if l.startswith('START ') ] - if len(last) > 0: - try: - info = last[-1].split(' ') + ready = False + if vm[i]['first_run_done']: + vm[i]['first_run_done'] = False + ready = True + else: + out = vm[i]['proc'].stdout.read() + vm[i]['out'] += out + if "READY" in out or "PASS" in out or "FAIL" in out or "SKIP" in out: + ready = True + if ready: scr.move(i + 1, 10) scr.clrtoeol() - scr.addstr(info[1]) updated = True - except: - pass + if not rerun_tests: + vm[i]['proc'].stdin.write('\n') + scr.addstr("shutting down") + else: + name = rerun_tests.pop(0) + vm[i]['proc'].stdin.write(name + '\n') + scr.addstr(name + "(*)") + except: + pass if not running: break @@ -111,9 +175,17 @@ def show_progress(scr): for f in failed: scr.addstr(f.split(' ')[1]) scr.addstr(' ') + + scr.move(0, 35) + scr.clrtoeol() + if rerun_tests: + scr.addstr("(RETRY FAILED %d)" % len(rerun_tests)) + elif first_run_failures: + scr.addstr("(RETRY FAILED)") + scr.refresh() - time.sleep(0.5) + time.sleep(0.25) scr.refresh() time.sleep(0.3) @@ -124,6 +196,7 @@ def main(): global dir global timestamp global tests + global first_run_failures if len(sys.argv) < 2: sys.exit("Usage: %s [--codecov] [params..]" % sys.argv[0]) @@ -146,6 +219,7 @@ def main(): codecov_args = [] codecov = False + first_run_failures = [] tests = [] cmd = [ '../run-tests.py', '-L' ] + sys.argv[idx:] lst = subprocess.Popen(cmd, stdout=subprocess.PIPE) @@ -214,6 +288,7 @@ def main(): '--ext', 'srv.%d' % (i + 1), '-i'] + codecov_args + extra_args vm[i] = {} + vm[i]['first_run_done'] = False vm[i]['proc'] = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, @@ -234,10 +309,23 @@ def main(): (started, passed, failed, skipped) = get_results() - if len(failed) > 0: + if first_run_failures: print "Failed test cases:" - for f in failed: - print f.split(' ')[1], + for f in first_run_failures: + print f, + print + double_failed = [] + for f in failed: + name = f.split(' ')[1] + double_failed.append(name) + for test in first_run_failures: + double_failed.remove(test) + if failed and not double_failed: + print "All failed cases passed on retry" + elif double_failed: + print "Failed even on retry:" + for f in double_failed: + print f, print print("TOTAL={} PASS={} FAIL={} SKIP={}".format(len(started), len(passed), len(failed), len(skipped))) print "Logs: " + dir + '/' + str(timestamp) @@ -257,5 +345,11 @@ def main(): subprocess.check_call(['./combine-codecov.sh', logdir]) print "file://%s/index.html" % logdir + if double_failed: + sys.exit(2) + if failed: + sys.exit(1) + sys.exit(0) + if __name__ == "__main__": main()