tests: Add run-tests.py --shuffle-test

This optional argument can be used to randomize the order in which the
test cases are run. This can provide more coverage on testing
interactions of common use cases in various different sequences. Such
issues have already been found even with the fixed order of test cases,
but being able to reorder the tests makes this more efficient.

Signed-hostap: Jouni Malinen <j@w1.fi>

tests/hwsim/run-tests.py

@@ -121,6 +121,9 @@ def main():
                         help='collect tracing per test case (in log directory)')
     parser.add_argument('-D', action='store_true', dest='dmesg',
                         help='collect dmesg per test case (in log directory)')
+    parser.add_argument('--shuffle-tests', action='store_true',
+                        dest='shuffle_tests',
+                        help='Shuffle test cases to randomize order')
     parser.add_argument('-f', dest='testmodules', metavar='<test module>',
                         help='execute only tests from these test modules',
                         type=str, choices=[[]] + test_modules, nargs='+')
@@ -228,6 +231,10 @@ def main():
             name = t.__name__.replace('test_', '', 1)
             report(conn, False, args.build, args.commit, run, name, 'NOTRUN', 0)
 
+    if args.shuffle_tests:
+        from random import shuffle
+        shuffle(tests_to_run)
+
     for t in tests_to_run:
         name = t.__name__.replace('test_', '', 1)
         if log_handler: