parallel-vm.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355
  1. #!/usr/bin/env python2
  2. #
  3. # Parallel VM test case executor
  4. # Copyright (c) 2014, Jouni Malinen <j@w1.fi>
  5. #
  6. # This software may be distributed under the terms of the BSD license.
  7. # See README for more details.
  8. import curses
  9. import fcntl
  10. import os
  11. import subprocess
  12. import sys
  13. import time
  14. def get_results():
  15. global vm
  16. started = []
  17. passed = []
  18. failed = []
  19. skipped = []
  20. for i in range(0, num_servers):
  21. lines = vm[i]['out'].splitlines()
  22. started += [ l for l in lines if l.startswith('START ') ]
  23. passed += [ l for l in lines if l.startswith('PASS ') ]
  24. failed += [ l for l in lines if l.startswith('FAIL ') ]
  25. skipped += [ l for l in lines if l.startswith('SKIP ') ]
  26. return (started, passed, failed, skipped)
  27. def show_progress(scr):
  28. global num_servers
  29. global vm
  30. global dir
  31. global timestamp
  32. global tests
  33. global first_run_failures
  34. total_tests = len(tests)
  35. scr.leaveok(1)
  36. scr.addstr(0, 0, "Parallel test execution status", curses.A_BOLD)
  37. for i in range(0, num_servers):
  38. scr.addstr(i + 1, 0, "VM %d:" % (i + 1), curses.A_BOLD)
  39. scr.addstr(i + 1, 10, "starting VM")
  40. scr.addstr(num_servers + 1, 0, "Total:", curses.A_BOLD)
  41. scr.addstr(num_servers + 1, 20, "TOTAL={} STARTED=0 PASS=0 FAIL=0 SKIP=0".format(total_tests))
  42. scr.refresh()
  43. completed_first_pass = False
  44. rerun_tests = []
  45. while True:
  46. running = False
  47. first_running = False
  48. updated = False
  49. for i in range(0, num_servers):
  50. if completed_first_pass:
  51. continue
  52. if vm[i]['first_run_done']:
  53. continue
  54. if not vm[i]['proc']:
  55. continue
  56. if vm[i]['proc'].poll() is not None:
  57. vm[i]['proc'] = None
  58. scr.move(i + 1, 10)
  59. scr.clrtoeol()
  60. log = '{}/{}.srv.{}/console'.format(dir, timestamp, i + 1)
  61. with open(log, 'r') as f:
  62. if "Kernel panic" in f.read():
  63. scr.addstr("kernel panic")
  64. else:
  65. scr.addstr("unexpected exit")
  66. updated = True
  67. continue
  68. running = True
  69. first_running = True
  70. try:
  71. err = vm[i]['proc'].stderr.read()
  72. vm[i]['err'] += err
  73. except:
  74. pass
  75. try:
  76. out = vm[i]['proc'].stdout.read()
  77. vm[i]['out'] += out
  78. if "READY" in out or "PASS" in out or "FAIL" in out or "SKIP" in out:
  79. scr.move(i + 1, 10)
  80. scr.clrtoeol()
  81. updated = True
  82. if not tests:
  83. vm[i]['first_run_done'] = True
  84. scr.addstr("completed first round")
  85. continue
  86. else:
  87. name = tests.pop(0)
  88. vm[i]['proc'].stdin.write(name + '\n')
  89. scr.addstr(name)
  90. except:
  91. pass
  92. if not first_running and not completed_first_pass:
  93. if tests:
  94. raise Exception("Unexpected test cases remaining from first round")
  95. completed_first_pass = True
  96. (started, passed, failed, skipped) = get_results()
  97. for f in failed:
  98. name = f.split(' ')[1]
  99. rerun_tests.append(name)
  100. first_run_failures.append(name)
  101. for i in range(num_servers):
  102. if not completed_first_pass:
  103. continue
  104. if not vm[i]['proc']:
  105. continue
  106. if vm[i]['proc'].poll() is not None:
  107. vm[i]['proc'] = None
  108. scr.move(i + 1, 10)
  109. scr.clrtoeol()
  110. log = '{}/{}.srv.{}/console'.format(dir, timestamp, i + 1)
  111. with open(log, 'r') as f:
  112. if "Kernel panic" in f.read():
  113. scr.addstr("kernel panic")
  114. else:
  115. scr.addstr("completed run")
  116. updated = True
  117. continue
  118. running = True
  119. try:
  120. err = vm[i]['proc'].stderr.read()
  121. vm[i]['err'] += err
  122. except:
  123. pass
  124. try:
  125. ready = False
  126. if vm[i]['first_run_done']:
  127. vm[i]['first_run_done'] = False
  128. ready = True
  129. else:
  130. out = vm[i]['proc'].stdout.read()
  131. vm[i]['out'] += out
  132. if "READY" in out or "PASS" in out or "FAIL" in out or "SKIP" in out:
  133. ready = True
  134. if ready:
  135. scr.move(i + 1, 10)
  136. scr.clrtoeol()
  137. updated = True
  138. if not rerun_tests:
  139. vm[i]['proc'].stdin.write('\n')
  140. scr.addstr("shutting down")
  141. else:
  142. name = rerun_tests.pop(0)
  143. vm[i]['proc'].stdin.write(name + '\n')
  144. scr.addstr(name + "(*)")
  145. except:
  146. pass
  147. if not running:
  148. break
  149. if updated:
  150. (started, passed, failed, skipped) = get_results()
  151. scr.move(num_servers + 1, 10)
  152. scr.clrtoeol()
  153. scr.addstr("{} %".format(int(100.0 * (len(passed) + len(failed) + len(skipped)) / total_tests)))
  154. scr.addstr(num_servers + 1, 20, "TOTAL={} STARTED={} PASS={} FAIL={} SKIP={}".format(total_tests, len(started), len(passed), len(failed), len(skipped)))
  155. if len(failed) > 0:
  156. scr.move(num_servers + 2, 0)
  157. scr.clrtoeol()
  158. scr.addstr("Failed test cases: ")
  159. for f in failed:
  160. scr.addstr(f.split(' ')[1])
  161. scr.addstr(' ')
  162. scr.move(0, 35)
  163. scr.clrtoeol()
  164. if rerun_tests:
  165. scr.addstr("(RETRY FAILED %d)" % len(rerun_tests))
  166. elif first_run_failures:
  167. scr.addstr("(RETRY FAILED)")
  168. scr.refresh()
  169. time.sleep(0.25)
  170. scr.refresh()
  171. time.sleep(0.3)
  172. def main():
  173. global num_servers
  174. global vm
  175. global dir
  176. global timestamp
  177. global tests
  178. global first_run_failures
  179. if len(sys.argv) < 2:
  180. sys.exit("Usage: %s <number of VMs> [--codecov] [params..]" % sys.argv[0])
  181. num_servers = int(sys.argv[1])
  182. if num_servers < 1:
  183. sys.exit("Too small number of VMs")
  184. timestamp = int(time.time())
  185. if len(sys.argv) > 2 and sys.argv[2] == "--codecov":
  186. idx = 3
  187. print "Code coverage - build separate binaries"
  188. logdir = "/tmp/hwsim-test-logs/" + str(timestamp)
  189. os.makedirs(logdir)
  190. subprocess.check_call(['./build-codecov.sh', logdir])
  191. codecov_args = ['--codecov_dir', logdir]
  192. codecov = True
  193. else:
  194. idx = 2
  195. codecov_args = []
  196. codecov = False
  197. first_run_failures = []
  198. tests = []
  199. cmd = [ '../run-tests.py', '-L' ] + sys.argv[idx:]
  200. lst = subprocess.Popen(cmd, stdout=subprocess.PIPE)
  201. for l in lst.stdout.readlines():
  202. name = l.split(' ')[0]
  203. tests.append(name)
  204. if len(tests) == 0:
  205. sys.exit("No test cases selected")
  206. if '-f' in sys.argv[idx:]:
  207. extra_args = sys.argv[idx:]
  208. else:
  209. extra_args = [x for x in sys.argv[idx:] if x not in tests]
  210. dir = '/tmp/hwsim-test-logs'
  211. try:
  212. os.mkdir(dir)
  213. except:
  214. pass
  215. if num_servers > 2 and len(tests) > 100:
  216. # Move test cases with long duration to the beginning as an
  217. # optimization to avoid last part of the test execution running a long
  218. # duration test case on a single VM while all other VMs have already
  219. # completed their work.
  220. long = [ "ap_roam_open",
  221. "ap_hs20_fetch_osu_stop",
  222. "ap_roam_wpa2_psk",
  223. "ibss_wpa_none_ccmp",
  224. "nfc_wps_er_handover_pk_hash_mismatch_sta",
  225. "go_neg_peers_force_diff_freq",
  226. "p2p_cli_invite",
  227. "sta_ap_scan_2b",
  228. "ap_pmf_sta_unprot_deauth_burst",
  229. "ap_bss_add_remove_during_ht_scan",
  230. "wext_scan_hidden",
  231. "autoscan_exponential",
  232. "nfc_p2p_client",
  233. "wnm_bss_keep_alive",
  234. "ap_inactivity_disconnect",
  235. "scan_bss_expiration_age",
  236. "autoscan_periodic",
  237. "discovery_group_client",
  238. "concurrent_p2pcli",
  239. "ap_bss_add_remove",
  240. "wpas_ap_wps",
  241. "wext_pmksa_cache",
  242. "ibss_wpa_none",
  243. "ap_ht_40mhz_intolerant_ap",
  244. "ibss_rsn",
  245. "discovery_pd_retries",
  246. "ap_wps_setup_locked_timeout",
  247. "ap_vht160",
  248. "dfs_radar",
  249. "dfs",
  250. "grpform_cred_ready_timeout",
  251. "ap_wps_pbc_timeout" ]
  252. for l in long:
  253. if l in tests:
  254. tests.remove(l)
  255. tests.insert(0, l)
  256. vm = {}
  257. for i in range(0, num_servers):
  258. print("\rStarting virtual machine {}/{}".format(i + 1, num_servers)),
  259. cmd = ['./vm-run.sh', '--delay', str(i), '--timestamp', str(timestamp),
  260. '--ext', 'srv.%d' % (i + 1),
  261. '-i'] + codecov_args + extra_args
  262. vm[i] = {}
  263. vm[i]['first_run_done'] = False
  264. vm[i]['proc'] = subprocess.Popen(cmd,
  265. stdin=subprocess.PIPE,
  266. stdout=subprocess.PIPE,
  267. stderr=subprocess.PIPE)
  268. vm[i]['out'] = ""
  269. vm[i]['err'] = ""
  270. for stream in [ vm[i]['proc'].stdout, vm[i]['proc'].stderr ]:
  271. fd = stream.fileno()
  272. fl = fcntl.fcntl(fd, fcntl.F_GETFL)
  273. fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
  274. print
  275. curses.wrapper(show_progress)
  276. with open('{}/{}-parallel.log'.format(dir, timestamp), 'w') as f:
  277. for i in range(0, num_servers):
  278. f.write('VM {}\n{}\n{}\n'.format(i, vm[i]['out'], vm[i]['err']))
  279. (started, passed, failed, skipped) = get_results()
  280. if first_run_failures:
  281. print "Failed test cases:"
  282. for f in first_run_failures:
  283. print f,
  284. print
  285. double_failed = []
  286. for f in failed:
  287. name = f.split(' ')[1]
  288. double_failed.append(name)
  289. for test in first_run_failures:
  290. double_failed.remove(test)
  291. if failed and not double_failed:
  292. print "All failed cases passed on retry"
  293. elif double_failed:
  294. print "Failed even on retry:"
  295. for f in double_failed:
  296. print f,
  297. print
  298. print("TOTAL={} PASS={} FAIL={} SKIP={}".format(len(started), len(passed), len(failed), len(skipped)))
  299. print "Logs: " + dir + '/' + str(timestamp)
  300. for i in range(0, num_servers):
  301. log = '{}/{}.srv.{}/console'.format(dir, timestamp, i + 1)
  302. with open(log, 'r') as f:
  303. if "Kernel panic" in f.read():
  304. print "Kernel panic in " + log
  305. if codecov:
  306. print "Code coverage - preparing report"
  307. for i in range(num_servers):
  308. subprocess.check_call(['./process-codecov.sh',
  309. logdir + ".srv.%d" % (i + 1),
  310. str(i)])
  311. subprocess.check_call(['./combine-codecov.sh', logdir])
  312. print "file://%s/index.html" % logdir
  313. if double_failed:
  314. sys.exit(2)
  315. if failed:
  316. sys.exit(1)
  317. sys.exit(0)
  318. if __name__ == "__main__":
  319. main()