Print this page
11814 test-runner exit code should reflect test results
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/test/test-runner/cmd/run
+++ new/usr/src/test/test-runner/cmd/run
1 1 #!@PYTHON@
2 2
3 3 #
4 4 # This file and its contents are supplied under the terms of the
5 5 # Common Development and Distribution License ("CDDL"), version 1.0.
6 6 # You may only use this file in accordance with the terms of version
|
↓ open down ↓ |
6 lines elided |
↑ open up ↑ |
7 7 # 1.0 of the CDDL.
8 8 #
9 9 # A full copy of the text of the CDDL should have accompanied this
10 10 # source. A copy of the CDDL is also available via the Internet at
11 11 # http://www.illumos.org/license/CDDL.
12 12 #
13 13
14 14 #
15 15 # Copyright (c) 2012, 2016 by Delphix. All rights reserved.
16 16 # Copyright (c) 2017, Chris Fraire <cfraire@me.com>.
17 -# Copyright 2018 Joyent, Inc.
17 +# Copyright 2019 Joyent, Inc.
18 18 #
19 19
20 20 import ConfigParser
21 21 import os
22 22 import logging
23 23 import platform
24 24 from logging.handlers import WatchedFileHandler
25 25 from datetime import datetime
26 26 from optparse import OptionParser
27 27 from pwd import getpwnam
28 28 from pwd import getpwuid
29 29 from select import select
30 30 from subprocess import PIPE
31 31 from subprocess import Popen
|
↓ open down ↓ |
4 lines elided |
↑ open up ↑ |
32 32 from sys import argv
33 33 from sys import maxint
34 34 from threading import Timer
35 35 from time import time
36 36
37 37 BASEDIR = '/var/tmp/test_results'
38 38 KILL = '/usr/bin/kill'
39 39 TRUE = '/usr/bin/true'
40 40 SUDO = '/usr/bin/sudo'
41 41
42 +retcode = 0
43 +
42 44 # Custom class to reopen the log file in case it is forcibly closed by a test.
43 45 class WatchedFileHandlerClosed(WatchedFileHandler):
44 46 """Watch files, including closed files.
45 47 Similar to (and inherits from) logging.handler.WatchedFileHandler,
46 48 except that IOErrors are handled by reopening the stream and retrying.
47 49 This will be retried up to a configurable number of times before
48 50 giving up, default 5.
49 51 """
50 52
51 53 def __init__(self, filename, mode='a', encoding=None, delay=0, max_tries=5):
52 54 self.max_tries = max_tries
53 55 self.tries = 0
54 56 WatchedFileHandler.__init__(self, filename, mode, encoding, delay)
55 57
56 58 def emit(self, record):
57 59 while True:
58 60 try:
59 61 WatchedFileHandler.emit(self, record)
60 62 self.tries = 0
61 63 return
62 64 except IOError as err:
63 65 if self.tries == self.max_tries:
64 66 raise
65 67 self.stream.close()
66 68 self.stream = self._open()
67 69 self.tries += 1
68 70
69 71 class Result(object):
70 72 total = 0
71 73 runresults = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'KILLED': 0}
72 74
73 75 def __init__(self):
74 76 self.starttime = None
75 77 self.returncode = None
76 78 self.runtime = ''
|
↓ open down ↓ |
25 lines elided |
↑ open up ↑ |
77 79 self.stdout = []
78 80 self.stderr = []
79 81 self.result = ''
80 82
81 83 def done(self, proc, killed):
82 84 """
83 85 Finalize the results of this Cmd.
84 86 Report SKIP for return codes 3,4 (NOTINUSE, UNSUPPORTED)
85 87 as defined in ../stf/include/stf.shlib
86 88 """
89 + global retcode
90 +
87 91 Result.total += 1
88 92 m, s = divmod(time() - self.starttime, 60)
89 93 self.runtime = '%02d:%02d' % (m, s)
90 94 self.returncode = proc.returncode
91 95 if killed:
92 96 self.result = 'KILLED'
93 97 Result.runresults['KILLED'] += 1
98 + retcode = 2;
94 99 elif self.returncode is 0:
95 100 self.result = 'PASS'
96 101 Result.runresults['PASS'] += 1
97 102 elif self.returncode is 3 or self.returncode is 4:
98 103 self.result = 'SKIP'
99 104 Result.runresults['SKIP'] += 1
100 105 elif self.returncode is not 0:
101 106 self.result = 'FAIL'
102 107 Result.runresults['FAIL'] += 1
108 + retcode = 1;
103 109
104 110
105 111 class Output(object):
106 112 """
107 113 This class is a slightly modified version of the 'Stream' class found
108 114 here: http://goo.gl/aSGfv
109 115 """
110 116 def __init__(self, stream):
111 117 self.stream = stream
112 118 self._buf = ''
113 119 self.lines = []
114 120
115 121 def fileno(self):
116 122 return self.stream.fileno()
117 123
118 124 def read(self, drain=0):
119 125 """
120 126 Read from the file descriptor. If 'drain' set, read until EOF.
121 127 """
122 128 while self._read() is not None:
123 129 if not drain:
124 130 break
125 131
126 132 def _read(self):
127 133 """
128 134 Read up to 4k of data from this output stream. Collect the output
129 135 up to the last newline, and append it to any leftover data from a
130 136 previous call. The lines are stored as a (timestamp, data) tuple
131 137 for easy sorting/merging later.
132 138 """
133 139 fd = self.fileno()
134 140 buf = os.read(fd, 4096)
135 141 if not buf:
136 142 return None
137 143 if '\n' not in buf:
138 144 self._buf += buf
139 145 return []
140 146
141 147 buf = self._buf + buf
142 148 tmp, rest = buf.rsplit('\n', 1)
143 149 self._buf = rest
144 150 now = datetime.now()
145 151 rows = tmp.split('\n')
146 152 self.lines += [(now, r) for r in rows]
147 153
148 154
149 155 class Cmd(object):
150 156 verified_users = []
151 157
152 158 def __init__(self, pathname, outputdir=None, timeout=None, user=None):
153 159 self.pathname = pathname
154 160 self.outputdir = outputdir or 'BASEDIR'
155 161 self.timeout = timeout
156 162 self.user = user or ''
157 163 self.killed = False
158 164 self.result = Result()
159 165
160 166 if self.timeout is None:
161 167 self.timeout = 60
162 168
163 169 def __str__(self):
164 170 return "Pathname: %s\nOutputdir: %s\nTimeout: %s\nUser: %s\n" % \
165 171 (self.pathname, self.outputdir, self.timeout, self.user)
166 172
167 173 def kill_cmd(self, proc):
168 174 """
169 175 Kill a running command due to timeout, or ^C from the keyboard. If
170 176 sudo is required, this user was verified previously.
171 177 """
172 178 self.killed = True
173 179 do_sudo = len(self.user) != 0
174 180 signal = '-TERM'
175 181
176 182 cmd = [SUDO, KILL, signal, str(proc.pid)]
177 183 if not do_sudo:
178 184 del cmd[0]
179 185
180 186 try:
181 187 kp = Popen(cmd)
182 188 kp.wait()
183 189 except:
184 190 pass
185 191
186 192 def update_cmd_privs(self, cmd, user):
187 193 """
188 194 If a user has been specified to run this Cmd and we're not already
189 195 running as that user, prepend the appropriate sudo command to run
190 196 as that user.
191 197 """
192 198 me = getpwuid(os.getuid())
193 199
194 200 if not user or user is me:
195 201 return cmd
196 202
197 203 ret = '%s -E -u %s %s' % (SUDO, user, cmd)
198 204 return ret.split(' ')
199 205
200 206 def collect_output(self, proc):
201 207 """
202 208 Read from stdout/stderr as data becomes available, until the
203 209 process is no longer running. Return the lines from the stdout and
204 210 stderr Output objects.
205 211 """
206 212 out = Output(proc.stdout)
207 213 err = Output(proc.stderr)
208 214 res = []
209 215 while proc.returncode is None:
210 216 proc.poll()
211 217 res = select([out, err], [], [], .1)
212 218 for fd in res[0]:
213 219 fd.read()
214 220 for fd in res[0]:
215 221 fd.read(drain=1)
216 222
217 223 return out.lines, err.lines
218 224
219 225 def run(self, options):
220 226 """
221 227 This is the main function that runs each individual test.
222 228 Determine whether or not the command requires sudo, and modify it
223 229 if needed. Run the command, and update the result object.
224 230 """
225 231 if options.dryrun is True:
226 232 print self
227 233 return
228 234
229 235 privcmd = self.update_cmd_privs(self.pathname, self.user)
230 236 try:
231 237 old = os.umask(0)
232 238 if not os.path.isdir(self.outputdir):
233 239 os.makedirs(self.outputdir, mode=0777)
234 240 os.umask(old)
235 241 except OSError, e:
236 242 fail('%s' % e)
237 243
238 244 try:
239 245 self.result.starttime = time()
240 246 proc = Popen(privcmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
241 247 proc.stdin.close()
242 248
243 249 # Allow a special timeout value of 0 to mean infinity
244 250 if int(self.timeout) == 0:
245 251 self.timeout = maxint
246 252 t = Timer(int(self.timeout), self.kill_cmd, [proc])
247 253 t.start()
248 254 self.result.stdout, self.result.stderr = self.collect_output(proc)
249 255 except KeyboardInterrupt:
250 256 self.kill_cmd(proc)
251 257 fail('\nRun terminated at user request.')
252 258 finally:
253 259 t.cancel()
254 260
255 261 self.result.done(proc, self.killed)
256 262
257 263 def skip(self):
258 264 """
259 265 Initialize enough of the test result that we can log a skipped
260 266 command.
261 267 """
262 268 Result.total += 1
263 269 Result.runresults['SKIP'] += 1
264 270 self.result.stdout = self.result.stderr = []
265 271 self.result.starttime = time()
266 272 m, s = divmod(time() - self.result.starttime, 60)
267 273 self.result.runtime = '%02d:%02d' % (m, s)
268 274 self.result.result = 'SKIP'
269 275
270 276 def log(self, logger, options):
271 277 """
272 278 This function is responsible for writing all output. This includes
273 279 the console output, the logfile of all results (with timestamped
274 280 merged stdout and stderr), and for each test, the unmodified
275 281 stdout/stderr/merged in it's own file.
276 282 """
277 283 if logger is None:
278 284 return
279 285
280 286 logname = getpwuid(os.getuid()).pw_name
281 287 user = ' (run as %s)' % (self.user if len(self.user) else logname)
282 288 msga = 'Test: %s%s ' % (self.pathname, user)
283 289 msgb = '[%s] [%s]' % (self.result.runtime, self.result.result)
284 290 pad = ' ' * (80 - (len(msga) + len(msgb)))
285 291
286 292 # If -q is specified, only print a line for tests that didn't pass.
287 293 # This means passing tests need to be logged as DEBUG, or the one
288 294 # line summary will only be printed in the logfile for failures.
289 295 if not options.quiet:
290 296 logger.info('%s%s%s' % (msga, pad, msgb))
291 297 elif self.result.result is not 'PASS':
292 298 logger.info('%s%s%s' % (msga, pad, msgb))
293 299 else:
294 300 logger.debug('%s%s%s' % (msga, pad, msgb))
295 301
296 302 lines = sorted(self.result.stdout + self.result.stderr,
297 303 cmp=lambda x, y: cmp(x[0], y[0]))
298 304
299 305 for dt, line in lines:
300 306 logger.debug('%s %s' % (dt.strftime("%H:%M:%S.%f ")[:11], line))
301 307
302 308 if len(self.result.stdout):
303 309 with open(os.path.join(self.outputdir, 'stdout'), 'w') as out:
304 310 for _, line in self.result.stdout:
305 311 os.write(out.fileno(), '%s\n' % line)
306 312 if len(self.result.stderr):
307 313 with open(os.path.join(self.outputdir, 'stderr'), 'w') as err:
308 314 for _, line in self.result.stderr:
309 315 os.write(err.fileno(), '%s\n' % line)
310 316 if len(self.result.stdout) and len(self.result.stderr):
311 317 with open(os.path.join(self.outputdir, 'merged'), 'w') as merged:
312 318 for _, line in lines:
313 319 os.write(merged.fileno(), '%s\n' % line)
314 320
315 321
316 322 class Test(Cmd):
317 323 props = ['outputdir', 'timeout', 'user', 'pre', 'pre_user', 'post',
318 324 'post_user']
319 325
320 326 def __init__(self, pathname, outputdir=None, timeout=None, user=None,
321 327 pre=None, pre_user=None, post=None, post_user=None):
322 328 super(Test, self).__init__(pathname, outputdir, timeout, user)
323 329 self.pre = pre or ''
324 330 self.pre_user = pre_user or ''
325 331 self.post = post or ''
326 332 self.post_user = post_user or ''
327 333
328 334 def __str__(self):
329 335 post_user = pre_user = ''
330 336 if len(self.pre_user):
331 337 pre_user = ' (as %s)' % (self.pre_user)
332 338 if len(self.post_user):
333 339 post_user = ' (as %s)' % (self.post_user)
334 340 return "Pathname: %s\nOutputdir: %s\nTimeout: %d\nPre: %s%s\nPost: " \
335 341 "%s%s\nUser: %s\n" % \
336 342 (self.pathname, self.outputdir, self.timeout, self.pre,
337 343 pre_user, self.post, post_user, self.user)
338 344
339 345 def verify(self, logger):
340 346 """
341 347 Check the pre/post scripts, user and Test. Omit the Test from this
342 348 run if there are any problems.
343 349 """
344 350 files = [self.pre, self.pathname, self.post]
345 351 users = [self.pre_user, self.user, self.post_user]
346 352
347 353 for f in [f for f in files if len(f)]:
348 354 if not verify_file(f):
349 355 logger.info("Warning: Test '%s' not added to this run because"
350 356 " it failed verification." % f)
351 357 return False
352 358
353 359 for user in [user for user in users if len(user)]:
354 360 if not verify_user(user, logger):
355 361 logger.info("Not adding Test '%s' to this run." %
356 362 self.pathname)
357 363 return False
358 364
359 365 return True
360 366
361 367 def run(self, logger, options):
362 368 """
363 369 Create Cmd instances for the pre/post scripts. If the pre script
364 370 doesn't pass, skip this Test. Run the post script regardless.
365 371 """
366 372 odir = os.path.join(self.outputdir, os.path.basename(self.pre))
367 373 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
368 374 user=self.pre_user)
369 375 test = Cmd(self.pathname, outputdir=self.outputdir,
370 376 timeout=self.timeout, user=self.user)
371 377 odir = os.path.join(self.outputdir, os.path.basename(self.post))
372 378 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
373 379 user=self.post_user)
374 380
375 381 cont = True
376 382 if len(pretest.pathname):
377 383 pretest.run(options)
378 384 cont = pretest.result.result is 'PASS'
379 385 pretest.log(logger, options)
380 386
381 387 if cont:
382 388 test.run(options)
383 389 else:
384 390 test.skip()
385 391
386 392 test.log(logger, options)
387 393
388 394 if len(posttest.pathname):
389 395 posttest.run(options)
390 396 posttest.log(logger, options)
391 397
392 398
393 399 class TestGroup(Test):
394 400 props = Test.props + ['tests']
395 401
396 402 def __init__(self, pathname, outputdir=None, timeout=None, user=None,
397 403 pre=None, pre_user=None, post=None, post_user=None,
398 404 tests=None):
399 405 super(TestGroup, self).__init__(pathname, outputdir, timeout, user,
400 406 pre, pre_user, post, post_user)
401 407 self.tests = tests or []
402 408
403 409 def __str__(self):
404 410 post_user = pre_user = ''
405 411 if len(self.pre_user):
406 412 pre_user = ' (as %s)' % (self.pre_user)
407 413 if len(self.post_user):
408 414 post_user = ' (as %s)' % (self.post_user)
409 415 return "Pathname: %s\nOutputdir: %s\nTests: %s\nTimeout: %d\n" \
410 416 "Pre: %s%s\nPost: %s%s\nUser: %s\n" % \
411 417 (self.pathname, self.outputdir, self.tests, self.timeout,
412 418 self.pre, pre_user, self.post, post_user, self.user)
413 419
414 420 def verify(self, logger):
415 421 """
416 422 Check the pre/post scripts, user and tests in this TestGroup. Omit
417 423 the TestGroup entirely, or simply delete the relevant tests in the
418 424 group, if that's all that's required.
419 425 """
420 426 # If the pre or post scripts are relative pathnames, convert to
421 427 # absolute, so they stand a chance of passing verification.
422 428 if len(self.pre) and not os.path.isabs(self.pre):
423 429 self.pre = os.path.join(self.pathname, self.pre)
424 430 if len(self.post) and not os.path.isabs(self.post):
425 431 self.post = os.path.join(self.pathname, self.post)
426 432
427 433 auxfiles = [self.pre, self.post]
428 434 users = [self.pre_user, self.user, self.post_user]
429 435
430 436 for f in [f for f in auxfiles if len(f)]:
431 437 if self.pathname != os.path.dirname(f):
432 438 logger.info("Warning: TestGroup '%s' not added to this run. "
433 439 "Auxiliary script '%s' exists in a different "
434 440 "directory." % (self.pathname, f))
435 441 return False
436 442
437 443 if not verify_file(f):
438 444 logger.info("Warning: TestGroup '%s' not added to this run. "
439 445 "Auxiliary script '%s' failed verification." %
440 446 (self.pathname, f))
441 447 return False
442 448
443 449 for user in [user for user in users if len(user)]:
444 450 if not verify_user(user, logger):
445 451 logger.info("Not adding TestGroup '%s' to this run." %
446 452 self.pathname)
447 453 return False
448 454
449 455 # If one of the tests is invalid, delete it, log it, and drive on.
450 456 self.tests[:] = [f for f in self.tests if
451 457 verify_file(os.path.join(self.pathname, f))]
452 458
453 459 return len(self.tests) is not 0
454 460
455 461 def run(self, logger, options):
456 462 """
457 463 Create Cmd instances for the pre/post scripts. If the pre script
458 464 doesn't pass, skip all the tests in this TestGroup. Run the post
459 465 script regardless.
460 466 """
461 467 odir = os.path.join(self.outputdir, os.path.basename(self.pre))
462 468 pretest = Cmd(self.pre, outputdir=odir, timeout=self.timeout,
463 469 user=self.pre_user)
464 470 odir = os.path.join(self.outputdir, os.path.basename(self.post))
465 471 posttest = Cmd(self.post, outputdir=odir, timeout=self.timeout,
466 472 user=self.post_user)
467 473
468 474 cont = True
469 475 if len(pretest.pathname):
470 476 pretest.run(options)
471 477 cont = pretest.result.result is 'PASS'
472 478 pretest.log(logger, options)
473 479
474 480 for fname in self.tests:
475 481 test = Cmd(os.path.join(self.pathname, fname),
476 482 outputdir=os.path.join(self.outputdir, fname),
477 483 timeout=self.timeout, user=self.user)
478 484 if cont:
479 485 test.run(options)
480 486 else:
481 487 test.skip()
482 488
483 489 test.log(logger, options)
484 490
485 491 if len(posttest.pathname):
486 492 posttest.run(options)
487 493 posttest.log(logger, options)
488 494
489 495
490 496 class TestRun(object):
491 497 props = ['quiet', 'outputdir']
492 498
493 499 def __init__(self, options):
494 500 self.tests = {}
495 501 self.testgroups = {}
496 502 self.starttime = time()
497 503 self.timestamp = datetime.now().strftime('%Y%m%dT%H%M%S')
498 504 self.outputdir = os.path.join(options.outputdir, self.timestamp)
499 505 self.logger = self.setup_logging(options)
500 506 self.defaults = [
501 507 ('outputdir', BASEDIR),
502 508 ('quiet', False),
503 509 ('timeout', 60),
504 510 ('user', ''),
505 511 ('pre', ''),
506 512 ('pre_user', ''),
507 513 ('post', ''),
508 514 ('post_user', '')
509 515 ]
510 516
511 517 def __str__(self):
512 518 s = 'TestRun:\n outputdir: %s\n' % self.outputdir
513 519 s += 'TESTS:\n'
514 520 for key in sorted(self.tests.keys()):
515 521 s += '%s%s' % (self.tests[key].__str__(), '\n')
516 522 s += 'TESTGROUPS:\n'
517 523 for key in sorted(self.testgroups.keys()):
518 524 s += '%s%s' % (self.testgroups[key].__str__(), '\n')
519 525 return s
520 526
521 527 def addtest(self, pathname, options):
522 528 """
523 529 Create a new Test, and apply any properties that were passed in
524 530 from the command line. If it passes verification, add it to the
525 531 TestRun.
526 532 """
527 533 test = Test(pathname)
528 534 for prop in Test.props:
529 535 setattr(test, prop, getattr(options, prop))
530 536
531 537 if test.verify(self.logger):
532 538 self.tests[pathname] = test
533 539
534 540 def addtestgroup(self, dirname, filenames, options):
535 541 """
536 542 Create a new TestGroup, and apply any properties that were passed
537 543 in from the command line. If it passes verification, add it to the
538 544 TestRun.
539 545 """
540 546 if dirname not in self.testgroups:
541 547 testgroup = TestGroup(dirname)
542 548 for prop in Test.props:
543 549 setattr(testgroup, prop, getattr(options, prop))
544 550
545 551 # Prevent pre/post scripts from running as regular tests
546 552 for f in [testgroup.pre, testgroup.post]:
547 553 if f in filenames:
548 554 del filenames[filenames.index(f)]
549 555
550 556 self.testgroups[dirname] = testgroup
551 557 self.testgroups[dirname].tests = sorted(filenames)
552 558
553 559 testgroup.verify(self.logger)
554 560
555 561 def read(self, logger, options):
556 562 """
557 563 Read in the specified runfile, and apply the TestRun properties
558 564 listed in the 'DEFAULT' section to our TestRun. Then read each
559 565 section, and apply the appropriate properties to the Test or
560 566 TestGroup. Properties from individual sections override those set
561 567 in the 'DEFAULT' section. If the Test or TestGroup passes
562 568 verification, add it to the TestRun.
563 569 """
564 570 config = ConfigParser.RawConfigParser()
565 571 if not len(config.read(options.runfile)):
566 572 fail("Coulnd't read config file %s" % options.runfile)
567 573
568 574 for opt in TestRun.props:
569 575 if config.has_option('DEFAULT', opt):
570 576 setattr(self, opt, config.get('DEFAULT', opt))
571 577 self.outputdir = os.path.join(self.outputdir, self.timestamp)
572 578
573 579 for section in config.sections():
574 580 if ('arch' in config.options(section) and
575 581 platform.machine() != config.get(section, 'arch')):
576 582 continue
577 583
578 584 if 'tests' in config.options(section):
579 585 testgroup = TestGroup(section)
580 586 for prop in TestGroup.props:
581 587 for sect in ['DEFAULT', section]:
582 588 if config.has_option(sect, prop):
583 589 setattr(testgroup, prop, config.get(sect, prop))
584 590
585 591 # Repopulate tests using eval to convert the string to a list
586 592 testgroup.tests = eval(config.get(section, 'tests'))
587 593
588 594 if testgroup.verify(logger):
589 595 self.testgroups[section] = testgroup
590 596
591 597 elif 'autotests' in config.options(section):
592 598 testgroup = TestGroup(section)
593 599 for prop in TestGroup.props:
594 600 for sect in ['DEFAULT', section]:
595 601 if config.has_option(sect, prop):
596 602 setattr(testgroup, prop, config.get(sect, prop))
597 603
598 604 filenames = os.listdir(section)
599 605 # only files starting with "tst." are considered tests
600 606 filenames = [f for f in filenames if f.startswith("tst.")]
601 607 testgroup.tests = sorted(filenames)
602 608
603 609 if testgroup.verify(logger):
604 610 self.testgroups[section] = testgroup
605 611
606 612 else:
607 613 test = Test(section)
608 614 for prop in Test.props:
609 615 for sect in ['DEFAULT', section]:
610 616 if config.has_option(sect, prop):
611 617 setattr(test, prop, config.get(sect, prop))
612 618
613 619 if test.verify(logger):
614 620 self.tests[section] = test
615 621
616 622 def write(self, options):
617 623 """
618 624 Create a configuration file for editing and later use. The
619 625 'DEFAULT' section of the config file is created from the
620 626 properties that were specified on the command line. Tests are
621 627 simply added as sections that inherit everything from the
622 628 'DEFAULT' section. TestGroups are the same, except they get an
623 629 option including all the tests to run in that directory.
624 630 """
625 631
626 632 defaults = dict([(prop, getattr(options, prop)) for prop, _ in
627 633 self.defaults])
628 634 config = ConfigParser.RawConfigParser(defaults)
629 635
630 636 for test in sorted(self.tests.keys()):
631 637 config.add_section(test)
632 638
633 639 for testgroup in sorted(self.testgroups.keys()):
634 640 config.add_section(testgroup)
635 641 config.set(testgroup, 'tests', self.testgroups[testgroup].tests)
636 642
637 643 try:
638 644 with open(options.template, 'w') as f:
639 645 return config.write(f)
640 646 except IOError:
641 647 fail('Could not open \'%s\' for writing.' % options.template)
642 648
643 649 def complete_outputdirs(self):
644 650 """
645 651 Collect all the pathnames for Tests, and TestGroups. Work
646 652 backwards one pathname component at a time, to create a unique
647 653 directory name in which to deposit test output. Tests will be able
648 654 to write output files directly in the newly modified outputdir.
649 655 TestGroups will be able to create one subdirectory per test in the
650 656 outputdir, and are guaranteed uniqueness because a group can only
651 657 contain files in one directory. Pre and post tests will create a
652 658 directory rooted at the outputdir of the Test or TestGroup in
653 659 question for their output.
654 660 """
655 661 done = False
656 662 components = 0
657 663 tmp_dict = dict(self.tests.items() + self.testgroups.items())
658 664 total = len(tmp_dict)
659 665 base = self.outputdir
660 666
661 667 while not done:
662 668 l = []
663 669 components -= 1
664 670 for testfile in tmp_dict.keys():
665 671 uniq = '/'.join(testfile.split('/')[components:]).lstrip('/')
666 672 if uniq not in l:
667 673 l.append(uniq)
668 674 tmp_dict[testfile].outputdir = os.path.join(base, uniq)
669 675 else:
670 676 break
671 677 done = total == len(l)
672 678
673 679 def setup_logging(self, options):
674 680 """
675 681 Two loggers are set up here. The first is for the logfile which
676 682 will contain one line summarizing the test, including the test
677 683 name, result, and running time. This logger will also capture the
678 684 timestamped combined stdout and stderr of each run. The second
679 685 logger is optional console output, which will contain only the one
680 686 line summary. The loggers are initialized at two different levels
681 687 to facilitate segregating the output.
682 688 """
683 689 if options.dryrun is True:
684 690 return
685 691
686 692 testlogger = logging.getLogger(__name__)
687 693 testlogger.setLevel(logging.DEBUG)
688 694
689 695 if options.cmd is not 'wrconfig':
690 696 try:
691 697 old = os.umask(0)
692 698 os.makedirs(self.outputdir, mode=0777)
693 699 os.umask(old)
694 700 except OSError, e:
695 701 fail('%s' % e)
696 702 filename = os.path.join(self.outputdir, 'log')
697 703
698 704 logfile = WatchedFileHandlerClosed(filename)
699 705 logfile.setLevel(logging.DEBUG)
700 706 logfilefmt = logging.Formatter('%(message)s')
701 707 logfile.setFormatter(logfilefmt)
702 708 testlogger.addHandler(logfile)
703 709
704 710 cons = logging.StreamHandler()
705 711 cons.setLevel(logging.INFO)
706 712 consfmt = logging.Formatter('%(message)s')
707 713 cons.setFormatter(consfmt)
708 714 testlogger.addHandler(cons)
709 715
710 716 return testlogger
711 717
712 718 def run(self, options):
713 719 """
714 720 Walk through all the Tests and TestGroups, calling run().
715 721 """
716 722 if not options.dryrun:
717 723 try:
718 724 os.chdir(self.outputdir)
719 725 except OSError:
720 726 fail('Could not change to directory %s' % self.outputdir)
721 727 for test in sorted(self.tests.keys()):
722 728 self.tests[test].run(self.logger, options)
723 729 for testgroup in sorted(self.testgroups.keys()):
724 730 self.testgroups[testgroup].run(self.logger, options)
725 731
726 732 def summary(self):
727 733 if Result.total is 0:
728 734 return
729 735
730 736 print '\nResults Summary'
731 737 for key in Result.runresults.keys():
732 738 if Result.runresults[key] is not 0:
733 739 print '%s\t% 4d' % (key, Result.runresults[key])
734 740
735 741 m, s = divmod(time() - self.starttime, 60)
736 742 h, m = divmod(m, 60)
737 743 print '\nRunning Time:\t%02d:%02d:%02d' % (h, m, s)
738 744 print 'Percent passed:\t%.1f%%' % ((float(Result.runresults['PASS']) /
739 745 float(Result.total)) * 100)
740 746 print 'Log directory:\t%s' % self.outputdir
741 747
742 748
743 749 def verify_file(pathname):
744 750 """
745 751 Verify that the supplied pathname is an executable regular file.
746 752 """
747 753 if os.path.isdir(pathname) or os.path.islink(pathname):
748 754 return False
749 755
750 756 if os.path.isfile(pathname) and os.access(pathname, os.X_OK):
751 757 return True
752 758
753 759 return False
754 760
755 761
756 762 def verify_user(user, logger):
757 763 """
758 764 Verify that the specified user exists on this system, and can execute
759 765 sudo without being prompted for a password.
760 766 """
761 767 testcmd = [SUDO, '-n', '-u', user, TRUE]
762 768
763 769 if user in Cmd.verified_users:
764 770 return True
765 771
766 772 try:
767 773 _ = getpwnam(user)
768 774 except KeyError:
769 775 logger.info("Warning: user '%s' does not exist.", user)
770 776 return False
771 777
772 778 p = Popen(testcmd)
773 779 p.wait()
774 780 if p.returncode is not 0:
775 781 logger.info("Warning: user '%s' cannot use passwordless sudo.", user)
776 782 return False
777 783 else:
778 784 Cmd.verified_users.append(user)
779 785
780 786 return True
781 787
782 788
783 789 def find_tests(testrun, options):
784 790 """
785 791 For the given list of pathnames, add files as Tests. For directories,
786 792 if do_groups is True, add the directory as a TestGroup. If False,
787 793 recursively search for executable files.
788 794 """
789 795
790 796 for p in sorted(options.pathnames):
791 797 if os.path.isdir(p):
792 798 for dirname, _, filenames in os.walk(p):
793 799 if options.do_groups:
794 800 testrun.addtestgroup(dirname, filenames, options)
795 801 else:
796 802 for f in sorted(filenames):
797 803 testrun.addtest(os.path.join(dirname, f), options)
798 804 else:
799 805 testrun.addtest(p, options)
800 806
801 807
802 808 def fail(retstr, ret=1):
803 809 print '%s: %s' % (argv[0], retstr)
804 810 exit(ret)
805 811
806 812
807 813 def options_cb(option, opt_str, value, parser):
808 814 path_options = ['runfile', 'outputdir', 'template']
809 815
810 816 if option.dest is 'runfile' and '-w' in parser.rargs or \
811 817 option.dest is 'template' and '-c' in parser.rargs:
812 818 fail('-c and -w are mutually exclusive.')
813 819
814 820 if opt_str in parser.rargs:
815 821 fail('%s may only be specified once.' % opt_str)
816 822
817 823 if option.dest is 'runfile':
818 824 parser.values.cmd = 'rdconfig'
819 825 if option.dest is 'template':
820 826 parser.values.cmd = 'wrconfig'
821 827
822 828 setattr(parser.values, option.dest, value)
823 829 if option.dest in path_options:
824 830 setattr(parser.values, option.dest, os.path.abspath(value))
825 831
826 832
827 833 def parse_args():
828 834 parser = OptionParser()
829 835 parser.add_option('-c', action='callback', callback=options_cb,
830 836 type='string', dest='runfile', metavar='runfile',
831 837 help='Specify tests to run via config file.')
832 838 parser.add_option('-d', action='store_true', default=False, dest='dryrun',
833 839 help='Dry run. Print tests, but take no other action.')
834 840 parser.add_option('-g', action='store_true', default=False,
835 841 dest='do_groups', help='Make directories TestGroups.')
836 842 parser.add_option('-o', action='callback', callback=options_cb,
837 843 default=BASEDIR, dest='outputdir', type='string',
838 844 metavar='outputdir', help='Specify an output directory.')
839 845 parser.add_option('-p', action='callback', callback=options_cb,
840 846 default='', dest='pre', metavar='script',
841 847 type='string', help='Specify a pre script.')
842 848 parser.add_option('-P', action='callback', callback=options_cb,
843 849 default='', dest='post', metavar='script',
844 850 type='string', help='Specify a post script.')
845 851 parser.add_option('-q', action='store_true', default=False, dest='quiet',
846 852 help='Silence on the console during a test run.')
847 853 parser.add_option('-t', action='callback', callback=options_cb, default=60,
848 854 dest='timeout', metavar='seconds', type='int',
849 855 help='Timeout (in seconds) for an individual test.')
850 856 parser.add_option('-u', action='callback', callback=options_cb,
851 857 default='', dest='user', metavar='user', type='string',
852 858 help='Specify a different user name to run as.')
853 859 parser.add_option('-w', action='callback', callback=options_cb,
854 860 default=None, dest='template', metavar='template',
855 861 type='string', help='Create a new config file.')
856 862 parser.add_option('-x', action='callback', callback=options_cb, default='',
857 863 dest='pre_user', metavar='pre_user', type='string',
858 864 help='Specify a user to execute the pre script.')
859 865 parser.add_option('-X', action='callback', callback=options_cb, default='',
860 866 dest='post_user', metavar='post_user', type='string',
861 867 help='Specify a user to execute the post script.')
862 868 (options, pathnames) = parser.parse_args()
863 869
864 870 if not options.runfile and not options.template:
865 871 options.cmd = 'runtests'
866 872
867 873 if options.runfile and len(pathnames):
868 874 fail('Extraneous arguments.')
869 875
870 876 options.pathnames = [os.path.abspath(path) for path in pathnames]
871 877
872 878 return options
873 879
874 880
875 881 def main():
876 882 options = parse_args()
877 883 testrun = TestRun(options)
878 884
879 885 if options.cmd is 'runtests':
880 886 find_tests(testrun, options)
881 887 elif options.cmd is 'rdconfig':
882 888 testrun.read(testrun.logger, options)
|
↓ open down ↓ |
770 lines elided |
↑ open up ↑ |
883 889 elif options.cmd is 'wrconfig':
884 890 find_tests(testrun, options)
885 891 testrun.write(options)
886 892 exit(0)
887 893 else:
888 894 fail('Unknown command specified')
889 895
890 896 testrun.complete_outputdirs()
891 897 testrun.run(options)
892 898 testrun.summary()
893 - exit(0)
899 + exit(retcode)
894 900
895 901
896 902 if __name__ == '__main__':
897 903 main()
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX