Print this page
new smatch
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/tools/smatch/src/validation/test-suite
+++ new/usr/src/tools/smatch/src/validation/test-suite
1 1 #!/bin/sh
2 2
3 3 #set -x
4 4
5 5 cd $(dirname "$0")
6 6
7 7 default_path=".."
8 8 default_cmd="sparse \$file"
9 -tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
9 +default_args="$SPARSE_TEST_ARGS"
10 +tests_list=""
10 11 prog_name=`basename $0`
11 12
12 13 if [ ! -x "$default_path/sparse-llvm" ]; then
13 - disabled_cmds="sparsec sparsei sparse-llvm"
14 + disabled_cmds="sparsec sparsei sparse-llvm sparse-llvm-dis"
14 15 fi
15 16
16 17 # flags:
17 18 # - some tests gave an unexpected result
18 19 failed=0
19 20
20 21 # counts:
21 22 # - tests that have not been converted to test-suite format
22 23 # - tests that are disabled
23 24 # - tests that passed
24 25 # - tests that failed
25 26 # - tests that failed but are known to fail
26 27 unhandled_tests=0
27 28 disabled_tests=0
28 29 ok_tests=0
29 30 ko_tests=0
30 31 known_ko_tests=0
31 32
32 33 # defaults to not verbose
33 34 [ -z "$V" ] && V=0
34 -[ $V -eq 0 ] && quiet=1 || quiet=0
35 +vquiet=""
36 +quiet=0
37 +abort=0
35 38
39 +
36 40 ##
41 +# verbose(string) - prints string if we are in verbose mode
42 +verbose()
43 +{
44 + [ "$V" -eq "1" ] && echo " $1"
45 + return 0
46 +}
47 +
48 +##
49 +# warning(string) - prints a warning
50 +warning()
51 +{
52 + [ "$quiet" -ne 1 ] && echo "warning: $1"
53 + return 0
54 +}
55 +
56 +##
57 +# error(string[, die]) - prints an error and exits with value die if given
58 +error()
59 +{
60 + [ "$quiet" -ne 1 ] && echo "error: $1"
61 + [ -n "$2" ] && exit $2
62 + return 0
63 +}
64 +
65 +
66 +##
37 67 # get_tag_value(file) - get the 'check-<...>' tags & values
38 68 get_tag_value()
39 69 {
40 70 check_name=""
41 71 check_command="$default_cmd"
42 72 check_exit_value=0
43 73 check_timeout=0
44 74 check_known_to_fail=0
45 75 check_error_ignore=0
46 76 check_output_ignore=0
47 77 check_output_contains=0
48 78 check_output_excludes=0
49 79 check_output_pattern=0
80 + check_arch_ignore=""
81 + check_arch_only=""
82 + check_assert=""
83 + check_cpp_if=""
50 84
51 85 lines=$(grep 'check-[a-z-]*' $1 | \
52 86 sed -e 's/^.*\(check-[a-z-]*:*\) *\(.*\)$/\1 \2/')
53 87
54 88 while read tag val; do
55 89 #echo "-> tag: '$tag'"
56 90 #echo "-> val: '$val'"
57 91 case $tag in
58 92 check-name:) check_name="$val" ;;
59 93 check-command:) check_command="$val" ;;
60 94 check-exit-value:) check_exit_value="$val" ;;
61 95 check-timeout:) [ -z "$val" ] && val=1
62 96 check_timeout="$val" ;;
63 97 check-known-to-fail) check_known_to_fail=1 ;;
64 98 check-error-ignore) check_error_ignore=1 ;;
65 99 check-output-ignore) check_output_ignore=1 ;;
66 100 check-output-contains:) check_output_contains=1 ;;
67 101 check-output-excludes:) check_output_excludes=1 ;;
68 - check-output-pattern-) check_output_pattern=1 ;;
102 + check-output-pattern) check_output_pattern=1 ;;
103 + check-arch-ignore:) arch=$(uname -m)
104 + check_arch_ignore="$val" ;;
105 + check-arch-only:) arch=$(uname -m)
106 + check_arch_only="$val" ;;
107 + check-assert:) check_assert="$val" ;;
108 + check-cpp-if:) check_cpp_if="$val" ;;
109 +
110 + check-description:) ;; # ignore
111 + check-note:) ;; # ignore
112 + check-warning:) ;; # ignore
113 + check-error-start) ;; # ignore
114 + check-error-end) ;; # ignore
115 + check-output-start) ;; # ignore
116 + check-output-end) ;; # ignore
117 + check-should-pass) ;; # ignore, unused annotation
118 + check-should-fail) ;; # ignore, unused annotation
119 + check-should-warn) ;; # ignore, unused annotation
120 + check-*) error "$1: unknown tag '$tag'" 1 ;;
69 121 esac
70 122 done << EOT
71 123 $lines
72 124 EOT
73 125 }
74 126
75 127 ##
76 128 # helper for has_(each|none)_patterns()
77 129 has_patterns()
78 130 {
79 131 ifile="$1"
80 132 patt="$2"
81 133 ofile="$3"
82 134 cmp="$4"
135 + msg="$5"
83 136 grep "$patt:" "$ifile" | \
84 137 sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
85 138 while read val; do
86 139 grep -s -q "$val" "$ofile"
87 140 if [ "$?" $cmp 0 ]; then
141 + error " Pattern '$val' unexpectedly $msg"
88 142 return 1
89 143 fi
90 144 done
91 145
92 146 return $?
93 147 }
94 148
95 149 ##
96 150 # has_each_patterns(ifile tag ofile) - does ofile contains some
97 151 # of the patterns given by ifile's tags?
98 152 #
99 153 # returns 0 if all present, 1 otherwise
100 154 has_each_patterns()
101 155 {
102 - has_patterns "$1" "$2" "$3" -ne
156 + has_patterns "$1" "$2" "$4" -ne "$3"
103 157 }
104 158
105 159 ##
106 160 # has_none_patterns(ifile tag ofile) - does ofile contains some
107 161 # of the patterns given by ifile's tags?
108 162 #
109 163 # returns 1 if any present, 0 otherwise
110 164 has_none_patterns()
111 165 {
112 - has_patterns "$1" "$2" "$3" -eq
166 + has_patterns "$1" "$2" "$4" -eq "$3"
113 167 }
114 168
115 169 ##
116 -# nbr_patterns(ifile tag ofile) - does ofile contains the
170 +# minmax_patterns(ifile tag ofile) - does ofile contains the
117 171 # the patterns given by ifile's tags
118 172 # the right number of time?
119 -nbr_patterns()
173 +minmax_patterns()
120 174 {
121 175 ifile="$1"
122 176 patt="$2"
123 177 ofile="$3"
124 - grep "$patt-[0-9][0-9]*-times:" "$ifile" | \
125 - sed -e "s/^.*$patt-\([0-9][0-9]*\)-times: *\(.*\)/\1 \2/" | \
126 - while read nbr pat; do
178 + grep "$patt([0-9-]*\(, *\)*[0-9-]*):" "$ifile" | \
179 + sed -e "s/^.*$patt(\([0-9]*\)): *\(.*\)/\1 eq \2/" \
180 + -e "s/^.*$patt(\([0-9-]*\), *\([0-9-]*\)): *\(.*\)/\1 \2 \3/" | \
181 + while read min max pat; do
127 182 n=$(grep -s "$pat" "$ofile" | wc -l)
128 - if [ "$n" -ne "$nbr" ]; then
183 + if [ "$max" = "eq" ]; then
184 + if [ "$n" -ne "$min" ]; then
185 + error " Pattern '$pat' expected $min times but got $n times"
129 186 return 1
187 + fi
188 + continue
130 189 fi
190 + if [ "$min" != '-' ]; then
191 + if [ "$n" -lt "$min" ]; then
192 + error " Pattern '$pat' expected min $min times but got $n times"
193 + return 1
194 + fi
195 + fi
196 + if [ "$max" != '-' ]; then
197 + if [ "$n" -gt "$max" ]; then
198 + error " Pattern '$pat' expected max $max times but got $n times"
199 + return 1
200 + fi
201 + fi
131 202 done
132 203
133 204 return $?
134 205 }
135 206
136 207 ##
137 -# verbose(string) - prints string if we are in verbose mode
138 -verbose()
208 +# arg_file(filename) - checks if filename exists
209 +arg_file()
139 210 {
140 - [ "$V" -eq "1" ] && echo " $1"
211 + [ -z "$1" ] && {
212 + do_usage
213 + exit 1
214 + }
215 + [ -e "$1" ] || {
216 + error "Can't open file $1"
217 + exit 1
218 + }
141 219 return 0
142 220 }
143 221
144 -##
145 -# error(string[, die]) - prints an error and exits with value die if given
146 -error()
147 -{
148 - [ "$quiet" -ne 1 ] && echo "error: $1"
149 - [ -n "$2" ] && exit $2
150 - return 0
151 -}
152 222
223 +##
153 224 do_usage()
154 225 {
155 226 echo "$prog_name - a tiny automatic testing script"
156 -echo "Usage: $prog_name [command] [command arguments]"
227 +echo "Usage: $prog_name [option(s)] [command] [arguments]"
157 228 echo
229 +echo "options:"
230 +echo " -a|--abort Abort the tests as soon as one fails."
231 +echo " -q|--quiet Be extra quiet while running the tests."
232 +echo " --args='...' Add these options to the test command."
233 +echo
158 234 echo "commands:"
159 -echo " none runs the whole test suite"
160 -echo " single file runs the test in 'file'"
161 -echo " format file [name [cmd]] helps writing a new test case using cmd"
235 +echo " [file ...] Runs the test suite on the given file(s)."
236 +echo " If a directory is given, run only those files."
237 +echo " If no file is given, run the whole testsuite."
238 +echo " single file Run the test in 'file'."
239 +echo " format file [name [cmd]] Help writing a new test case using cmd."
162 240 echo
163 -echo " help prints usage"
241 +echo " [command] help Print usage."
164 242 }
165 243
244 +disable()
245 +{
246 + disabled_tests=$(($disabled_tests + 1))
247 + if [ -z "$vquiet" ]; then
248 + echo " SKIP $1 ($2)"
249 + fi
250 +}
251 +
166 252 ##
167 253 # do_test(file) - tries to validate a test case
168 254 #
169 255 # it "parses" file, looking for check-* tags and tries to validate
170 256 # the test against an expected result
171 257 # returns:
172 258 # - 0 if the test passed,
173 259 # - 1 if it failed,
174 260 # - 2 if it is not a "test-suite" test.
175 261 # - 3 if the test is disabled.
176 262 do_test()
177 263 {
178 264 test_failed=0
179 265 file="$1"
266 + quiet=0
180 267
181 268 get_tag_value $file
182 269
183 270 # can this test be handled by test-suite ?
184 271 # (it has to have a check-name key in it)
185 272 if [ "$check_name" = "" ]; then
186 - echo "warning: test '$file' unhandled"
273 + warning "$file: test unhandled"
187 274 unhandled_tests=$(($unhandled_tests + 1))
188 275 return 2
189 276 fi
190 277 test_name="$check_name"
191 278
192 279 # does the test provide a specific command ?
193 280 if [ "$check_command" = "" ]; then
194 281 check_command="$defaut_command"
195 282 fi
196 283
197 284 # check for disabled commands
198 285 set -- $check_command
199 286 base_cmd=$1
200 287 for i in $disabled_cmds; do
201 288 if [ "$i" = "$base_cmd" ] ; then
202 - disabled_tests=$(($disabled_tests + 1))
203 - echo " DISABLE $test_name ($file)"
289 + disable "$test_name" "$file"
204 290 return 3
205 291 fi
206 292 done
293 + if [ "$check_arch_ignore" != "" ]; then
294 + if echo $arch | egrep -q -w "$check_arch_ignore"; then
295 + disable "$test_name" "$file"
296 + return 3
297 + fi
298 + fi
299 + if [ "$check_arch_only" != "" ]; then
300 + if ! (echo $arch | egrep -q -w "$check_arch_only"); then
301 + disable "$test_name" "$file"
302 + return 3
303 + fi
304 + fi
305 + if [ "$check_assert" != "" ]; then
306 + res=$(../sparse - 2>&1 >/dev/null <<- EOF
307 + _Static_assert($check_assert, "$check_assert");
308 + EOF
309 + )
310 + if [ "$res" != "" ]; then
311 + disable "$test_name" "$file"
312 + return 3
313 + fi
314 + fi
315 + if [ "$check_cpp_if" != "" ]; then
316 + res=$(../sparse -E - 2>/dev/null <<- EOF
317 + #if !($check_cpp_if)
318 + fail
319 + #endif
320 + EOF
321 + )
322 + if [ "$res" != "" ]; then
323 + disable "$test_name" "$file"
324 + return 3
325 + fi
326 + fi
207 327
208 - cmd=`eval echo $default_path/$check_command`
328 + if [ -z "$vquiet" ]; then
329 + echo " TEST $test_name ($file)"
330 + fi
209 331
210 - echo " TEST $test_name ($file)"
332 + verbose "Using command : $(echo "$@")"
211 333
212 - verbose "Using command : $cmd"
213 -
214 334 # grab the expected exit value
215 335 expected_exit_value=$check_exit_value
216 336 verbose "Expecting exit value: $expected_exit_value"
217 337
218 338 # do we want a timeout?
339 + pre_cmd=""
219 340 if [ $check_timeout -ne 0 ]; then
220 - cmd="timeout -k 1s $check_timeout $cmd"
341 + pre_cmd="timeout -k 1s $check_timeout"
221 342 fi
222 343
344 + shift
345 + # launch the test command and
223 346 # grab the actual output & exit value
224 - $cmd 1> $file.output.got 2> $file.error.got
347 + eval $pre_cmd $default_path/$base_cmd $default_args "$@" \
348 + 1> $file.output.got 2> $file.error.got
225 349 actual_exit_value=$?
226 350
227 351 must_fail=$check_known_to_fail
228 - quiet=0
229 352 [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
230 353 known_ko_tests=$(($known_ko_tests + $must_fail))
231 354
232 - for stream in output error; do
355 + for stream in error output; do
233 356 eval ignore=\$check_${stream}_ignore
234 357 [ $ignore -eq 1 ] && continue
235 358
236 359 # grab the expected output
237 360 sed -n "/check-$stream-start/,/check-$stream-end/p" $file \
238 361 | grep -v check-$stream > "$file".$stream.expected
239 362
240 363 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
241 364 if [ "$?" -ne "0" ]; then
242 365 error "actual $stream text does not match expected $stream text."
243 366 error "see $file.$stream.* for further investigation."
244 367 [ $quiet -ne 1 ] && cat "$file".$stream.diff
245 368 test_failed=1
246 369 fi
|
↓ open down ↓ |
4 lines elided |
↑ open up ↑ |
247 370 done
248 371
249 372 if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
250 373 error "Actual exit value does not match the expected one."
251 374 error "expected $expected_exit_value, got $actual_exit_value."
252 375 test_failed=1
253 376 fi
254 377
255 378 # verify the 'check-output-contains/excludes' tags
256 379 if [ $check_output_contains -eq 1 ]; then
257 - has_each_patterns "$file" 'check-output-contains' $file.output.got
380 + has_each_patterns "$file" 'check-output-contains' absent $file.output.got
258 381 if [ "$?" -ne "0" ]; then
259 - error "Actual output doesn't contain some of the expected patterns."
260 382 test_failed=1
261 383 fi
262 384 fi
263 385 if [ $check_output_excludes -eq 1 ]; then
264 - has_none_patterns "$file" 'check-output-excludes' $file.output.got
386 + has_none_patterns "$file" 'check-output-excludes' present $file.output.got
265 387 if [ "$?" -ne "0" ]; then
266 - error "Actual output contains some patterns which are not expected."
267 388 test_failed=1
268 389 fi
269 390 fi
270 391 if [ $check_output_pattern -eq 1 ]; then
271 - # verify the 'check-output-pattern-X-times' tags
272 - nbr_patterns "$file" 'check-output-pattern' $file.output.got
392 + # verify the 'check-output-pattern(...)' tags
393 + minmax_patterns "$file" 'check-output-pattern' $file.output.got
273 394 if [ "$?" -ne "0" ]; then
274 - error "Actual output doesn't contain the pattern the expected number."
275 395 test_failed=1
276 396 fi
277 397 fi
278 398
279 - [ "$test_failed" -eq "$must_fail" ] || failed=1
280 -
281 399 if [ "$must_fail" -eq "1" ]; then
282 400 if [ "$test_failed" -eq "1" ]; then
283 - echo "info: test '$file' is known to fail"
401 + [ -z "$vquiet" ] && \
402 + echo "info: XFAIL: test '$file' is known to fail"
284 403 else
285 - echo "error: test '$file' is known to fail but succeed!"
286 - test_failed=1
404 + echo "error: XPASS: test '$file' is known to fail but succeed!"
287 405 fi
406 + else
407 + if [ "$test_failed" -eq "1" ]; then
408 + echo "error: FAIL: test '$file' failed"
409 + else
410 + [ "$V" -ne "0" ] && \
411 + echo "info: PASS: test '$file' passed"
412 + fi
288 413 fi
289 414
415 + if [ "$test_failed" -ne "$must_fail" ]; then
416 + [ $abort -eq 1 ] && exit 1
417 + test_failed=1
418 + failed=1
419 + fi
420 +
290 421 if [ "$test_failed" -eq "1" ]; then
291 422 ko_tests=$(($ko_tests + 1))
292 423 else
293 424 ok_tests=$(($ok_tests + 1))
294 425 rm -f $file.{error,output}.{expected,got,diff}
295 426 fi
296 427 return $test_failed
297 428 }
298 429
299 430 do_test_suite()
300 431 {
301 432 for i in $tests_list; do
302 433 do_test "$i"
303 434 done
304 435
436 + OK=OK
437 + [ $failed -eq 0 ] || OK=KO
438 +
305 439 # prints some numbers
306 440 tests_nr=$(($ok_tests + $ko_tests))
307 - echo -n "Out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
308 - echo " ($known_ko_tests of them are known to fail)"
441 + echo "$OK: out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
442 + if [ "$known_ko_tests" -ne 0 ]; then
443 + echo " $known_ko_tests of them are known to fail"
444 + fi
309 445 if [ "$unhandled_tests" -ne "0" ]; then
310 - echo "$unhandled_tests tests could not be handled by $prog_name"
446 + echo " $unhandled_tests tests could not be handled by $prog_name"
311 447 fi
312 448 if [ "$disabled_tests" -ne "0" ]; then
313 - echo "$disabled_tests tests were disabled"
449 + echo " $disabled_tests tests were disabled"
314 450 fi
315 451 }
316 452
317 453 ##
318 -# do_format(file[, name[, cmd]]) - helps a test writer to format test-suite tags
454 +do_format_help() {
455 +echo "Usage: $prog_name [option(s)] [--]format file [name [cmd]]"
456 +echo
457 +echo "options:"
458 +echo " -a append the created test to the input file"
459 +echo " -f write a test known to fail"
460 +echo " -l write a test for linearized code"
461 +echo
462 +echo "argument(s):"
463 +echo " file file containing the test case(s)"
464 +echo " name name for the test case (defaults to file)"
465 +echo " cmd command to be used (defaults to 'sparse \$file')"
466 +}
467 +
468 +##
469 +# do_format([options,] file[, name[, cmd]]) - helps a test writer to format test-suite tags
319 470 do_format()
320 471 {
321 - if [ -z "$2" ]; then
322 - fname="$1"
323 - fcmd=$default_cmd
324 - elif [ -z "$3" ]; then
325 - fname="$2"
326 - fcmd=$default_cmd
327 - else
328 - fname="$2"
329 - fcmd="$3"
330 - fi
472 + def_cmd="$default_cmd"
473 + append=0
474 + linear=0
475 + fail=0
476 +
477 + while [ $# -gt 1 ] ; do
478 + case "$1" in
479 + -a)
480 + append=1 ;;
481 + -f)
482 + fail=1 ;;
483 + -l)
484 + def_cmd='test-linearize -Wno-decl $file'
485 + linear=1 ;;
486 + help|-*)
487 + do_format_help
488 + return 0
489 + ;;
490 + *) break ;;
491 + esac
492 + shift
493 + continue
494 + done
495 +
496 + arg_file "$1" || return 1
497 +
331 498 file="$1"
499 + fname="$2"
500 + [ -z "$fname" ] && fname="$(basename "$1" .c)"
501 + fcmd="$3"
502 + [ -z "$fcmd" ] && fcmd="$def_cmd"
503 +
332 504 cmd=`eval echo $default_path/$fcmd`
333 505 $cmd 1> $file.output.got 2> $file.error.got
334 506 fexit_value=$?
507 + [ $append != 0 ] && exec >> $file
335 508 cat <<_EOF
509 +
336 510 /*
337 511 * check-name: $fname
338 512 _EOF
339 513 if [ "$fcmd" != "$default_cmd" ]; then
340 514 echo " * check-command: $fcmd"
341 515 fi
342 516 if [ "$fexit_value" -ne "0" ]; then
343 517 echo " * check-exit-value: $fexit_value"
344 518 fi
519 + if [ $fail != 0 ]; then
520 + echo " * check-known-to-fail"
521 + fi
522 + if [ $linear != 0 ]; then
523 + echo ' *'
524 + echo ' * check-output-ignore'
525 + echo ' * check-output-contains: xyz\\\\.'
526 + echo ' * check-output-excludes: \\\\.'
527 + fi
345 528 for stream in output error; do
346 529 if [ -s "$file.$stream.got" ]; then
347 530 echo " *"
348 531 echo " * check-$stream-start"
349 532 cat "$file.$stream.got"
350 533 echo " * check-$stream-end"
351 534 fi
352 535 done
353 536 echo " */"
354 537 return 0
355 538 }
356 539
357 -##
358 -# arg_file(filename) - checks if filename exists
359 -arg_file()
360 -{
361 - [ -z "$1" ] && {
362 - do_usage
363 - exit 1
364 - }
365 - [ -e "$1" ] || {
366 - error "Can't open file $1"
367 - exit 1
368 - }
369 - return 0
370 -}
540 +## allow flags from environment
541 +set -- $SPARSE_TEST_FLAGS "$@"
371 542
372 -case "$1" in
373 - '')
374 - do_test_suite
543 +## process the flags
544 +while [ "$#" -gt "0" ]; do
545 + case "$1" in
546 + -a|--abort)
547 + abort=1
375 548 ;;
376 - single)
549 + -q|--quiet)
550 + vquiet=1
551 + ;;
552 + --args=*)
553 + default_args="${1#--args=}";
554 + ;;
555 +
556 + single|--single)
377 557 arg_file "$2"
378 558 do_test "$2"
379 559 case "$?" in
380 560 0) echo "$2 passed !";;
381 561 1) echo "$2 failed !";;
382 562 2) echo "$2 can't be handled by $prog_name";;
383 563 esac
564 + exit $failed
384 565 ;;
385 - format)
386 - arg_file "$2"
387 - do_format "$2" "$3" "$4"
566 + format|--format)
567 + shift
568 + do_format "$@"
569 + exit 0
388 570 ;;
389 - help | *)
571 + help)
390 572 do_usage
391 573 exit 1
392 574 ;;
393 -esac
394 575
576 + *.c|*.cdoc)
577 + tests_list="$tests_list $1"
578 + ;;
579 + *)
580 + if [ ! -d "$1" ]; then
581 + do_usage
582 + exit 1
583 + fi
584 + tests_list="$tests_list $(find "$1" -name '*.c' | sort)"
585 + ;;
586 + esac
587 + shift
588 +done
589 +
590 +if [ -z "$tests_list" ]; then
591 + tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
592 +fi
593 +
594 +do_test_suite
395 595 exit $failed
396 596
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX