1 #!/bin/sh
2
3 #set -x
4
5 cd $(dirname "$0")
6
7 default_path=".."
8 default_cmd="sparse \$file"
9 tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
10 prog_name=`basename $0`
11
12 if [ ! -x "$default_path/sparse-llvm" ]; then
13 disabled_cmds="sparsec sparsei sparse-llvm"
14 fi
15
16 # flags:
17 # - some tests gave an unexpected result
18 failed=0
19
20 # counts:
21 # - tests that have not been converted to test-suite format
22 # - tests that are disabled
23 # - tests that passed
24 # - tests that failed
25 # - tests that failed but are known to fail
26 unhandled_tests=0
27 disabled_tests=0
28 ok_tests=0
29 ko_tests=0
30 known_ko_tests=0
31
32 # defaults to not verbose
33 [ -z "$V" ] && V=0
34 [ $V -eq 0 ] && quiet=1 || quiet=0
35
36 ##
37 # get_tag_value(file) - get the 'check-<...>' tags & values
38 get_tag_value()
39 {
40 check_name=""
41 check_command="$default_cmd"
42 check_exit_value=0
43 check_timeout=0
44 check_known_to_fail=0
45 check_error_ignore=0
46 check_output_ignore=0
47 check_output_contains=0
48 check_output_excludes=0
49 check_output_pattern=0
50
51 lines=$(grep 'check-[a-z-]*' $1 | \
52 sed -e 's/^.*\(check-[a-z-]*:*\) *\(.*\)$/\1 \2/')
53
54 while read tag val; do
55 #echo "-> tag: '$tag'"
56 #echo "-> val: '$val'"
57 case $tag in
58 check-name:) check_name="$val" ;;
59 check-command:) check_command="$val" ;;
60 check-exit-value:) check_exit_value="$val" ;;
61 check-timeout:) [ -z "$val" ] && val=1
62 check_timeout="$val" ;;
63 check-known-to-fail) check_known_to_fail=1 ;;
64 check-error-ignore) check_error_ignore=1 ;;
65 check-output-ignore) check_output_ignore=1 ;;
66 check-output-contains:) check_output_contains=1 ;;
67 check-output-excludes:) check_output_excludes=1 ;;
68 check-output-pattern-) check_output_pattern=1 ;;
69 esac
70 done << EOT
71 $lines
72 EOT
73 }
74
75 ##
76 # helper for has_(each|none)_patterns()
77 has_patterns()
78 {
79 ifile="$1"
80 patt="$2"
81 ofile="$3"
82 cmp="$4"
83 grep "$patt:" "$ifile" | \
84 sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
85 while read val; do
86 grep -s -q "$val" "$ofile"
87 if [ "$?" $cmp 0 ]; then
88 return 1
89 fi
90 done
91
92 return $?
93 }
94
95 ##
96 # has_each_patterns(ifile tag ofile) - does ofile contains some
97 # of the patterns given by ifile's tags?
98 #
99 # returns 0 if all present, 1 otherwise
100 has_each_patterns()
101 {
102 has_patterns "$1" "$2" "$3" -ne
103 }
104
105 ##
106 # has_none_patterns(ifile tag ofile) - does ofile contains some
107 # of the patterns given by ifile's tags?
108 #
109 # returns 1 if any present, 0 otherwise
110 has_none_patterns()
111 {
112 has_patterns "$1" "$2" "$3" -eq
113 }
114
115 ##
116 # nbr_patterns(ifile tag ofile) - does ofile contains the
117 # the patterns given by ifile's tags
118 # the right number of time?
119 nbr_patterns()
120 {
121 ifile="$1"
122 patt="$2"
123 ofile="$3"
124 grep "$patt-[0-9][0-9]*-times:" "$ifile" | \
125 sed -e "s/^.*$patt-\([0-9][0-9]*\)-times: *\(.*\)/\1 \2/" | \
126 while read nbr pat; do
127 n=$(grep -s "$pat" "$ofile" | wc -l)
128 if [ "$n" -ne "$nbr" ]; then
129 return 1
130 fi
131 done
132
133 return $?
134 }
135
136 ##
137 # verbose(string) - prints string if we are in verbose mode
138 verbose()
139 {
140 [ "$V" -eq "1" ] && echo " $1"
141 return 0
142 }
143
144 ##
145 # error(string[, die]) - prints an error and exits with value die if given
146 error()
147 {
148 [ "$quiet" -ne 1 ] && echo "error: $1"
149 [ -n "$2" ] && exit $2
150 return 0
151 }
152
153 do_usage()
154 {
155 echo "$prog_name - a tiny automatic testing script"
156 echo "Usage: $prog_name [command] [command arguments]"
157 echo
158 echo "commands:"
159 echo " none runs the whole test suite"
160 echo " single file runs the test in 'file'"
161 echo " format file [name [cmd]] helps writing a new test case using cmd"
162 echo
163 echo " help prints usage"
164 }
165
166 ##
167 # do_test(file) - tries to validate a test case
168 #
169 # it "parses" file, looking for check-* tags and tries to validate
170 # the test against an expected result
171 # returns:
172 # - 0 if the test passed,
173 # - 1 if it failed,
174 # - 2 if it is not a "test-suite" test.
175 # - 3 if the test is disabled.
176 do_test()
177 {
178 test_failed=0
179 file="$1"
180
181 get_tag_value $file
182
183 # can this test be handled by test-suite ?
184 # (it has to have a check-name key in it)
185 if [ "$check_name" = "" ]; then
186 echo "warning: test '$file' unhandled"
187 unhandled_tests=$(($unhandled_tests + 1))
188 return 2
189 fi
190 test_name="$check_name"
191
192 # does the test provide a specific command ?
193 if [ "$check_command" = "" ]; then
194 check_command="$defaut_command"
195 fi
196
197 # check for disabled commands
198 set -- $check_command
199 base_cmd=$1
200 for i in $disabled_cmds; do
201 if [ "$i" = "$base_cmd" ] ; then
202 disabled_tests=$(($disabled_tests + 1))
203 echo " DISABLE $test_name ($file)"
204 return 3
205 fi
206 done
207
208 cmd=`eval echo $default_path/$check_command`
209
210 echo " TEST $test_name ($file)"
211
212 verbose "Using command : $cmd"
213
214 # grab the expected exit value
215 expected_exit_value=$check_exit_value
216 verbose "Expecting exit value: $expected_exit_value"
217
218 # do we want a timeout?
219 if [ $check_timeout -ne 0 ]; then
220 cmd="timeout -k 1s $check_timeout $cmd"
221 fi
222
223 # grab the actual output & exit value
224 $cmd 1> $file.output.got 2> $file.error.got
225 actual_exit_value=$?
226
227 must_fail=$check_known_to_fail
228 quiet=0
229 [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
230 known_ko_tests=$(($known_ko_tests + $must_fail))
231
232 for stream in output error; do
233 eval ignore=\$check_${stream}_ignore
234 [ $ignore -eq 1 ] && continue
235
236 # grab the expected output
237 sed -n "/check-$stream-start/,/check-$stream-end/p" $file \
238 | grep -v check-$stream > "$file".$stream.expected
239
240 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
241 if [ "$?" -ne "0" ]; then
242 error "actual $stream text does not match expected $stream text."
243 error "see $file.$stream.* for further investigation."
244 [ $quiet -ne 1 ] && cat "$file".$stream.diff
245 test_failed=1
246 fi
247 done
248
249 if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
250 error "Actual exit value does not match the expected one."
251 error "expected $expected_exit_value, got $actual_exit_value."
252 test_failed=1
253 fi
254
255 # verify the 'check-output-contains/excludes' tags
256 if [ $check_output_contains -eq 1 ]; then
257 has_each_patterns "$file" 'check-output-contains' $file.output.got
258 if [ "$?" -ne "0" ]; then
259 error "Actual output doesn't contain some of the expected patterns."
260 test_failed=1
261 fi
262 fi
263 if [ $check_output_excludes -eq 1 ]; then
264 has_none_patterns "$file" 'check-output-excludes' $file.output.got
265 if [ "$?" -ne "0" ]; then
266 error "Actual output contains some patterns which are not expected."
267 test_failed=1
268 fi
269 fi
270 if [ $check_output_pattern -eq 1 ]; then
271 # verify the 'check-output-pattern-X-times' tags
272 nbr_patterns "$file" 'check-output-pattern' $file.output.got
273 if [ "$?" -ne "0" ]; then
274 error "Actual output doesn't contain the pattern the expected number."
275 test_failed=1
276 fi
277 fi
278
279 [ "$test_failed" -eq "$must_fail" ] || failed=1
280
281 if [ "$must_fail" -eq "1" ]; then
282 if [ "$test_failed" -eq "1" ]; then
283 echo "info: test '$file' is known to fail"
284 else
285 echo "error: test '$file' is known to fail but succeed!"
286 test_failed=1
287 fi
288 fi
289
290 if [ "$test_failed" -eq "1" ]; then
291 ko_tests=$(($ko_tests + 1))
292 else
293 ok_tests=$(($ok_tests + 1))
294 rm -f $file.{error,output}.{expected,got,diff}
295 fi
296 return $test_failed
297 }
298
299 do_test_suite()
300 {
301 for i in $tests_list; do
302 do_test "$i"
303 done
304
305 # prints some numbers
306 tests_nr=$(($ok_tests + $ko_tests))
307 echo -n "Out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
308 echo " ($known_ko_tests of them are known to fail)"
309 if [ "$unhandled_tests" -ne "0" ]; then
310 echo "$unhandled_tests tests could not be handled by $prog_name"
311 fi
312 if [ "$disabled_tests" -ne "0" ]; then
313 echo "$disabled_tests tests were disabled"
314 fi
315 }
316
317 ##
318 # do_format(file[, name[, cmd]]) - helps a test writer to format test-suite tags
319 do_format()
320 {
321 if [ -z "$2" ]; then
322 fname="$1"
323 fcmd=$default_cmd
324 elif [ -z "$3" ]; then
325 fname="$2"
326 fcmd=$default_cmd
327 else
328 fname="$2"
329 fcmd="$3"
330 fi
331 file="$1"
332 cmd=`eval echo $default_path/$fcmd`
333 $cmd 1> $file.output.got 2> $file.error.got
334 fexit_value=$?
335 cat <<_EOF
336 /*
337 * check-name: $fname
338 _EOF
339 if [ "$fcmd" != "$default_cmd" ]; then
340 echo " * check-command: $fcmd"
341 fi
342 if [ "$fexit_value" -ne "0" ]; then
343 echo " * check-exit-value: $fexit_value"
344 fi
345 for stream in output error; do
346 if [ -s "$file.$stream.got" ]; then
347 echo " *"
348 echo " * check-$stream-start"
349 cat "$file.$stream.got"
350 echo " * check-$stream-end"
351 fi
352 done
353 echo " */"
354 return 0
355 }
356
357 ##
358 # arg_file(filename) - checks if filename exists
359 arg_file()
360 {
361 [ -z "$1" ] && {
362 do_usage
363 exit 1
364 }
365 [ -e "$1" ] || {
366 error "Can't open file $1"
367 exit 1
368 }
369 return 0
370 }
371
372 case "$1" in
373 '')
374 do_test_suite
375 ;;
376 single)
377 arg_file "$2"
378 do_test "$2"
379 case "$?" in
380 0) echo "$2 passed !";;
381 1) echo "$2 failed !";;
382 2) echo "$2 can't be handled by $prog_name";;
383 esac
384 ;;
385 format)
386 arg_file "$2"
387 do_format "$2" "$3" "$4"
388 ;;
389 help | *)
390 do_usage
391 exit 1
392 ;;
393 esac
394
395 exit $failed
396
|
1 #!/bin/sh
2
3 #set -x
4
5 cd $(dirname "$0")
6
7 default_path=".."
8 default_cmd="sparse \$file"
9 default_args="$SPARSE_TEST_ARGS"
10 tests_list=""
11 prog_name=`basename $0`
12
13 if [ ! -x "$default_path/sparse-llvm" ]; then
14 disabled_cmds="sparsec sparsei sparse-llvm sparse-llvm-dis"
15 fi
16
17 # flags:
18 # - some tests gave an unexpected result
19 failed=0
20
21 # counts:
22 # - tests that have not been converted to test-suite format
23 # - tests that are disabled
24 # - tests that passed
25 # - tests that failed
26 # - tests that failed but are known to fail
27 unhandled_tests=0
28 disabled_tests=0
29 ok_tests=0
30 ko_tests=0
31 known_ko_tests=0
32
33 # defaults to not verbose
34 [ -z "$V" ] && V=0
35 vquiet=""
36 quiet=0
37 abort=0
38
39
40 ##
41 # verbose(string) - prints string if we are in verbose mode
42 verbose()
43 {
44 [ "$V" -eq "1" ] && echo " $1"
45 return 0
46 }
47
48 ##
49 # warning(string) - prints a warning
50 warning()
51 {
52 [ "$quiet" -ne 1 ] && echo "warning: $1"
53 return 0
54 }
55
56 ##
57 # error(string[, die]) - prints an error and exits with value die if given
58 error()
59 {
60 [ "$quiet" -ne 1 ] && echo "error: $1"
61 [ -n "$2" ] && exit $2
62 return 0
63 }
64
65
66 ##
67 # get_tag_value(file) - get the 'check-<...>' tags & values
68 get_tag_value()
69 {
70 check_name=""
71 check_command="$default_cmd"
72 check_exit_value=0
73 check_timeout=0
74 check_known_to_fail=0
75 check_error_ignore=0
76 check_output_ignore=0
77 check_output_contains=0
78 check_output_excludes=0
79 check_output_pattern=0
80 check_arch_ignore=""
81 check_arch_only=""
82 check_assert=""
83 check_cpp_if=""
84
85 lines=$(grep 'check-[a-z-]*' $1 | \
86 sed -e 's/^.*\(check-[a-z-]*:*\) *\(.*\)$/\1 \2/')
87
88 while read tag val; do
89 #echo "-> tag: '$tag'"
90 #echo "-> val: '$val'"
91 case $tag in
92 check-name:) check_name="$val" ;;
93 check-command:) check_command="$val" ;;
94 check-exit-value:) check_exit_value="$val" ;;
95 check-timeout:) [ -z "$val" ] && val=1
96 check_timeout="$val" ;;
97 check-known-to-fail) check_known_to_fail=1 ;;
98 check-error-ignore) check_error_ignore=1 ;;
99 check-output-ignore) check_output_ignore=1 ;;
100 check-output-contains:) check_output_contains=1 ;;
101 check-output-excludes:) check_output_excludes=1 ;;
102 check-output-pattern) check_output_pattern=1 ;;
103 check-arch-ignore:) arch=$(uname -m)
104 check_arch_ignore="$val" ;;
105 check-arch-only:) arch=$(uname -m)
106 check_arch_only="$val" ;;
107 check-assert:) check_assert="$val" ;;
108 check-cpp-if:) check_cpp_if="$val" ;;
109
110 check-description:) ;; # ignore
111 check-note:) ;; # ignore
112 check-warning:) ;; # ignore
113 check-error-start) ;; # ignore
114 check-error-end) ;; # ignore
115 check-output-start) ;; # ignore
116 check-output-end) ;; # ignore
117 check-should-pass) ;; # ignore, unused annotation
118 check-should-fail) ;; # ignore, unused annotation
119 check-should-warn) ;; # ignore, unused annotation
120 check-*) error "$1: unknown tag '$tag'" 1 ;;
121 esac
122 done << EOT
123 $lines
124 EOT
125 }
126
127 ##
128 # helper for has_(each|none)_patterns()
129 has_patterns()
130 {
131 ifile="$1"
132 patt="$2"
133 ofile="$3"
134 cmp="$4"
135 msg="$5"
136 grep "$patt:" "$ifile" | \
137 sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
138 while read val; do
139 grep -s -q "$val" "$ofile"
140 if [ "$?" $cmp 0 ]; then
141 error " Pattern '$val' unexpectedly $msg"
142 return 1
143 fi
144 done
145
146 return $?
147 }
148
149 ##
150 # has_each_patterns(ifile tag ofile) - does ofile contains some
151 # of the patterns given by ifile's tags?
152 #
153 # returns 0 if all present, 1 otherwise
154 has_each_patterns()
155 {
156 has_patterns "$1" "$2" "$4" -ne "$3"
157 }
158
159 ##
160 # has_none_patterns(ifile tag ofile) - does ofile contains some
161 # of the patterns given by ifile's tags?
162 #
163 # returns 1 if any present, 0 otherwise
164 has_none_patterns()
165 {
166 has_patterns "$1" "$2" "$4" -eq "$3"
167 }
168
169 ##
170 # minmax_patterns(ifile tag ofile) - does ofile contains the
171 # the patterns given by ifile's tags
172 # the right number of time?
173 minmax_patterns()
174 {
175 ifile="$1"
176 patt="$2"
177 ofile="$3"
178 grep "$patt([0-9-]*\(, *\)*[0-9-]*):" "$ifile" | \
179 sed -e "s/^.*$patt(\([0-9]*\)): *\(.*\)/\1 eq \2/" \
180 -e "s/^.*$patt(\([0-9-]*\), *\([0-9-]*\)): *\(.*\)/\1 \2 \3/" | \
181 while read min max pat; do
182 n=$(grep -s "$pat" "$ofile" | wc -l)
183 if [ "$max" = "eq" ]; then
184 if [ "$n" -ne "$min" ]; then
185 error " Pattern '$pat' expected $min times but got $n times"
186 return 1
187 fi
188 continue
189 fi
190 if [ "$min" != '-' ]; then
191 if [ "$n" -lt "$min" ]; then
192 error " Pattern '$pat' expected min $min times but got $n times"
193 return 1
194 fi
195 fi
196 if [ "$max" != '-' ]; then
197 if [ "$n" -gt "$max" ]; then
198 error " Pattern '$pat' expected max $max times but got $n times"
199 return 1
200 fi
201 fi
202 done
203
204 return $?
205 }
206
207 ##
208 # arg_file(filename) - checks if filename exists
209 arg_file()
210 {
211 [ -z "$1" ] && {
212 do_usage
213 exit 1
214 }
215 [ -e "$1" ] || {
216 error "Can't open file $1"
217 exit 1
218 }
219 return 0
220 }
221
222
223 ##
224 do_usage()
225 {
226 echo "$prog_name - a tiny automatic testing script"
227 echo "Usage: $prog_name [option(s)] [command] [arguments]"
228 echo
229 echo "options:"
230 echo " -a|--abort Abort the tests as soon as one fails."
231 echo " -q|--quiet Be extra quiet while running the tests."
232 echo " --args='...' Add these options to the test command."
233 echo
234 echo "commands:"
235 echo " [file ...] Runs the test suite on the given file(s)."
236 echo " If a directory is given, run only those files."
237 echo " If no file is given, run the whole testsuite."
238 echo " single file Run the test in 'file'."
239 echo " format file [name [cmd]] Help writing a new test case using cmd."
240 echo
241 echo " [command] help Print usage."
242 }
243
244 disable()
245 {
246 disabled_tests=$(($disabled_tests + 1))
247 if [ -z "$vquiet" ]; then
248 echo " SKIP $1 ($2)"
249 fi
250 }
251
252 ##
253 # do_test(file) - tries to validate a test case
254 #
255 # it "parses" file, looking for check-* tags and tries to validate
256 # the test against an expected result
257 # returns:
258 # - 0 if the test passed,
259 # - 1 if it failed,
260 # - 2 if it is not a "test-suite" test.
261 # - 3 if the test is disabled.
262 do_test()
263 {
264 test_failed=0
265 file="$1"
266 quiet=0
267
268 get_tag_value $file
269
270 # can this test be handled by test-suite ?
271 # (it has to have a check-name key in it)
272 if [ "$check_name" = "" ]; then
273 warning "$file: test unhandled"
274 unhandled_tests=$(($unhandled_tests + 1))
275 return 2
276 fi
277 test_name="$check_name"
278
279 # does the test provide a specific command ?
280 if [ "$check_command" = "" ]; then
281 check_command="$defaut_command"
282 fi
283
284 # check for disabled commands
285 set -- $check_command
286 base_cmd=$1
287 for i in $disabled_cmds; do
288 if [ "$i" = "$base_cmd" ] ; then
289 disable "$test_name" "$file"
290 return 3
291 fi
292 done
293 if [ "$check_arch_ignore" != "" ]; then
294 if echo $arch | egrep -q -w "$check_arch_ignore"; then
295 disable "$test_name" "$file"
296 return 3
297 fi
298 fi
299 if [ "$check_arch_only" != "" ]; then
300 if ! (echo $arch | egrep -q -w "$check_arch_only"); then
301 disable "$test_name" "$file"
302 return 3
303 fi
304 fi
305 if [ "$check_assert" != "" ]; then
306 res=$(../sparse - 2>&1 >/dev/null <<- EOF
307 _Static_assert($check_assert, "$check_assert");
308 EOF
309 )
310 if [ "$res" != "" ]; then
311 disable "$test_name" "$file"
312 return 3
313 fi
314 fi
315 if [ "$check_cpp_if" != "" ]; then
316 res=$(../sparse -E - 2>/dev/null <<- EOF
317 #if !($check_cpp_if)
318 fail
319 #endif
320 EOF
321 )
322 if [ "$res" != "" ]; then
323 disable "$test_name" "$file"
324 return 3
325 fi
326 fi
327
328 if [ -z "$vquiet" ]; then
329 echo " TEST $test_name ($file)"
330 fi
331
332 verbose "Using command : $(echo "$@")"
333
334 # grab the expected exit value
335 expected_exit_value=$check_exit_value
336 verbose "Expecting exit value: $expected_exit_value"
337
338 # do we want a timeout?
339 pre_cmd=""
340 if [ $check_timeout -ne 0 ]; then
341 pre_cmd="timeout -k 1s $check_timeout"
342 fi
343
344 shift
345 # launch the test command and
346 # grab the actual output & exit value
347 eval $pre_cmd $default_path/$base_cmd $default_args "$@" \
348 1> $file.output.got 2> $file.error.got
349 actual_exit_value=$?
350
351 must_fail=$check_known_to_fail
352 [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
353 known_ko_tests=$(($known_ko_tests + $must_fail))
354
355 for stream in error output; do
356 eval ignore=\$check_${stream}_ignore
357 [ $ignore -eq 1 ] && continue
358
359 # grab the expected output
360 sed -n "/check-$stream-start/,/check-$stream-end/p" $file \
361 | grep -v check-$stream > "$file".$stream.expected
362
363 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
364 if [ "$?" -ne "0" ]; then
365 error "actual $stream text does not match expected $stream text."
366 error "see $file.$stream.* for further investigation."
367 [ $quiet -ne 1 ] && cat "$file".$stream.diff
368 test_failed=1
369 fi
370 done
371
372 if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
373 error "Actual exit value does not match the expected one."
374 error "expected $expected_exit_value, got $actual_exit_value."
375 test_failed=1
376 fi
377
378 # verify the 'check-output-contains/excludes' tags
379 if [ $check_output_contains -eq 1 ]; then
380 has_each_patterns "$file" 'check-output-contains' absent $file.output.got
381 if [ "$?" -ne "0" ]; then
382 test_failed=1
383 fi
384 fi
385 if [ $check_output_excludes -eq 1 ]; then
386 has_none_patterns "$file" 'check-output-excludes' present $file.output.got
387 if [ "$?" -ne "0" ]; then
388 test_failed=1
389 fi
390 fi
391 if [ $check_output_pattern -eq 1 ]; then
392 # verify the 'check-output-pattern(...)' tags
393 minmax_patterns "$file" 'check-output-pattern' $file.output.got
394 if [ "$?" -ne "0" ]; then
395 test_failed=1
396 fi
397 fi
398
399 if [ "$must_fail" -eq "1" ]; then
400 if [ "$test_failed" -eq "1" ]; then
401 [ -z "$vquiet" ] && \
402 echo "info: XFAIL: test '$file' is known to fail"
403 else
404 echo "error: XPASS: test '$file' is known to fail but succeed!"
405 fi
406 else
407 if [ "$test_failed" -eq "1" ]; then
408 echo "error: FAIL: test '$file' failed"
409 else
410 [ "$V" -ne "0" ] && \
411 echo "info: PASS: test '$file' passed"
412 fi
413 fi
414
415 if [ "$test_failed" -ne "$must_fail" ]; then
416 [ $abort -eq 1 ] && exit 1
417 test_failed=1
418 failed=1
419 fi
420
421 if [ "$test_failed" -eq "1" ]; then
422 ko_tests=$(($ko_tests + 1))
423 else
424 ok_tests=$(($ok_tests + 1))
425 rm -f $file.{error,output}.{expected,got,diff}
426 fi
427 return $test_failed
428 }
429
430 do_test_suite()
431 {
432 for i in $tests_list; do
433 do_test "$i"
434 done
435
436 OK=OK
437 [ $failed -eq 0 ] || OK=KO
438
439 # prints some numbers
440 tests_nr=$(($ok_tests + $ko_tests))
441 echo "$OK: out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
442 if [ "$known_ko_tests" -ne 0 ]; then
443 echo " $known_ko_tests of them are known to fail"
444 fi
445 if [ "$unhandled_tests" -ne "0" ]; then
446 echo " $unhandled_tests tests could not be handled by $prog_name"
447 fi
448 if [ "$disabled_tests" -ne "0" ]; then
449 echo " $disabled_tests tests were disabled"
450 fi
451 }
452
453 ##
454 do_format_help() {
455 echo "Usage: $prog_name [option(s)] [--]format file [name [cmd]]"
456 echo
457 echo "options:"
458 echo " -a append the created test to the input file"
459 echo " -f write a test known to fail"
460 echo " -l write a test for linearized code"
461 echo
462 echo "argument(s):"
463 echo " file file containing the test case(s)"
464 echo " name name for the test case (defaults to file)"
465 echo " cmd command to be used (defaults to 'sparse \$file')"
466 }
467
468 ##
469 # do_format([options,] file[, name[, cmd]]) - helps a test writer to format test-suite tags
470 do_format()
471 {
472 def_cmd="$default_cmd"
473 append=0
474 linear=0
475 fail=0
476
477 while [ $# -gt 1 ] ; do
478 case "$1" in
479 -a)
480 append=1 ;;
481 -f)
482 fail=1 ;;
483 -l)
484 def_cmd='test-linearize -Wno-decl $file'
485 linear=1 ;;
486 help|-*)
487 do_format_help
488 return 0
489 ;;
490 *) break ;;
491 esac
492 shift
493 continue
494 done
495
496 arg_file "$1" || return 1
497
498 file="$1"
499 fname="$2"
500 [ -z "$fname" ] && fname="$(basename "$1" .c)"
501 fcmd="$3"
502 [ -z "$fcmd" ] && fcmd="$def_cmd"
503
504 cmd=`eval echo $default_path/$fcmd`
505 $cmd 1> $file.output.got 2> $file.error.got
506 fexit_value=$?
507 [ $append != 0 ] && exec >> $file
508 cat <<_EOF
509
510 /*
511 * check-name: $fname
512 _EOF
513 if [ "$fcmd" != "$default_cmd" ]; then
514 echo " * check-command: $fcmd"
515 fi
516 if [ "$fexit_value" -ne "0" ]; then
517 echo " * check-exit-value: $fexit_value"
518 fi
519 if [ $fail != 0 ]; then
520 echo " * check-known-to-fail"
521 fi
522 if [ $linear != 0 ]; then
523 echo ' *'
524 echo ' * check-output-ignore'
525 echo ' * check-output-contains: xyz\\\\.'
526 echo ' * check-output-excludes: \\\\.'
527 fi
528 for stream in output error; do
529 if [ -s "$file.$stream.got" ]; then
530 echo " *"
531 echo " * check-$stream-start"
532 cat "$file.$stream.got"
533 echo " * check-$stream-end"
534 fi
535 done
536 echo " */"
537 return 0
538 }
539
540 ## allow flags from environment
541 set -- $SPARSE_TEST_FLAGS "$@"
542
543 ## process the flags
544 while [ "$#" -gt "0" ]; do
545 case "$1" in
546 -a|--abort)
547 abort=1
548 ;;
549 -q|--quiet)
550 vquiet=1
551 ;;
552 --args=*)
553 default_args="${1#--args=}";
554 ;;
555
556 single|--single)
557 arg_file "$2"
558 do_test "$2"
559 case "$?" in
560 0) echo "$2 passed !";;
561 1) echo "$2 failed !";;
562 2) echo "$2 can't be handled by $prog_name";;
563 esac
564 exit $failed
565 ;;
566 format|--format)
567 shift
568 do_format "$@"
569 exit 0
570 ;;
571 help)
572 do_usage
573 exit 1
574 ;;
575
576 *.c|*.cdoc)
577 tests_list="$tests_list $1"
578 ;;
579 *)
580 if [ ! -d "$1" ]; then
581 do_usage
582 exit 1
583 fi
584 tests_list="$tests_list $(find "$1" -name '*.c' | sort)"
585 ;;
586 esac
587 shift
588 done
589
590 if [ -z "$tests_list" ]; then
591 tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
592 fi
593
594 do_test_suite
595 exit $failed
596
|