1 #!/bin/sh
2
3 #set -x
4
5 cd $(dirname "$0")
6
7 default_path=".."
8 default_cmd="sparse \$file"
9 tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
10 prog_name=`basename $0`
11
12 if [ ! -x "$default_path/sparse-llvm" ]; then
13 disabled_cmds="sparsec sparsei sparse-llvm"
14 fi
15
16 # flags:
17 # - some tests gave an unexpected result
18 failed=0
19
20 # counts:
21 # - tests that have not been converted to test-suite format
22 # - tests that are disabled
23 # - tests that passed
24 # - tests that failed
25 # - tests that failed but are known to fail
26 unhandled_tests=0
27 disabled_tests=0
28 ok_tests=0
29 ko_tests=0
30 known_ko_tests=0
31
32 # defaults to not verbose
33 [ -z "$V" ] && V=0
34 [ $V -eq 0 ] && quiet=1 || quiet=0
35
36 ##
37 # get_tag_value(file) - get the 'check-<...>' tags & values
38 get_tag_value()
39 {
40 check_name=""
41 check_command="$default_cmd"
42 check_exit_value=0
43 check_timeout=0
44 check_known_to_fail=0
45 check_error_ignore=0
46 check_output_ignore=0
47 check_output_contains=0
48 check_output_excludes=0
49 check_output_pattern=0
50
51 lines=$(grep 'check-[a-z-]*' $1 | \
52 sed -e 's/^.*\(check-[a-z-]*:*\) *\(.*\)$/\1 \2/')
53
54 while read tag val; do
55 #echo "-> tag: '$tag'"
56 #echo "-> val: '$val'"
57 case $tag in
58 check-name:) check_name="$val" ;;
59 check-command:) check_command="$val" ;;
60 check-exit-value:) check_exit_value="$val" ;;
61 check-timeout:) [ -z "$val" ] && val=1
62 check_timeout="$val" ;;
63 check-known-to-fail) check_known_to_fail=1 ;;
64 check-error-ignore) check_error_ignore=1 ;;
65 check-output-ignore) check_output_ignore=1 ;;
66 check-output-contains:) check_output_contains=1 ;;
67 check-output-excludes:) check_output_excludes=1 ;;
68 check-output-pattern-) check_output_pattern=1 ;;
69 esac
70 done << EOT
71 $lines
72 EOT
73 }
74
75 ##
76 # helper for has_(each|none)_patterns()
77 has_patterns()
78 {
79 ifile="$1"
80 patt="$2"
81 ofile="$3"
82 cmp="$4"
83 grep "$patt:" "$ifile" | \
84 sed -e "s/^.*$patt: *\(.*\)$/\1/" | \
85 while read val; do
86 grep -s -q "$val" "$ofile"
87 if [ "$?" $cmp 0 ]; then
88 return 1
89 fi
90 done
91
92 return $?
93 }
94
95 ##
96 # has_each_patterns(ifile tag ofile) - does ofile contains some
97 # of the patterns given by ifile's tags?
98 #
99 # returns 0 if all present, 1 otherwise
100 has_each_patterns()
101 {
102 has_patterns "$1" "$2" "$3" -ne
103 }
104
105 ##
106 # has_none_patterns(ifile tag ofile) - does ofile contains some
107 # of the patterns given by ifile's tags?
108 #
109 # returns 1 if any present, 0 otherwise
110 has_none_patterns()
111 {
112 has_patterns "$1" "$2" "$3" -eq
113 }
114
115 ##
116 # nbr_patterns(ifile tag ofile) - does ofile contains the
117 # the patterns given by ifile's tags
118 # the right number of time?
119 nbr_patterns()
120 {
121 ifile="$1"
122 patt="$2"
123 ofile="$3"
124 grep "$patt-[0-9][0-9]*-times:" "$ifile" | \
125 sed -e "s/^.*$patt-\([0-9][0-9]*\)-times: *\(.*\)/\1 \2/" | \
126 while read nbr pat; do
127 n=$(grep -s "$pat" "$ofile" | wc -l)
128 if [ "$n" -ne "$nbr" ]; then
129 return 1
130 fi
131 done
132
133 return $?
134 }
135
136 ##
137 # verbose(string) - prints string if we are in verbose mode
138 verbose()
139 {
140 [ "$V" -eq "1" ] && echo " $1"
141 return 0
142 }
143
144 ##
145 # error(string[, die]) - prints an error and exits with value die if given
146 error()
147 {
148 [ "$quiet" -ne 1 ] && echo "error: $1"
149 [ -n "$2" ] && exit $2
150 return 0
151 }
152
153 do_usage()
154 {
155 echo "$prog_name - a tiny automatic testing script"
156 echo "Usage: $prog_name [command] [command arguments]"
157 echo
158 echo "commands:"
159 echo " none runs the whole test suite"
160 echo " single file runs the test in 'file'"
161 echo " format file [name [cmd]] helps writing a new test case using cmd"
162 echo
163 echo " help prints usage"
164 }
165
166 ##
167 # do_test(file) - tries to validate a test case
168 #
169 # it "parses" file, looking for check-* tags and tries to validate
170 # the test against an expected result
171 # returns:
172 # - 0 if the test passed,
173 # - 1 if it failed,
174 # - 2 if it is not a "test-suite" test.
175 # - 3 if the test is disabled.
176 do_test()
177 {
178 test_failed=0
179 file="$1"
180
181 get_tag_value $file
182
183 # can this test be handled by test-suite ?
184 # (it has to have a check-name key in it)
185 if [ "$check_name" = "" ]; then
186 echo "warning: test '$file' unhandled"
187 unhandled_tests=$(($unhandled_tests + 1))
188 return 2
189 fi
190 test_name="$check_name"
191
192 # does the test provide a specific command ?
193 if [ "$check_command" = "" ]; then
194 check_command="$defaut_command"
195 fi
196
197 # check for disabled commands
198 set -- $check_command
199 base_cmd=$1
200 for i in $disabled_cmds; do
201 if [ "$i" = "$base_cmd" ] ; then
202 disabled_tests=$(($disabled_tests + 1))
203 echo " DISABLE $test_name ($file)"
204 return 3
205 fi
206 done
207
208 cmd=`eval echo $default_path/$check_command`
209
210 echo " TEST $test_name ($file)"
211
212 verbose "Using command : $cmd"
213
214 # grab the expected exit value
215 expected_exit_value=$check_exit_value
216 verbose "Expecting exit value: $expected_exit_value"
217
218 # do we want a timeout?
219 if [ $check_timeout -ne 0 ]; then
220 cmd="timeout -k 1s $check_timeout $cmd"
221 fi
222
223 # grab the actual output & exit value
224 $cmd 1> $file.output.got 2> $file.error.got
225 actual_exit_value=$?
226
227 must_fail=$check_known_to_fail
228 quiet=0
229 [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1
230 known_ko_tests=$(($known_ko_tests + $must_fail))
231
232 for stream in output error; do
233 eval ignore=\$check_${stream}_ignore
234 [ $ignore -eq 1 ] && continue
235
236 # grab the expected output
237 sed -n "/check-$stream-start/,/check-$stream-end/p" $file \
238 | grep -v check-$stream > "$file".$stream.expected
239
240 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
241 if [ "$?" -ne "0" ]; then
242 error "actual $stream text does not match expected $stream text."
243 error "see $file.$stream.* for further investigation."
244 [ $quiet -ne 1 ] && cat "$file".$stream.diff
245 test_failed=1
246 fi
247 done
248
249 if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
250 error "Actual exit value does not match the expected one."
251 error "expected $expected_exit_value, got $actual_exit_value."
252 test_failed=1
253 fi
254
255 # verify the 'check-output-contains/excludes' tags
256 if [ $check_output_contains -eq 1 ]; then
257 has_each_patterns "$file" 'check-output-contains' $file.output.got
258 if [ "$?" -ne "0" ]; then
259 error "Actual output doesn't contain some of the expected patterns."
260 test_failed=1
261 fi
262 fi
263 if [ $check_output_excludes -eq 1 ]; then
264 has_none_patterns "$file" 'check-output-excludes' $file.output.got
265 if [ "$?" -ne "0" ]; then
266 error "Actual output contains some patterns which are not expected."
267 test_failed=1
268 fi
269 fi
270 if [ $check_output_pattern -eq 1 ]; then
271 # verify the 'check-output-pattern-X-times' tags
272 nbr_patterns "$file" 'check-output-pattern' $file.output.got
273 if [ "$?" -ne "0" ]; then
274 error "Actual output doesn't contain the pattern the expected number."
275 test_failed=1
276 fi
277 fi
278
279 [ "$test_failed" -eq "$must_fail" ] || failed=1
280
281 if [ "$must_fail" -eq "1" ]; then
282 if [ "$test_failed" -eq "1" ]; then
283 echo "info: test '$file' is known to fail"
284 else
285 echo "error: test '$file' is known to fail but succeed!"
286 test_failed=1
287 fi
288 fi
289
290 if [ "$test_failed" -eq "1" ]; then
291 ko_tests=$(($ko_tests + 1))
292 else
293 ok_tests=$(($ok_tests + 1))
294 rm -f $file.{error,output}.{expected,got,diff}
295 fi
296 return $test_failed
297 }
298
299 do_test_suite()
300 {
301 for i in $tests_list; do
302 do_test "$i"
303 done
304
305 # prints some numbers
306 tests_nr=$(($ok_tests + $ko_tests))
307 echo -n "Out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
308 echo " ($known_ko_tests of them are known to fail)"
309 if [ "$unhandled_tests" -ne "0" ]; then
310 echo "$unhandled_tests tests could not be handled by $prog_name"
311 fi
312 if [ "$disabled_tests" -ne "0" ]; then
313 echo "$disabled_tests tests were disabled"
314 fi
315 }
316
317 ##
318 # do_format(file[, name[, cmd]]) - helps a test writer to format test-suite tags
319 do_format()
320 {
321 if [ -z "$2" ]; then
322 fname="$1"
323 fcmd=$default_cmd
324 elif [ -z "$3" ]; then
325 fname="$2"
326 fcmd=$default_cmd
327 else
328 fname="$2"
329 fcmd="$3"
330 fi
331 file="$1"
332 cmd=`eval echo $default_path/$fcmd`
333 $cmd 1> $file.output.got 2> $file.error.got
334 fexit_value=$?
335 cat <<_EOF
336 /*
337 * check-name: $fname
338 _EOF
339 if [ "$fcmd" != "$default_cmd" ]; then
340 echo " * check-command: $fcmd"
341 fi
342 if [ "$fexit_value" -ne "0" ]; then
343 echo " * check-exit-value: $fexit_value"
344 fi
345 for stream in output error; do
346 if [ -s "$file.$stream.got" ]; then
347 echo " *"
348 echo " * check-$stream-start"
349 cat "$file.$stream.got"
350 echo " * check-$stream-end"
351 fi
352 done
353 echo " */"
354 return 0
355 }
356
357 ##
358 # arg_file(filename) - checks if filename exists
359 arg_file()
360 {
361 [ -z "$1" ] && {
362 do_usage
363 exit 1
364 }
365 [ -e "$1" ] || {
366 error "Can't open file $1"
367 exit 1
368 }
369 return 0
370 }
371
372 case "$1" in
373 '')
374 do_test_suite
375 ;;
376 single)
377 arg_file "$2"
378 do_test "$2"
379 case "$?" in
380 0) echo "$2 passed !";;
381 1) echo "$2 failed !";;
382 2) echo "$2 can't be handled by $prog_name";;
383 esac
384 ;;
385 format)
386 arg_file "$2"
387 do_format "$2" "$3" "$4"
388 ;;
389 help | *)
390 do_usage
391 exit 1
392 ;;
393 esac
394
395 exit $failed
396