1 #!/bin/sh 2 3 #set -x 4 5 cd $(dirname "$0") 6 7 default_path=".." 8 default_cmd="sparse \$file" 9 tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort` 10 prog_name=`basename $0` 11 12 if [ ! -x "$default_path/sparse-llvm" ]; then 13 disabled_cmds="sparsec sparsei sparse-llvm" 14 fi 15 16 # flags: 17 # - some tests gave an unexpected result 18 failed=0 19 20 # counts: 21 # - tests that have not been converted to test-suite format 22 # - tests that are disabled 23 # - tests that passed 24 # - tests that failed 25 # - tests that failed but are known to fail 26 unhandled_tests=0 27 disabled_tests=0 28 ok_tests=0 29 ko_tests=0 30 known_ko_tests=0 31 32 # defaults to not verbose 33 [ -z "$V" ] && V=0 34 [ $V -eq 0 ] && quiet=1 || quiet=0 35 36 ## 37 # get_tag_value(file) - get the 'check-<...>' tags & values 38 get_tag_value() 39 { 40 check_name="" 41 check_command="$default_cmd" 42 check_exit_value=0 43 check_timeout=0 44 check_known_to_fail=0 45 check_error_ignore=0 46 check_output_ignore=0 47 check_output_contains=0 48 check_output_excludes=0 49 check_output_pattern=0 50 51 lines=$(grep 'check-[a-z-]*' $1 | \ 52 sed -e 's/^.*\(check-[a-z-]*:*\) *\(.*\)$/\1 \2/') 53 54 while read tag val; do 55 #echo "-> tag: '$tag'" 56 #echo "-> val: '$val'" 57 case $tag in 58 check-name:) check_name="$val" ;; 59 check-command:) check_command="$val" ;; 60 check-exit-value:) check_exit_value="$val" ;; 61 check-timeout:) [ -z "$val" ] && val=1 62 check_timeout="$val" ;; 63 check-known-to-fail) check_known_to_fail=1 ;; 64 check-error-ignore) check_error_ignore=1 ;; 65 check-output-ignore) check_output_ignore=1 ;; 66 check-output-contains:) check_output_contains=1 ;; 67 check-output-excludes:) check_output_excludes=1 ;; 68 check-output-pattern-) check_output_pattern=1 ;; 69 esac 70 done << EOT 71 $lines 72 EOT 73 } 74 75 ## 76 # helper for has_(each|none)_patterns() 77 has_patterns() 78 { 79 ifile="$1" 80 patt="$2" 81 ofile="$3" 82 cmp="$4" 83 grep "$patt:" "$ifile" | \ 84 sed -e "s/^.*$patt: *\(.*\)$/\1/" | \ 85 while read val; do 86 grep -s -q "$val" "$ofile" 87 if [ "$?" $cmp 0 ]; then 88 return 1 89 fi 90 done 91 92 return $? 93 } 94 95 ## 96 # has_each_patterns(ifile tag ofile) - does ofile contains some 97 # of the patterns given by ifile's tags? 98 # 99 # returns 0 if all present, 1 otherwise 100 has_each_patterns() 101 { 102 has_patterns "$1" "$2" "$3" -ne 103 } 104 105 ## 106 # has_none_patterns(ifile tag ofile) - does ofile contains some 107 # of the patterns given by ifile's tags? 108 # 109 # returns 1 if any present, 0 otherwise 110 has_none_patterns() 111 { 112 has_patterns "$1" "$2" "$3" -eq 113 } 114 115 ## 116 # nbr_patterns(ifile tag ofile) - does ofile contains the 117 # the patterns given by ifile's tags 118 # the right number of time? 119 nbr_patterns() 120 { 121 ifile="$1" 122 patt="$2" 123 ofile="$3" 124 grep "$patt-[0-9][0-9]*-times:" "$ifile" | \ 125 sed -e "s/^.*$patt-\([0-9][0-9]*\)-times: *\(.*\)/\1 \2/" | \ 126 while read nbr pat; do 127 n=$(grep -s "$pat" "$ofile" | wc -l) 128 if [ "$n" -ne "$nbr" ]; then 129 return 1 130 fi 131 done 132 133 return $? 134 } 135 136 ## 137 # verbose(string) - prints string if we are in verbose mode 138 verbose() 139 { 140 [ "$V" -eq "1" ] && echo " $1" 141 return 0 142 } 143 144 ## 145 # error(string[, die]) - prints an error and exits with value die if given 146 error() 147 { 148 [ "$quiet" -ne 1 ] && echo "error: $1" 149 [ -n "$2" ] && exit $2 150 return 0 151 } 152 153 do_usage() 154 { 155 echo "$prog_name - a tiny automatic testing script" 156 echo "Usage: $prog_name [command] [command arguments]" 157 echo 158 echo "commands:" 159 echo " none runs the whole test suite" 160 echo " single file runs the test in 'file'" 161 echo " format file [name [cmd]] helps writing a new test case using cmd" 162 echo 163 echo " help prints usage" 164 } 165 166 ## 167 # do_test(file) - tries to validate a test case 168 # 169 # it "parses" file, looking for check-* tags and tries to validate 170 # the test against an expected result 171 # returns: 172 # - 0 if the test passed, 173 # - 1 if it failed, 174 # - 2 if it is not a "test-suite" test. 175 # - 3 if the test is disabled. 176 do_test() 177 { 178 test_failed=0 179 file="$1" 180 181 get_tag_value $file 182 183 # can this test be handled by test-suite ? 184 # (it has to have a check-name key in it) 185 if [ "$check_name" = "" ]; then 186 echo "warning: test '$file' unhandled" 187 unhandled_tests=$(($unhandled_tests + 1)) 188 return 2 189 fi 190 test_name="$check_name" 191 192 # does the test provide a specific command ? 193 if [ "$check_command" = "" ]; then 194 check_command="$defaut_command" 195 fi 196 197 # check for disabled commands 198 set -- $check_command 199 base_cmd=$1 200 for i in $disabled_cmds; do 201 if [ "$i" = "$base_cmd" ] ; then 202 disabled_tests=$(($disabled_tests + 1)) 203 echo " DISABLE $test_name ($file)" 204 return 3 205 fi 206 done 207 208 cmd=`eval echo $default_path/$check_command` 209 210 echo " TEST $test_name ($file)" 211 212 verbose "Using command : $cmd" 213 214 # grab the expected exit value 215 expected_exit_value=$check_exit_value 216 verbose "Expecting exit value: $expected_exit_value" 217 218 # do we want a timeout? 219 if [ $check_timeout -ne 0 ]; then 220 cmd="timeout -k 1s $check_timeout $cmd" 221 fi 222 223 # grab the actual output & exit value 224 $cmd 1> $file.output.got 2> $file.error.got 225 actual_exit_value=$? 226 227 must_fail=$check_known_to_fail 228 quiet=0 229 [ $must_fail -eq 1 ] && [ $V -eq 0 ] && quiet=1 230 known_ko_tests=$(($known_ko_tests + $must_fail)) 231 232 for stream in output error; do 233 eval ignore=\$check_${stream}_ignore 234 [ $ignore -eq 1 ] && continue 235 236 # grab the expected output 237 sed -n "/check-$stream-start/,/check-$stream-end/p" $file \ 238 | grep -v check-$stream > "$file".$stream.expected 239 240 diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff 241 if [ "$?" -ne "0" ]; then 242 error "actual $stream text does not match expected $stream text." 243 error "see $file.$stream.* for further investigation." 244 [ $quiet -ne 1 ] && cat "$file".$stream.diff 245 test_failed=1 246 fi 247 done 248 249 if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then 250 error "Actual exit value does not match the expected one." 251 error "expected $expected_exit_value, got $actual_exit_value." 252 test_failed=1 253 fi 254 255 # verify the 'check-output-contains/excludes' tags 256 if [ $check_output_contains -eq 1 ]; then 257 has_each_patterns "$file" 'check-output-contains' $file.output.got 258 if [ "$?" -ne "0" ]; then 259 error "Actual output doesn't contain some of the expected patterns." 260 test_failed=1 261 fi 262 fi 263 if [ $check_output_excludes -eq 1 ]; then 264 has_none_patterns "$file" 'check-output-excludes' $file.output.got 265 if [ "$?" -ne "0" ]; then 266 error "Actual output contains some patterns which are not expected." 267 test_failed=1 268 fi 269 fi 270 if [ $check_output_pattern -eq 1 ]; then 271 # verify the 'check-output-pattern-X-times' tags 272 nbr_patterns "$file" 'check-output-pattern' $file.output.got 273 if [ "$?" -ne "0" ]; then 274 error "Actual output doesn't contain the pattern the expected number." 275 test_failed=1 276 fi 277 fi 278 279 [ "$test_failed" -eq "$must_fail" ] || failed=1 280 281 if [ "$must_fail" -eq "1" ]; then 282 if [ "$test_failed" -eq "1" ]; then 283 echo "info: test '$file' is known to fail" 284 else 285 echo "error: test '$file' is known to fail but succeed!" 286 test_failed=1 287 fi 288 fi 289 290 if [ "$test_failed" -eq "1" ]; then 291 ko_tests=$(($ko_tests + 1)) 292 else 293 ok_tests=$(($ok_tests + 1)) 294 rm -f $file.{error,output}.{expected,got,diff} 295 fi 296 return $test_failed 297 } 298 299 do_test_suite() 300 { 301 for i in $tests_list; do 302 do_test "$i" 303 done 304 305 # prints some numbers 306 tests_nr=$(($ok_tests + $ko_tests)) 307 echo -n "Out of $tests_nr tests, $ok_tests passed, $ko_tests failed" 308 echo " ($known_ko_tests of them are known to fail)" 309 if [ "$unhandled_tests" -ne "0" ]; then 310 echo "$unhandled_tests tests could not be handled by $prog_name" 311 fi 312 if [ "$disabled_tests" -ne "0" ]; then 313 echo "$disabled_tests tests were disabled" 314 fi 315 } 316 317 ## 318 # do_format(file[, name[, cmd]]) - helps a test writer to format test-suite tags 319 do_format() 320 { 321 if [ -z "$2" ]; then 322 fname="$1" 323 fcmd=$default_cmd 324 elif [ -z "$3" ]; then 325 fname="$2" 326 fcmd=$default_cmd 327 else 328 fname="$2" 329 fcmd="$3" 330 fi 331 file="$1" 332 cmd=`eval echo $default_path/$fcmd` 333 $cmd 1> $file.output.got 2> $file.error.got 334 fexit_value=$? 335 cat <<_EOF 336 /* 337 * check-name: $fname 338 _EOF 339 if [ "$fcmd" != "$default_cmd" ]; then 340 echo " * check-command: $fcmd" 341 fi 342 if [ "$fexit_value" -ne "0" ]; then 343 echo " * check-exit-value: $fexit_value" 344 fi 345 for stream in output error; do 346 if [ -s "$file.$stream.got" ]; then 347 echo " *" 348 echo " * check-$stream-start" 349 cat "$file.$stream.got" 350 echo " * check-$stream-end" 351 fi 352 done 353 echo " */" 354 return 0 355 } 356 357 ## 358 # arg_file(filename) - checks if filename exists 359 arg_file() 360 { 361 [ -z "$1" ] && { 362 do_usage 363 exit 1 364 } 365 [ -e "$1" ] || { 366 error "Can't open file $1" 367 exit 1 368 } 369 return 0 370 } 371 372 case "$1" in 373 '') 374 do_test_suite 375 ;; 376 single) 377 arg_file "$2" 378 do_test "$2" 379 case "$?" in 380 0) echo "$2 passed !";; 381 1) echo "$2 failed !";; 382 2) echo "$2 can't be handled by $prog_name";; 383 esac 384 ;; 385 format) 386 arg_file "$2" 387 do_format "$2" "$3" "$4" 388 ;; 389 help | *) 390 do_usage 391 exit 1 392 ;; 393 esac 394 395 exit $failed 396