complex-lowering: Better handling of PAREN_EXPR [PR68855]
[official-gcc.git] / gcc / testsuite / lib / gcov.exp
blobdd47d66d1b2eef374aa1bcdaebf47d5bbeea6c7e
1 # Copyright (C) 1997-2024 Free Software Foundation, Inc.
3 # This program is free software; you can redistribute it and/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation; either version 3 of the License, or
6 # (at your option) any later version.
7 #
8 # This program is distributed in the hope that it will be useful,
9 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License for more details.
13 # You should have received a copy of the GNU General Public License
14 # along with GCC; see the file COPYING3. If not see
15 # <http://www.gnu.org/licenses/>.
17 # Verify various kinds of gcov output: line counts, branch percentages,
18 # and call return percentages. None of this is language-specific.
20 load_lib "target-supports.exp"
22 global GCOV
25 # clean-gcov-file -- delete a working file the compiler creates for gcov
27 # TESTCASE is the name of the test.
28 # SUFFIX is file suffix
30 proc clean-gcov-file { testcase suffix } {
31 set basename [file tail $testcase]
32 set base [file rootname $basename]
33 remote_file host delete $base.$suffix
37 # clean-gcov -- delete the working files the compiler creates for gcov
39 # TESTCASE is the name of the test.
41 proc clean-gcov { testcase } {
42 clean-gcov-file $testcase "gcno"
43 clean-gcov-file $testcase "gcda"
44 clean-gcov-file $testcase "h.gcov"
45 remote_file host delete "$testcase.gcov"
49 # verify-lines -- check that line counts are as expected
51 # TESTNAME is the name of the test, including unique flags.
52 # TESTCASE is the name of the test file.
53 # FILE is the name of the gcov output file.
55 proc verify-lines { testname testcase file } {
56 #send_user "verify-lines\n"
57 global subdir
59 set failed 0
60 set fd [open $file r]
61 while { [gets $fd line] >= 0 } {
62 # We want to match both "-" and "#####" as count as well as numbers,
63 # since we want to detect lines that shouldn't be marked as covered.
64 if [regexp "^ *(\[^:]*): *(\[0-9\\-#]+):.*count\\((\[0-9\\-#=\\.kMGTPEZY\*]+)\\)(.*)" \
65 "$line" all is n shouldbe rest] {
66 if [regexp "^ *{(.*)}" $rest all xfailed] {
67 switch [dg-process-target $xfailed] {
68 "N" { continue }
69 "F" { setup_xfail "*-*-*" }
72 if { $is == "" } {
73 fail "$testname line $n: no data available"
74 incr failed
75 } elseif { $is != $shouldbe } {
76 fail "$testname line $n: is $is:should be $shouldbe"
77 incr failed
78 } else {
79 pass "$testname count for line $n"
83 close $fd
84 return $failed
89 # verify-branches -- check that branch percentages are as expected
91 # TESTNAME is the name of the test, including unique flags.
92 # TESTCASE is the name of the test file.
93 # FILE is the name of the gcov output file.
95 # Checks are based on comments in the source file. This means to look for
96 # branch percentages 10 or 90, 20 or 80, and # 70 or 30:
97 # /* branch(10, 20, 70) */
98 # This means that all specified percentages should have been seen by now:
99 # /* branch(end) */
100 # All specified percentages must also be seen by the next branch(n) or
101 # by the end of the file.
103 # Each check depends on the compiler having generated the expected
104 # branch instructions. Don't check for branches that might be
105 # optimized away or replaced with predicated instructions.
107 proc verify-branches { testname testcase file } {
108 #send_user "verify-branches\n"
110 set failed 0
111 set shouldbe ""
112 set fd [open $file r]
113 set n 0
114 while { [gets $fd line] >= 0 } {
115 regexp "^\[^:\]+: *(\[0-9\]+):" "$line" all n
116 if [regexp "branch" $line] {
117 verbose "Processing branch line $n: $line" 3
118 if [regexp "branch\\((\[0-9 \]+)\\)" "$line" all new_shouldbe] {
119 # All percentages in the current list should have been seen.
120 if {[llength $shouldbe] != 0} {
121 fail "$testname line $n: expected branch percentages not found: $shouldbe"
122 incr failed
123 set shouldbe ""
125 set shouldbe $new_shouldbe
126 #send_user "$n: looking for: $shouldbe\n"
127 # Record the percentages to check for. Replace percentage
128 # n > 50 with 100-n, since block ordering affects the
129 # direction of a branch.
130 for {set i 0} {$i < [llength $shouldbe]} {incr i} {
131 set num [lindex $shouldbe $i]
132 if {$num > 50} {
133 set shouldbe [lreplace $shouldbe $i $i [expr 100 - $num]]
136 } elseif [regexp "branch +\[0-9\]+ taken (-\[0-9\]+)%" "$line" \
137 all taken] {
138 # Percentages should never be negative.
139 fail "$testname line $n: negative percentage: $taken"
140 incr failed
141 } elseif [regexp "branch +\[0-9\]+ taken (\[0-9\]+)%" "$line" \
142 all taken] {
143 #send_user "$n: taken = $taken\n"
144 # Percentages should never be greater than 100.
145 if {$taken > 100} {
146 fail "$testname line $n: branch percentage greater than 100: $taken"
147 incr failed
149 if {$taken > 50} {
150 set taken [expr 100 - $taken]
152 # If this percentage is one to check for then remove it
153 # from the list. It's normal to ignore some reports.
154 set i [lsearch $shouldbe $taken]
155 if {$i != -1} {
156 set shouldbe [lreplace $shouldbe $i $i]
158 } elseif [regexp "branch\\(end\\)" "$line"] {
159 # All percentages in the list should have been seen by now.
160 if {[llength $shouldbe] != 0} {
161 fail "$testname line n: expected branch percentages not found: $shouldbe"
162 incr failed
164 set shouldbe ""
168 # All percentages in the list should have been seen.
169 if {[llength $shouldbe] != 0} {
170 fail "$testname line $n: expected branch percentages not found: $shouldbe"
171 incr failed
173 close $fd
174 return $failed
178 # verify-conditions -- check that conditions are checked as expected
180 # TESTNAME is the name of the test, including unique flags.
181 # TESTCASE is the name of the test file.
182 # FILE is the name of the gcov output file.
184 # Checks are based on comments in the source file. Condition coverage comes
185 # with with two types of output, a summary and a list of the uncovered
186 # conditions. Both must be checked to pass the test
188 # To check for conditions, add a comment the line of a conditional:
189 # /* conditions(n/m) true(0 1) false(1) */
191 # where n/m are the covered and total conditions in the expression. The true()
192 # and false() take the indices expected *not* covered.
194 # This means that all coverage statements should have been seen:
195 # /* conditions(end) */
197 # If all conditions are covered i.e. n == m, then conditions(end) can be
198 # omitted. If either true() or false() are empty they can be omitted too.
200 # In some very specific cases there is a need to match multiple conditions on
201 # the same line, for example if (a && fn (b || c) && d), which is interpreted
202 # roughly as tmp _bc = b || c; if (a && _bc && d). The argument to fn is
203 # considered its own expression and its coverage report will be written on the
204 # same line. For these cases, use conditions(n/m; n/m;...) true(0 1;;...)
205 # where ; marks the end of the list element where the ith list matches the ith
206 # expression. The true()/false() matchers can be omitted if no expression
207 # expects them, otherwise use the empty list if all true/false outcomes are
208 # covered.
210 # C++ can insert conditionals in the CFG that are not present in source code.
211 # These must be manually suppressed since unexpected and unhandled conditions
212 # are an error (to help combat regressions). Output can be suppressed with
213 # conditions(suppress) and conditions(end). suppress should usually be on a
214 # closing brace.
216 # Some expressions, when using unnamed temporaries as operands, will have
217 # destructors in expressions. The coverage of the destructor will be reported
218 # on the same line as the expression itself, but suppress() would also swallow
219 # the expected tested-for messages. To handle these, use the destructor() [1]
220 # which will suppress everything from and including the second "conditions
221 # covered".
223 # [1] it is important that the destructor() is *on the same line* as the
224 # conditions(m/n)
225 proc verify-conditions { testname testcase file } {
226 set failed 0
227 set suppress 0
228 set destructor 0
229 set should ""
230 set shouldt ""
231 set shouldf ""
232 set shouldall ""
233 set fd [open $file r]
234 set lineno 0
235 set checks [list]
236 set keywords {"end" "suppress"}
237 while {[gets $fd line] >= 0} {
238 regexp "^\[^:\]+: *(\[0-9\]+):" "$line" all lineno
239 set prefix "$testname line $lineno"
241 if {![regexp "condition" $line]} {
242 continue
245 # Missing coverage for both true and false will cause a failure, but
246 # only count it once for the report.
247 set ok 1
248 if [regexp {conditions *\([0-9a-z/; ]+\)} "$line" all] {
249 # *Very* coarse sanity check: conditions() should either be a
250 # keyword or n/m, anything else means a buggy test case. end is
251 # optional for cases where all conditions are covered, since it
252 # only expects a single line of output.
253 regexp {conditions *\(([0-9a-z/]+)\)} "$line" all e
254 if {([lsearch -exact $keywords $e] >= 0 || [regexp {\d+/\d+} "$e"]) == 0} {
255 fail "$prefix: expected conditions (n/m), (suppress) or (end); was ($e)"
256 incr failed
257 continue
260 # Any keyword means a new context. Set the error flag if not all
261 # expected output has been seen, and reset the state.
262 if {[llength $shouldt] != 0} {
263 fail "$prefix: expected 'not covered (true)' for terms: $shouldt"
264 set ok 0
267 if {[llength $shouldf] != 0} {
268 fail "$prefix: expected 'not covered (false)' for terms: $shouldf"
269 set ok 0
272 if {$shouldall ne ""} {
273 fail "$prefix: coverage summary not found; expected $shouldall"
274 set ok 0
277 if {[llength $checks] != 0} {
278 set missing [llength checks]
279 fail "$prefix: expected $missing more conditions"
280 set ok 0
283 set suppress 0
284 set destructor 0
285 set setup 0
286 set checks [list]
288 if [regexp {destructor\(\)} "$line"] {
289 set destructor 1
292 # Find the expressions on this line. There may be more, to support
293 # constructs like (a && fn (b && c) && d).
294 # The match produces lists like [conditions(n/m) n m]
295 set argconds ""
296 set argtrue ""
297 set argfalse ""
298 regexp {conditions *\(([0-9 /;]+)\)} $line _ argconds
299 regexp {true *\(([0-9 ;]+)\)} $line _ argtrue
300 regexp {false *\(([0-9 ;]+)\)} $line _ argfalse
301 set condv [split $argconds ";"]
302 set truev [split $argtrue ";"]
303 set falsev [split $argfalse ";"]
304 set ncases [llength $condv]
306 for {set i 0} {$i < $ncases} {incr i} {
307 set summary [lindex $condv $i]
308 set n [lindex [split $summary "/"] 0]
309 set m [lindex [split $summary "/"] 1]
310 set newt [lindex $truev $i]
311 set newf [lindex $falsev $i]
313 # Sanity check - if the true() and false() vectors should have
314 # m-n elements to cover all uncovered conditions. Because of
315 # masking it can sometimes be surprising what terms are
316 # independent, so this makes for more robust test at the cost
317 # of being slightly more annoying to write.
318 set nterms [expr [llength $newt] + [llength $newf]]
319 set nexpected [expr {$m - $n}]
320 if {$nterms != $nexpected} {
321 fail "$prefix: expected $nexpected uncovered terms; got $nterms"
322 set ok 0
324 set shouldall $e
325 set should ""
326 set shouldt $newt
327 set shouldf $newf
328 set shouldall [regsub -all { } "$n/$m" ""]
329 lappend checks [list $should $shouldt $shouldf $shouldall $newt $newf]
332 if {[llength $checks] > 0} {
333 # no-op - the stack of checks to do is set up
334 } elseif {$e == "end"} {
335 # no-op - state should already been reset, and errors flagged
336 } elseif {$e == "suppress"} {
337 set suppress 1
338 } else {
339 # this should be unreachable,
340 fail "$prefix: unhandled control ($e), should be unreachable"
341 set ok 0
343 } elseif {$suppress == 1} {
344 # ignore everything in a suppress block. C++ especially can insert
345 # conditionals in exceptions and destructors which would otherwise
346 # be considered unhandled.
347 continue
348 } elseif [regexp {condition +(\d+) not covered \((.*)\)} "$line" all cond condv] {
349 foreach v {true false} {
350 if [regexp $v $condv] {
351 if {"$v" == "true"} {
352 set should shouldt
353 } else {
354 set should shouldf
357 set i [lsearch [set $should] $cond]
358 if {$i != -1} {
359 set $should [lreplace [set $should] $i $i]
360 } else {
361 fail "$prefix: unexpected uncovered term $cond ($v)"
362 set ok 0
366 } elseif [regexp {condition outcomes covered (\d+/\d+)} "$line" all cond] {
367 # the destructor-generated "conditions covered" lines will be
368 # written after all expression-related output. Handle these by
369 # turning on suppression if the destructor-suppression is
370 # requested.
371 if {$shouldall == "" && $destructor == 1} {
372 set suppress 1
373 continue
376 if {[llength $checks] == 0} {
377 fail "$prefix: unexpected summary $cond"
378 set ok 0
379 } else {
380 # Report any missing conditions from the previous set if this
381 # is not the first condition block
382 if {$setup == 1} {
383 if {[llength $shouldt] != 0} {
384 fail "$prefix: expected 'not covered (true)' for terms: $shouldt"
385 set ok 0
387 if {[llength $shouldf] != 0} {
388 fail "$prefix: expected 'not covered (false)' for terms: $shouldf"
389 set ok 0
391 if {$shouldall ne ""} {
392 fail "$prefix: coverage summary not found; expected $shouldall"
393 set ok 0
396 set setup 1
397 set current [lindex $checks 0]
398 set checks [lreplace $checks 0 0]
399 set should [lindex $current 0]
400 set shouldt [lindex $current 1]
401 set shouldf [lindex $current 2]
402 set shouldall [lindex $current 3]
403 set newt [lindex $current 4]
404 set newf [lindex $current 5]
406 if {$cond == $shouldall} {
407 set shouldall ""
408 } else {
409 fail "$prefix: unexpected summary - expected $shouldall, got $cond"
410 set ok 0
415 if {$ok != 1} {
416 incr failed
419 close $fd
420 return $failed
424 # verify-calls -- check that call return percentages are as expected
426 # TESTNAME is the name of the test, including unique flags.
427 # TESTCASE is the name of the test file.
428 # FILE is the name of the gcov output file.
430 # Checks are based on comments in the source file. This means to look for
431 # call return percentages 50, 20, 33:
432 # /* returns(50, 20, 33) */
433 # This means that all specified percentages should have been seen by now:
434 # /* returns(end) */
435 # All specified percentages must also be seen by the next returns(n) or
436 # by the end of the file.
438 # Each check depends on the compiler having generated the expected
439 # call instructions. Don't check for calls that are inserted by the
440 # compiler or that might be inlined.
442 proc verify-calls { testname testcase file } {
443 #send_user "verify-calls\n"
445 set failed 0
446 set shouldbe ""
447 set fd [open $file r]
448 set n 0
449 while { [gets $fd line] >= 0 } {
450 regexp "^\[^:\]+: *(\[0-9\]+):" "$line" all n
451 if [regexp "return" $line] {
452 verbose "Processing returns line $n: $line" 3
453 if [regexp "returns\\((\[0-9 \]+)\\)" "$line" all new_shouldbe] {
454 # All percentages in the current list should have been seen.
455 if {[llength $shouldbe] != 0} {
456 fail "$testname line $n: expected return percentages not found: $shouldbe"
457 incr failed
458 set shouldbe ""
460 # Record the percentages to check for.
461 set shouldbe $new_shouldbe
462 } elseif [regexp "call +\[0-9\]+ returned (-\[0-9\]+)%" "$line" \
463 all returns] {
464 # Percentages should never be negative.
465 fail "$testname line $n: negative percentage: $returns"
466 incr failed
467 } elseif [regexp "call +\[0-9\]+ returned (\[0-9\]+)%" "$line" \
468 all returns] {
469 # For branches we check that percentages are not greater than
470 # 100 but call return percentages can be, as for setjmp(), so
471 # don't count that as an error.
473 # If this percentage is one to check for then remove it
474 # from the list. It's normal to ignore some reports.
475 set i [lsearch $shouldbe $returns]
476 if {$i != -1} {
477 set shouldbe [lreplace $shouldbe $i $i]
479 } elseif [regexp "returns\\(end\\)" "$line"] {
480 # All percentages in the list should have been seen by now.
481 if {[llength $shouldbe] != 0} {
482 fail "$testname line $n: expected return percentages not found: $shouldbe"
483 incr failed
485 set shouldbe ""
489 # All percentages in the list should have been seen.
490 if {[llength $shouldbe] != 0} {
491 fail "$testname line $n: expected return percentages not found: $shouldbe"
492 incr failed
494 close $fd
495 return $failed
498 proc gcov-pytest-format-line { args } {
499 global subdir
501 set testcase [lindex $args 0]
502 set pytest_script [lindex $args 1]
503 set output_line [lindex $args 2]
505 set index [string first "::" $output_line]
506 set test_output [string range $output_line [expr $index + 2] [string length $output_line]]
508 return "$subdir/$testcase ${pytest_script}::${test_output}"
511 # Call by dg-final to run gcov --json-format which produces a JSON file
512 # that is later analysed by a pytest Python script.
513 # We pass filename of a test via GCOV_PATH environment variable.
515 proc run-gcov-pytest { args } {
516 global GCOV
517 global srcdir subdir
518 # Extract the test file name from the arguments.
519 set testcase [lindex $args 0]
521 verbose "Running $GCOV $testcase in $srcdir/$subdir" 2
522 set testcase [remote_download host $testcase]
523 set result [remote_exec host $GCOV "$testcase -i -abc"]
525 set pytest_script [lindex $args 1]
526 if { ![check_effective_target_pytest3] } {
527 unsupported "$pytest_script pytest python3 is missing"
528 return
531 setenv GCOV_PATH $testcase
532 spawn -noecho python3 -m pytest --color=no -rap -s --tb=no $srcdir/$subdir/$pytest_script
534 set prefix "\[^\r\n\]*"
535 expect {
536 -re "FAILED($prefix)\[^\r\n\]+\r\n" {
537 set output [gcov-pytest-format-line $testcase $pytest_script $expect_out(1,string)]
538 fail $output
539 exp_continue
541 -re "ERROR($prefix)\[^\r\n\]+\r\n" {
542 set output [gcov-pytest-format-line $testcase $pytest_script $expect_out(1,string)]
543 fail $output
544 exp_continue
546 -re "PASSED($prefix)\[^\r\n\]+\r\n" {
547 set output [gcov-pytest-format-line $testcase $pytest_script $expect_out(1,string)]
548 pass $output
549 exp_continue
553 clean-gcov $testcase
556 # Called by dg-final to run gcov and analyze the results.
558 # ARGS consists of the optional strings "branches" and/or "calls",
559 # (indicating that these things should be verified) followed by a
560 # list of arguments to provide to gcov, including the name of the
561 # source file.
563 proc run-gcov { args } {
564 global GCOV
565 global srcdir subdir
567 set gcov_args ""
568 set gcov_verify_calls 0
569 set gcov_verify_branches 0
570 set gcov_verify_conditions 0
571 set gcov_verify_lines 1
572 set gcov_verify_intermediate 0
573 set gcov_remove_gcda 0
574 set xfailed 0
576 foreach a $args {
577 if { $a == "calls" } {
578 set gcov_verify_calls 1
579 } elseif { $a == "branches" } {
580 set gcov_verify_branches 1
581 } elseif { $a == "conditions" } {
582 set gcov_verify_conditions 1
583 } elseif { $a == "intermediate" } {
584 set gcov_verify_intermediate 1
585 set gcov_verify_calls 0
586 set gcov_verify_branches 0
587 set gcov_verify_conditions 0
588 set gcov_verify_lines 0
589 } elseif { $a == "remove-gcda" } {
590 set gcov_remove_gcda 1
591 } elseif { $gcov_args == "" } {
592 set gcov_args $a
593 } else {
594 switch [dg-process-target $a] {
595 "N" { return }
596 "F" { set xfailed 1 }
601 set testname [testname-for-summary]
603 # Extract the test file name from the arguments.
604 set testcase [lindex $gcov_args end]
606 if { $gcov_remove_gcda } {
607 verbose "Removing $testcase.gcda"
608 clean-gcov-file $testcase "gcda"
611 verbose "Running $GCOV $testcase" 2
612 set testcase [remote_download host $testcase]
613 set result [remote_exec host $GCOV $gcov_args]
614 if { [lindex $result 0] != 0 } {
615 if { $xfailed } {
616 setup_xfail "*-*-*"
618 fail "$testname gcov failed: [lindex $result 1]"
619 clean-gcov $testcase
620 return
623 set builtin_index [string first "File '<built-in>'" $result]
624 if { $builtin_index != -1 } {
625 fail "$testname gcov failed: <built-in>.gcov should not be created"
626 clean-gcov $testcase
627 return
630 # Get the gcov output file after making sure it exists.
631 set files [glob -nocomplain $testcase.gcov]
632 if { $files == "" } {
633 if { $xfailed } {
634 setup_xfail "*-*-*"
636 fail "$testname gcov failed: $testcase.gcov does not exist"
637 clean-gcov $testcase
638 return
640 remote_upload host $testcase.gcov $testcase.gcov
642 # Check that line execution counts are as expected.
643 if { $gcov_verify_lines } {
644 # Check that line execution counts are as expected.
645 set lfailed [verify-lines $testname $testcase $testcase.gcov]
646 } else {
647 set lfailed 0
650 # If requested via the .x file, check that branch and call information
651 # is correct.
652 if { $gcov_verify_branches } {
653 set bfailed [verify-branches $testname $testcase $testcase.gcov]
654 } else {
655 set bfailed 0
657 if { $gcov_verify_conditions } {
658 set cdfailed [verify-conditions $testname $testcase $testcase.gcov]
659 } else {
660 set cdfailed 0
662 if { $gcov_verify_calls } {
663 set cfailed [verify-calls $testname $testcase $testcase.gcov]
664 } else {
665 set cfailed 0
667 if { $gcov_verify_intermediate } {
668 # Check that intermediate format has the expected format
669 set ifailed [verify-intermediate $testname $testcase $testcase.gcov]
670 } else {
671 set ifailed 0
674 # Report whether the gcov test passed or failed. If there were
675 # multiple failures then the message is a summary.
676 set tfailed [expr $lfailed + $bfailed + $cdfailed + $cfailed + $ifailed]
677 if { $xfailed } {
678 setup_xfail "*-*-*"
680 if { $tfailed > 0 } {
681 fail "$testname gcov: $lfailed failures in line counts, $bfailed in branch percentages, $cdfailed in condition/decision, $cfailed in return percentages, $ifailed in intermediate format"
682 if { $xfailed } {
683 clean-gcov $testcase
685 } else {
686 pass "$testname gcov"
687 clean-gcov $testcase