1 # Copyright
(C
) 1997-2024 Free Software Foundation
, Inc.
3 # This
program is free software
; you can redistribute it and
/or modify
4 # it under the terms of the GNU General Public License as published by
5 # the Free Software Foundation
; either version
3 of the License
, or
6 #
(at your option
) any later version.
8 # This
program is distributed in the hope that it will be useful
,
9 # but WITHOUT
ANY WARRANTY
; without even the implied warranty of
10 # MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the
11 # GNU General Public License
for more details.
13 # You should have received a copy of the GNU General Public License
14 # along with GCC
; see the file COPYING3.
If not see
15 #
<http
://www.gnu.org
/licenses
/>.
17 # Verify various kinds of gcov output
: line counts
, branch percentages
,
18 # and
call return percentages.
None of this is language
-specific.
20 load_lib
"target-supports.exp"
25 # clean
-gcov
-file
-- delete a working file the compiler creates
for gcov
27 # TESTCASE is the
name of the test.
28 # SUFFIX is file suffix
30 proc clean
-gcov
-file
{ testcase suffix
} {
31 set basename
[file tail $testcase
]
32 set base
[file rootname $basename
]
33 remote_file host
delete $base.$suffix
37 # clean
-gcov
-- delete the working files the compiler creates
for gcov
39 # TESTCASE is the
name of the test.
41 proc clean
-gcov
{ testcase
} {
42 clean
-gcov
-file $testcase
"gcno"
43 clean
-gcov
-file $testcase
"gcda"
44 clean
-gcov
-file $testcase
"h.gcov"
45 remote_file host
delete "$testcase.gcov"
49 # verify
-lines
-- check that line counts are as expected
51 # TESTNAME is the
name of the test
, including unique flags.
52 # TESTCASE is the
name of the test file.
53 # FILE is the
name of the gcov output file.
55 proc verify
-lines
{ testname testcase file
} {
56 #send_user
"verify-lines\n"
61 while { [gets $fd line
] >= 0 } {
62 # We want to match both
"-" and "#####" as count as well as numbers,
63 # since we want to detect lines that shouldn
't be marked as covered.
64 if [regexp "^ *(\[^:]*): *(\[0-9\\-#]+):.*count\\((\[0-9\\-#=\\.kMGTPEZY\*]+)\\)(.*)" \
65 "$line" all is n shouldbe rest] {
66 if [regexp "^ *{(.*)}" $rest all xfailed] {
67 switch [dg-process-target $xfailed] {
69 "F" { setup_xfail "*-*-*" }
73 fail "$testname line $n: no data available"
75 } elseif { $is != $shouldbe } {
76 fail "$testname line $n: is $is:should be $shouldbe"
79 pass "$testname count for line $n"
89 # verify-branches -- check that branch percentages are as expected
91 # TESTNAME is the name of the test, including unique flags.
92 # TESTCASE is the name of the test file.
93 # FILE is the name of the gcov output file.
95 # Checks are based on comments in the source file. This means to look for
96 # branch percentages 10 or 90, 20 or 80, and # 70 or 30:
97 # /* branch(10, 20, 70) */
98 # This means that all specified percentages should have been seen by now:
100 # All specified percentages must also be seen by the next branch(n) or
101 # by the end of the file.
103 # Each check depends on the compiler having generated the expected
104 # branch instructions. Don't check
for branches that might be
105 # optimized away or replaced with predicated instructions.
107 proc verify
-branches
{ testname testcase file
} {
108 #send_user
"verify-branches\n"
112 set fd
[open $file r
]
114 while { [gets $fd line
] >= 0 } {
115 regexp
"^\[^:\]+: *(\[0-9\]+):" "$line" all n
116 if [regexp
"branch" $line] {
117 verbose
"Processing branch line $n: $line" 3
118 if [regexp
"branch\\((\[0-9 \]+)\\)" "$line" all new_shouldbe] {
119 # All percentages in the current list should have been seen.
120 if {[llength $shouldbe
] != 0} {
121 fail
"$testname line $n: expected branch percentages not found: $shouldbe"
125 set shouldbe $new_shouldbe
126 #send_user
"$n: looking for: $shouldbe\n"
127 # Record the percentages to check
for. Replace
percentage
128 # n
> 50 with
100-n
, since block ordering affects the
129 # direction of a branch.
130 for {set i
0} {$i
< [llength $shouldbe
]} {incr i
} {
131 set num
[lindex $shouldbe $i
]
133 set shouldbe
[lreplace $shouldbe $i $i
[expr
100 - $num
]]
136 } elseif
[regexp
"branch +\[0-9\]+ taken (-\[0-9\]+)%" "$line" \
138 # Percentages should never be negative.
139 fail
"$testname line $n: negative percentage: $taken"
141 } elseif
[regexp
"branch +\[0-9\]+ taken (\[0-9\]+)%" "$line" \
143 #send_user
"$n: taken = $taken\n"
144 # Percentages should never be greater than
100.
146 fail
"$testname line $n: branch percentage greater than 100: $taken"
150 set taken
[expr
100 - $taken
]
152 #
If this
percentage is one to check
for then remove it
153 # from the list. It
's normal to ignore some reports.
154 set i [lsearch $shouldbe $taken]
156 set shouldbe [lreplace $shouldbe $i $i]
158 } elseif [regexp "branch\\(end\\)" "$line"] {
159 # All percentages in the list should have been seen by now.
160 if {[llength $shouldbe] != 0} {
161 fail "$testname line n: expected branch percentages not found: $shouldbe"
168 # All percentages in the list should have been seen.
169 if {[llength $shouldbe] != 0} {
170 fail "$testname line $n: expected branch percentages not found: $shouldbe"
178 # verify-conditions -- check that conditions are checked as expected
180 # TESTNAME is the name of the test, including unique flags.
181 # TESTCASE is the name of the test file.
182 # FILE is the name of the gcov output file.
184 # Checks are based on comments in the source file. Condition coverage comes
185 # with with two types of output, a summary and a list of the uncovered
186 # conditions. Both must be checked to pass the test
188 # To check for conditions, add a comment the line of a conditional:
189 # /* conditions(n/m) true(0 1) false(1) */
191 # where n/m are the covered and total conditions in the expression. The true()
192 # and false() take the indices expected *not* covered.
194 # This means that all coverage statements should have been seen:
195 # /* conditions(end) */
197 # If all conditions are covered i.e. n == m, then conditions(end) can be
198 # omitted. If either true() or false() are empty they can be omitted too.
200 # In some very specific cases there is a need to match multiple conditions on
201 # the same line, for example if (a && fn (b || c) && d), which is interpreted
202 # roughly as tmp _bc = b || c; if (a && _bc && d). The argument to fn is
203 # considered its own expression and its coverage report will be written on the
204 # same line. For these cases, use conditions(n/m; n/m;...) true(0 1;;...)
205 # where ; marks the end of the list element where the ith list matches the ith
206 # expression. The true()/false() matchers can be omitted if no expression
207 # expects them, otherwise use the empty list if all true/false outcomes are
210 # C++ can insert conditionals in the CFG that are not present in source code.
211 # These must be manually suppressed since unexpected and unhandled conditions
212 # are an error (to help combat regressions). Output can be suppressed with
213 # conditions(suppress) and conditions(end). suppress should usually be on a
216 # Some expressions, when using unnamed temporaries as operands, will have
217 # destructors in expressions. The coverage of the destructor will be reported
218 # on the same line as the expression itself, but suppress() would also swallow
219 # the expected tested-for messages. To handle these, use the destructor() [1]
220 # which will suppress everything from and including the second "conditions
223 # [1] it is important that the destructor() is *on the same line* as the
225 proc verify-conditions { testname testcase file } {
233 set fd [open $file r]
236 set keywords {"end" "suppress"}
237 while {[gets $fd line] >= 0} {
238 regexp "^\[^:\]+: *(\[0-9\]+):" "$line" all lineno
239 set prefix "$testname line $lineno"
241 if {![regexp "condition" $line]} {
245 # Missing coverage for both true and false will cause a failure, but
246 # only count it once for the report.
248 if [regexp {conditions *\([0-9a-z/; ]+\)} "$line" all] {
249 # *Very* coarse sanity check: conditions() should either be a
250 # keyword or n/m, anything else means a buggy test case. end is
251 # optional for cases where all conditions are covered, since it
252 # only expects a single line of output.
253 regexp {conditions *\(([0-9a-z/]+)\)} "$line" all e
254 if {([lsearch -exact $keywords $e] >= 0 || [regexp {\d+/\d+} "$e"]) == 0} {
255 fail "$prefix: expected conditions (n/m), (suppress) or (end); was ($e)"
260 # Any keyword means a new context. Set the error flag if not all
261 # expected output has been seen, and reset the state.
262 if {[llength $shouldt] != 0} {
263 fail "$prefix: expected 'not covered
(true
)' for terms: $shouldt"
267 if {[llength $shouldf] != 0} {
268 fail "$prefix: expected 'not covered
(false
)' for terms: $shouldf"
272 if {$shouldall ne ""} {
273 fail "$prefix: coverage summary not found; expected $shouldall"
277 if {[llength $checks] != 0} {
278 set missing [llength checks]
279 fail "$prefix: expected $missing more conditions"
288 if [regexp {destructor\(\)} "$line"] {
292 # Find the expressions on this line. There may be more, to support
293 # constructs like (a && fn (b && c) && d).
294 # The match produces lists like [conditions(n/m) n m]
298 regexp {conditions *\(([0-9 /;]+)\)} $line _ argconds
299 regexp {true *\(([0-9 ;]+)\)} $line _ argtrue
300 regexp {false *\(([0-9 ;]+)\)} $line _ argfalse
301 set condv [split $argconds ";"]
302 set truev [split $argtrue ";"]
303 set falsev [split $argfalse ";"]
304 set ncases [llength $condv]
306 for {set i 0} {$i < $ncases} {incr i} {
307 set summary [lindex $condv $i]
308 set n [lindex [split $summary "/"] 0]
309 set m [lindex [split $summary "/"] 1]
310 set newt [lindex $truev $i]
311 set newf [lindex $falsev $i]
313 # Sanity check - if the true() and false() vectors should have
314 # m-n elements to cover all uncovered conditions. Because of
315 # masking it can sometimes be surprising what terms are
316 # independent, so this makes for more robust test at the cost
317 # of being slightly more annoying to write.
318 set nterms [expr [llength $newt] + [llength $newf]]
319 set nexpected [expr {$m - $n}]
320 if {$nterms != $nexpected} {
321 fail "$prefix: expected $nexpected uncovered terms; got $nterms"
328 set shouldall [regsub -all { } "$n/$m" ""]
329 lappend checks [list $should $shouldt $shouldf $shouldall $newt $newf]
332 if {[llength $checks] > 0} {
333 # no-op - the stack of checks to do is set up
334 } elseif {$e == "end"} {
335 # no-op - state should already been reset, and errors flagged
336 } elseif {$e == "suppress"} {
339 # this should be unreachable,
340 fail "$prefix: unhandled control ($e), should be unreachable"
343 } elseif {$suppress == 1} {
344 # ignore everything in a suppress block. C++ especially can insert
345 # conditionals in exceptions and destructors which would otherwise
346 # be considered unhandled.
348 } elseif [regexp {condition +(\d+) not covered \((.*)\)} "$line" all cond condv] {
349 foreach v {true false} {
350 if [regexp $v $condv] {
351 if {"$v" == "true"} {
357 set i [lsearch [set $should] $cond]
359 set $should [lreplace [set $should] $i $i]
361 fail "$prefix: unexpected uncovered term $cond ($v)"
366 } elseif [regexp {condition outcomes covered (\d+/\d+)} "$line" all cond] {
367 # the destructor-generated "conditions covered" lines will be
368 # written after all expression-related output. Handle these by
369 # turning on suppression if the destructor-suppression is
371 if {$shouldall == "" && $destructor == 1} {
376 if {[llength $checks] == 0} {
377 fail "$prefix: unexpected summary $cond"
380 # Report any missing conditions from the previous set if this
381 # is not the first condition block
383 if {[llength $shouldt] != 0} {
384 fail "$prefix: expected 'not covered
(true
)' for terms: $shouldt"
387 if {[llength $shouldf] != 0} {
388 fail "$prefix: expected 'not covered
(false
)' for terms: $shouldf"
391 if {$shouldall ne ""} {
392 fail "$prefix: coverage summary not found; expected $shouldall"
397 set current [lindex $checks 0]
398 set checks [lreplace $checks 0 0]
399 set should [lindex $current 0]
400 set shouldt [lindex $current 1]
401 set shouldf [lindex $current 2]
402 set shouldall [lindex $current 3]
403 set newt [lindex $current 4]
404 set newf [lindex $current 5]
406 if {$cond == $shouldall} {
409 fail "$prefix: unexpected summary - expected $shouldall, got $cond"
424 # verify-calls -- check that call return percentages are as expected
426 # TESTNAME is the name of the test, including unique flags.
427 # TESTCASE is the name of the test file.
428 # FILE is the name of the gcov output file.
430 # Checks are based on comments in the source file. This means to look for
431 # call return percentages 50, 20, 33:
432 # /* returns(50, 20, 33) */
433 # This means that all specified percentages should have been seen by now:
435 # All specified percentages must also be seen by the next returns(n) or
436 # by the end of the file.
438 # Each check depends on the compiler having generated the expected
439 # call instructions. Don't check
for calls that are inserted by the
440 # compiler or that might be inlined.
442 proc verify
-calls
{ testname testcase file
} {
443 #send_user
"verify-calls\n"
447 set fd
[open $file r
]
449 while { [gets $fd line
] >= 0 } {
450 regexp
"^\[^:\]+: *(\[0-9\]+):" "$line" all n
451 if [regexp
"return" $line] {
452 verbose
"Processing returns line $n: $line" 3
453 if [regexp
"returns\\((\[0-9 \]+)\\)" "$line" all new_shouldbe] {
454 # All percentages in the current list should have been seen.
455 if {[llength $shouldbe
] != 0} {
456 fail
"$testname line $n: expected return percentages not found: $shouldbe"
460 # Record the percentages to check
for.
461 set shouldbe $new_shouldbe
462 } elseif
[regexp
"call +\[0-9\]+ returned (-\[0-9\]+)%" "$line" \
464 # Percentages should never be negative.
465 fail
"$testname line $n: negative percentage: $returns"
467 } elseif
[regexp
"call +\[0-9\]+ returned (\[0-9\]+)%" "$line" \
469 #
For branches we check that percentages are not greater than
470 #
100 but
call return percentages can be
, as
for setjmp
(), so
471 # don
't count that as an error.
473 # If this percentage is one to check for then remove it
474 # from the list. It's
normal to ignore some reports.
475 set i
[lsearch $shouldbe $returns
]
477 set shouldbe
[lreplace $shouldbe $i $i
]
479 } elseif
[regexp
"returns\\(end\\)" "$line"] {
480 # All percentages in the list should have been seen by now.
481 if {[llength $shouldbe
] != 0} {
482 fail
"$testname line $n: expected return percentages not found: $shouldbe"
489 # All percentages in the list should have been seen.
490 if {[llength $shouldbe
] != 0} {
491 fail
"$testname line $n: expected return percentages not found: $shouldbe"
498 proc gcov
-pytest
-format
-line
{ args } {
501 set testcase
[lindex $
args 0]
502 set pytest_script
[lindex $
args 1]
503 set output_line
[lindex $
args 2]
505 set index
[string first
"::" $output_line]
506 set test_output
[string range $output_line
[expr $index
+ 2] [string length $output_line
]]
508 return "$subdir/$testcase ${pytest_script}::${test_output}"
511 #
Call by dg
-final to run gcov
--json
-format which produces a JSON file
512 # that is later analysed by a pytest Python script.
513 # We pass filename of a test via GCOV_PATH environment
variable.
515 proc run
-gcov
-pytest
{ args } {
518 # Extract the test file
name from the arguments.
519 set testcase
[lindex $
args 0]
521 verbose
"Running $GCOV $testcase in $srcdir/$subdir" 2
522 set testcase
[remote_download host $testcase
]
523 set result
[remote_exec host $GCOV
"$testcase -i -abc"]
525 set pytest_script
[lindex $
args 1]
526 if { ![check_effective_target_pytest3
] } {
527 unsupported
"$pytest_script pytest python3 is missing"
531 setenv GCOV_PATH $testcase
532 spawn
-noecho python3
-m pytest
--color
=no
-rap
-s
--tb
=no $srcdir
/$subdir
/$pytest_script
534 set prefix
"\[^\r\n\]*"
536 -re
"FAILED($prefix)\[^\r\n\]+\r\n" {
537 set output
[gcov
-pytest
-format
-line $testcase $pytest_script $expect_out
(1,string
)]
541 -re
"ERROR($prefix)\[^\r\n\]+\r\n" {
542 set output
[gcov
-pytest
-format
-line $testcase $pytest_script $expect_out
(1,string
)]
546 -re
"PASSED($prefix)\[^\r\n\]+\r\n" {
547 set output
[gcov
-pytest
-format
-line $testcase $pytest_script $expect_out
(1,string
)]
556 # Called by dg
-final to run gcov and analyze the results.
558 #
ARGS consists of the optional strings
"branches" and/or "calls",
559 #
(indicating that these things should be verified
) followed by a
560 # list of arguments to provide to gcov
, including the
name of the
563 proc run
-gcov
{ args } {
568 set gcov_verify_calls
0
569 set gcov_verify_branches
0
570 set gcov_verify_conditions
0
571 set gcov_verify_lines
1
572 set gcov_verify_intermediate
0
573 set gcov_remove_gcda
0
577 if { $a
== "calls" } {
578 set gcov_verify_calls
1
579 } elseif
{ $a
== "branches" } {
580 set gcov_verify_branches
1
581 } elseif
{ $a
== "conditions" } {
582 set gcov_verify_conditions
1
583 } elseif
{ $a
== "intermediate" } {
584 set gcov_verify_intermediate
1
585 set gcov_verify_calls
0
586 set gcov_verify_branches
0
587 set gcov_verify_conditions
0
588 set gcov_verify_lines
0
589 } elseif
{ $a
== "remove-gcda" } {
590 set gcov_remove_gcda
1
591 } elseif
{ $gcov_args
== "" } {
594 switch [dg
-process
-target $a
] {
596 "F" { set xfailed 1 }
601 set testname
[testname
-for-summary
]
603 # Extract the test file
name from the arguments.
604 set testcase
[lindex $gcov_args end
]
606 if { $gcov_remove_gcda
} {
607 verbose
"Removing $testcase.gcda"
608 clean
-gcov
-file $testcase
"gcda"
611 verbose
"Running $GCOV $testcase" 2
612 set testcase
[remote_download host $testcase
]
613 set result
[remote_exec host $GCOV $gcov_args
]
614 if { [lindex $result
0] != 0 } {
618 fail
"$testname gcov failed: [lindex $result 1]"
623 set builtin_index
[string first
"File '<built-in>'" $result]
624 if { $builtin_index
!= -1 } {
625 fail
"$testname gcov failed: <built-in>.gcov should not be created"
630 #
Get the gcov output file after making sure it
exists.
631 set files
[glob
-nocomplain $testcase.gcov
]
632 if { $files
== "" } {
636 fail
"$testname gcov failed: $testcase.gcov does not exist"
640 remote_upload host $testcase.gcov $testcase.gcov
642 # Check that line execution counts are as expected.
643 if { $gcov_verify_lines
} {
644 # Check that line execution counts are as expected.
645 set lfailed
[verify
-lines $testname $testcase $testcase.gcov
]
650 #
If requested via the .x file
, check that branch and
call information
652 if { $gcov_verify_branches
} {
653 set bfailed
[verify
-branches $testname $testcase $testcase.gcov
]
657 if { $gcov_verify_conditions
} {
658 set cdfailed
[verify
-conditions $testname $testcase $testcase.gcov
]
662 if { $gcov_verify_calls
} {
663 set cfailed
[verify
-calls $testname $testcase $testcase.gcov
]
667 if { $gcov_verify_intermediate
} {
668 # Check that intermediate format has the expected format
669 set ifailed
[verify
-intermediate $testname $testcase $testcase.gcov
]
674 #
Report whether the gcov test passed or failed.
If there were
675 # multiple failures
then the message is a summary.
676 set tfailed
[expr $lfailed
+ $bfailed
+ $cdfailed
+ $cfailed
+ $ifailed
]
680 if { $tfailed
> 0 } {
681 fail
"$testname gcov: $lfailed failures in line counts, $bfailed in branch percentages, $cdfailed in condition/decision, $cfailed in return percentages, $ifailed in intermediate format"
686 pass
"$testname gcov"