t/chainlint.pl

   1 #!/usr/bin/env perl
   2 #
   3 # Copyright (c) 2021-2022 Eric Sunshine <sunshine@sunshineco.com>
   4 #
   5 # This tool scans shell scripts for test definitions and checks those tests for
   6 # problems, such as broken &&-chains, which might hide bugs in the tests
   7 # themselves or in behaviors being exercised by the tests.
   8 #
   9 # Input arguments are pathnames of shell scripts containing test definitions,
  10 # or globs referencing a collection of scripts. For each problem discovered,
  11 # the pathname of the script containing the test is printed along with the test
  12 # name and the test body with a `?!FOO?!` annotation at the location of each
  13 # detected problem, where "FOO" is a tag such as "AMP" which indicates a broken
  14 # &&-chain. Returns zero if no problems are discovered, otherwise non-zero.
  15
  16 use warnings;
  17 use strict;
  18 use Config;
  19 use File::Glob;
  20 use Getopt::Long;
  21
  22 my $jobs = -1;
  23 my $show_stats;
  24 my $emit_all;
  25
  26 # Lexer tokenizes POSIX shell scripts. It is roughly modeled after section 2.3
  27 # "Token Recognition" of POSIX chapter 2 "Shell Command Language". Although
  28 # similar to lexical analyzers for other languages, this one differs in a few
  29 # substantial ways due to quirks of the shell command language.
  30 #
  31 # For instance, in many languages, newline is just whitespace like space or
  32 # TAB, but in shell a newline is a command separator, thus a distinct lexical
  33 # token. A newline is significant and returned as a distinct token even at the
  34 # end of a shell comment.
  35 #
  36 # In other languages, `1+2` would typically be scanned as three tokens
  37 # (`1`, `+`, and `2`), but in shell it is a single token. However, the similar
  38 # `1 + 2`, which embeds whitepace, is scanned as three token in shell, as well.
  39 # In shell, several characters with special meaning lose that meaning when not
  40 # surrounded by whitespace. For instance, the negation operator `!` is special
  41 # when standing alone surrounded by whitespace; whereas in `foo!uucp` it is
  42 # just a plain character in the longer token "foo!uucp". In many other
  43 # languages, `"string"/foo:'string'` might be scanned as five tokens ("string",
  44 # `/`, `foo`, `:`, and 'string'), but in shell, it is just a single token.
  45 #
  46 # The lexical analyzer for the shell command language is also somewhat unusual
  47 # in that it recursively invokes the parser to handle the body of `$(...)`
  48 # expressions which can contain arbitrary shell code. Such expressions may be
  49 # encountered both inside and outside of double-quoted strings.
  50 #
  51 # The lexical analyzer is responsible for consuming shell here-doc bodies which
  52 # extend from the line following a `<<TAG` operator until a line consisting
  53 # solely of `TAG`. Here-doc consumption begins when a newline is encountered.
  54 # It is legal for multiple here-doc `<<TAG` operators to be present on a single
  55 # line, in which case their bodies must be present one following the next, and
  56 # are consumed in the (left-to-right) order the `<<TAG` operators appear on the
  57 # line. A special complication is that the bodies of all here-docs must be
  58 # consumed when the newline is encountered even if the parse context depth has
  59 # changed. For instance, in `cat <<A && x=$(cat <<B &&\n`, bodies of here-docs
  60 # "A" and "B" must be consumed even though "A" was introduced outside the
  61 # recursive parse context in which "B" was introduced and in which the newline
  62 # is encountered.
  63 package Lexer;
  64
  65 sub new {
  66         my ($class, $parser, $s) = @_;
  67         bless {
  68                 parser => $parser,
  69                 buff => $s,
  70                 lineno => 1,
  71                 heretags => []
  72         } => $class;
  73 }
  74
  75 sub scan_heredoc_tag {
  76         my $self = shift @_;
  77         ${$self->{buff}} =~ /\G(-?)/gc;
  78         my $indented = $1;
  79         my $token = $self->scan_token();
  80         return "<<$indented" unless $token;
  81         my $tag = $token->[0];
  82         $tag =~ s/['"\\]//g;
  83         push(@{$self->{heretags}}, $indented ? "\t$tag" : "$tag");
  84         return "<<$indented$tag";
  85 }
  86
  87 sub scan_op {
  88         my ($self, $c) = @_;
  89         my $b = $self->{buff};
  90         return $c unless $$b =~ /\G(.)/sgc;
  91         my $cc = $c . $1;
  92         return scan_heredoc_tag($self) if $cc eq '<<';
  93         return $cc if $cc =~ /^(?:&&|\|\||>>|;;|<&|>&|<>|>\|)$/;
  94         pos($$b)--;
  95         return $c;
  96 }
  97
  98 sub scan_sqstring {
  99         my $self = shift @_;
 100         ${$self->{buff}} =~ /\G([^']*'|.*\z)/sgc;
 101         my $s = $1;
 102         $self->{lineno} += () = $s =~ /\n/sg;
 103         return "'" . $s;
 104 }
 105
 106 sub scan_dqstring {
 107         my $self = shift @_;
 108         my $b = $self->{buff};
 109         my $s = '"';
 110         while (1) {
 111                 # slurp up non-special characters
 112                 $s .= $1 if $$b =~ /\G([^"\$\\]+)/gc;
 113                 # handle special characters
 114                 last unless $$b =~ /\G(.)/sgc;
 115                 my $c = $1;
 116                 $s .= '"', last if $c eq '"';
 117                 $s .= '$' . $self->scan_dollar(), next if $c eq '$';
 118                 if ($c eq '\\') {
 119                         $s .= '\\', last unless $$b =~ /\G(.)/sgc;
 120                         $c = $1;
 121                         $self->{lineno}++, next if $c eq "\n"; # line splice
 122                         # backslash escapes only $, `, ", \ in dq-string
 123                         $s .= '\\' unless $c =~ /^[\$`"\\]$/;
 124                         $s .= $c;
 125                         next;
 126                 }
 127                 die("internal error scanning dq-string '$c'\n");
 128         }
 129         $self->{lineno} += () = $s =~ /\n/sg;
 130         return $s;
 131 }
 132
 133 sub scan_balanced {
 134         my ($self, $c1, $c2) = @_;
 135         my $b = $self->{buff};
 136         my $depth = 1;
 137         my $s = $c1;
 138         while ($$b =~ /\G([^\Q$c1$c2\E]*(?:[\Q$c1$c2\E]|\z))/gc) {
 139                 $s .= $1;
 140                 $depth++, next if $s =~ /\Q$c1\E$/;
 141                 $depth--;
 142                 last if $depth == 0;
 143         }
 144         $self->{lineno} += () = $s =~ /\n/sg;
 145         return $s;
 146 }
 147
 148 sub scan_subst {
 149         my $self = shift @_;
 150         my @tokens = $self->{parser}->parse(qr/^\)$/);
 151         $self->{parser}->next_token(); # closing ")"
 152         return @tokens;
 153 }
 154
 155 sub scan_dollar {
 156         my $self = shift @_;
 157         my $b = $self->{buff};
 158         return $self->scan_balanced('(', ')') if $$b =~ /\G\((?=\()/gc; # $((...))
 159         return '(' . join(' ', map {$_->[0]} $self->scan_subst()) . ')' if $$b =~ /\G\(/gc; # $(...)
 160         return $self->scan_balanced('{', '}') if $$b =~ /\G\{/gc; # ${...}
 161         return $1 if $$b =~ /\G(\w+)/gc; # $var
 162         return $1 if $$b =~ /\G([@*#?$!0-9-])/gc; # $*, $1, $$, etc.
 163         return '';
 164 }
 165
 166 sub swallow_heredocs {
 167         my $self = shift @_;
 168         my $b = $self->{buff};
 169         my $tags = $self->{heretags};
 170         while (my $tag = shift @$tags) {
 171                 my $start = pos($$b);
 172                 my $indent = $tag =~ s/^\t// ? '\\s*' : '';
 173                 $$b =~ /(?:\G|\n)$indent\Q$tag\E(?:\n|\z)/gc;
 174                 my $body = substr($$b, $start, pos($$b) - $start);
 175                 $self->{lineno} += () = $body =~ /\n/sg;
 176         }
 177 }
 178
 179 sub scan_token {
 180         my $self = shift @_;
 181         my $b = $self->{buff};
 182         my $token = '';
 183         my ($start, $startln);
 184 RESTART:
 185         $startln = $self->{lineno};
 186         $$b =~ /\G[ \t]+/gc; # skip whitespace (but not newline)
 187         $start = pos($$b) || 0;
 188         $self->{lineno}++, return ["\n", $start, pos($$b), $startln, $startln] if $$b =~ /\G#[^\n]*(?:\n|\z)/gc; # comment
 189         while (1) {
 190                 # slurp up non-special characters
 191                 $token .= $1 if $$b =~ /\G([^\\;&|<>(){}'"\$\s]+)/gc;
 192                 # handle special characters
 193                 last unless $$b =~ /\G(.)/sgc;
 194                 my $c = $1;
 195                 pos($$b)--, last if $c =~ /^[ \t]$/; # whitespace ends token
 196                 pos($$b)--, last if length($token) && $c =~ /^[;&|<>(){}\n]$/;
 197                 $token .= $self->scan_sqstring(), next if $c eq "'";
 198                 $token .= $self->scan_dqstring(), next if $c eq '"';
 199                 $token .= $c . $self->scan_dollar(), next if $c eq '$';
 200                 $self->{lineno}++, $self->swallow_heredocs(), $token = $c, last if $c eq "\n";
 201                 $token = $self->scan_op($c), last if $c =~ /^[;&|<>]$/;
 202                 $token = $c, last if $c =~ /^[(){}]$/;
 203                 if ($c eq '\\') {
 204                         $token .= '\\', last unless $$b =~ /\G(.)/sgc;
 205                         $c = $1;
 206                         $self->{lineno}++, next if $c eq "\n" && length($token); # line splice
 207                         $self->{lineno}++, goto RESTART if $c eq "\n"; # line splice
 208                         $token .= '\\' . $c;
 209                         next;
 210                 }
 211                 die("internal error scanning character '$c'\n");
 212         }
 213         return length($token) ? [$token, $start, pos($$b), $startln, $self->{lineno}] : undef;
 214 }
 215
 216 # ShellParser parses POSIX shell scripts (with minor extensions for Bash). It
 217 # is a recursive descent parser very roughly modeled after section 2.10 "Shell
 218 # Grammar" of POSIX chapter 2 "Shell Command Language".
 219 package ShellParser;
 220
 221 sub new {
 222         my ($class, $s) = @_;
 223         my $self = bless {
 224                 buff => [],
 225                 stop => [],
 226                 output => []
 227         } => $class;
 228         $self->{lexer} = Lexer->new($self, $s);
 229         return $self;
 230 }
 231
 232 sub next_token {
 233         my $self = shift @_;
 234         return pop(@{$self->{buff}}) if @{$self->{buff}};
 235         return $self->{lexer}->scan_token();
 236 }
 237
 238 sub untoken {
 239         my $self = shift @_;
 240         push(@{$self->{buff}}, @_);
 241 }
 242
 243 sub peek {
 244         my $self = shift @_;
 245         my $token = $self->next_token();
 246         return undef unless defined($token);
 247         $self->untoken($token);
 248         return $token;
 249 }
 250
 251 sub stop_at {
 252         my ($self, $token) = @_;
 253         return 1 unless defined($token);
 254         my $stop = ${$self->{stop}}[-1] if @{$self->{stop}};
 255         return defined($stop) && $token->[0] =~ $stop;
 256 }
 257
 258 sub expect {
 259         my ($self, $expect) = @_;
 260         my $token = $self->next_token();
 261         return $token if defined($token) && $token->[0] eq $expect;
 262         push(@{$self->{output}}, "?!ERR?! expected '$expect' but found '" . (defined($token) ? $token->[0] : "<end-of-input>") . "'\n");
 263         $self->untoken($token) if defined($token);
 264         return ();
 265 }
 266
 267 sub optional_newlines {
 268         my $self = shift @_;
 269         my @tokens;
 270         while (my $token = $self->peek()) {
 271                 last unless $token->[0] eq "\n";
 272                 push(@tokens, $self->next_token());
 273         }
 274         return @tokens;
 275 }
 276
 277 sub parse_group {
 278         my $self = shift @_;
 279         return ($self->parse(qr/^}$/),
 280                 $self->expect('}'));
 281 }
 282
 283 sub parse_subshell {
 284         my $self = shift @_;
 285         return ($self->parse(qr/^\)$/),
 286                 $self->expect(')'));
 287 }
 288
 289 sub parse_case_pattern {
 290         my $self = shift @_;
 291         my @tokens;
 292         while (defined(my $token = $self->next_token())) {
 293                 push(@tokens, $token);
 294                 last if $token->[0] eq ')';
 295         }
 296         return @tokens;
 297 }
 298
 299 sub parse_case {
 300         my $self = shift @_;
 301         my @tokens;
 302         push(@tokens,
 303              $self->next_token(), # subject
 304              $self->optional_newlines(),
 305              $self->expect('in'),
 306              $self->optional_newlines());
 307         while (1) {
 308                 my $token = $self->peek();
 309                 last unless defined($token) && $token->[0] ne 'esac';
 310                 push(@tokens,
 311                      $self->parse_case_pattern(),
 312                      $self->optional_newlines(),
 313                      $self->parse(qr/^(?:;;|esac)$/)); # item body
 314                 $token = $self->peek();
 315                 last unless defined($token) && $token->[0] ne 'esac';
 316                 push(@tokens,
 317                      $self->expect(';;'),
 318                      $self->optional_newlines());
 319         }
 320         push(@tokens, $self->expect('esac'));
 321         return @tokens;
 322 }
 323
 324 sub parse_for {
 325         my $self = shift @_;
 326         my @tokens;
 327         push(@tokens,
 328              $self->next_token(), # variable
 329              $self->optional_newlines());
 330         my $token = $self->peek();
 331         if (defined($token) && $token->[0] eq 'in') {
 332                 push(@tokens,
 333                      $self->expect('in'),
 334                      $self->optional_newlines());
 335         }
 336         push(@tokens,
 337              $self->parse(qr/^do$/), # items
 338              $self->expect('do'),
 339              $self->optional_newlines(),
 340              $self->parse_loop_body(),
 341              $self->expect('done'));
 342         return @tokens;
 343 }
 344
 345 sub parse_if {
 346         my $self = shift @_;
 347         my @tokens;
 348         while (1) {
 349                 push(@tokens,
 350                      $self->parse(qr/^then$/), # if/elif condition
 351                      $self->expect('then'),
 352                      $self->optional_newlines(),
 353                      $self->parse(qr/^(?:elif|else|fi)$/)); # if/elif body
 354                 my $token = $self->peek();
 355                 last unless defined($token) && $token->[0] eq 'elif';
 356                 push(@tokens, $self->expect('elif'));
 357         }
 358         my $token = $self->peek();
 359         if (defined($token) && $token->[0] eq 'else') {
 360                 push(@tokens,
 361                      $self->expect('else'),
 362                      $self->optional_newlines(),
 363                      $self->parse(qr/^fi$/)); # else body
 364         }
 365         push(@tokens, $self->expect('fi'));
 366         return @tokens;
 367 }
 368
 369 sub parse_loop_body {
 370         my $self = shift @_;
 371         return $self->parse(qr/^done$/);
 372 }
 373
 374 sub parse_loop {
 375         my $self = shift @_;
 376         return ($self->parse(qr/^do$/), # condition
 377                 $self->expect('do'),
 378                 $self->optional_newlines(),
 379                 $self->parse_loop_body(),
 380                 $self->expect('done'));
 381 }
 382
 383 sub parse_func {
 384         my $self = shift @_;
 385         return ($self->expect('('),
 386                 $self->expect(')'),
 387                 $self->optional_newlines(),
 388                 $self->parse_cmd()); # body
 389 }
 390
 391 sub parse_bash_array_assignment {
 392         my $self = shift @_;
 393         my @tokens = $self->expect('(');
 394         while (defined(my $token = $self->next_token())) {
 395                 push(@tokens, $token);
 396                 last if $token->[0] eq ')';
 397         }
 398         return @tokens;
 399 }
 400
 401 my %compound = (
 402         '{' => \&parse_group,
 403         '(' => \&parse_subshell,
 404         'case' => \&parse_case,
 405         'for' => \&parse_for,
 406         'if' => \&parse_if,
 407         'until' => \&parse_loop,
 408         'while' => \&parse_loop);
 409
 410 sub parse_cmd {
 411         my $self = shift @_;
 412         my $cmd = $self->next_token();
 413         return () unless defined($cmd);
 414         return $cmd if $cmd->[0] eq "\n";
 415
 416         my $token;
 417         my @tokens = $cmd;
 418         if ($cmd->[0] eq '!') {
 419                 push(@tokens, $self->parse_cmd());
 420                 return @tokens;
 421         } elsif (my $f = $compound{$cmd->[0]}) {
 422                 push(@tokens, $self->$f());
 423         } elsif (defined($token = $self->peek()) && $token->[0] eq '(') {
 424                 if ($cmd->[0] !~ /\w=$/) {
 425                         push(@tokens, $self->parse_func());
 426                         return @tokens;
 427                 }
 428                 my @array = $self->parse_bash_array_assignment();
 429                 $tokens[-1]->[0] .= join(' ', map {$_->[0]} @array);
 430                 $tokens[-1]->[2] = $array[$#array][2] if @array;
 431         }
 432
 433         while (defined(my $token = $self->next_token())) {
 434                 $self->untoken($token), last if $self->stop_at($token);
 435                 push(@tokens, $token);
 436                 last if $token->[0] =~ /^(?:[;&\n|]|&&|\|\|)$/;
 437         }
 438         push(@tokens, $self->next_token()) if $tokens[-1]->[0] ne "\n" && defined($token = $self->peek()) && $token->[0] eq "\n";
 439         return @tokens;
 440 }
 441
 442 sub accumulate {
 443         my ($self, $tokens, $cmd) = @_;
 444         push(@$tokens, @$cmd);
 445 }
 446
 447 sub parse {
 448         my ($self, $stop) = @_;
 449         push(@{$self->{stop}}, $stop);
 450         goto DONE if $self->stop_at($self->peek());
 451         my @tokens;
 452         while (my @cmd = $self->parse_cmd()) {
 453                 $self->accumulate(\@tokens, \@cmd);
 454                 last if $self->stop_at($self->peek());
 455         }
 456 DONE:
 457         pop(@{$self->{stop}});
 458         return @tokens;
 459 }
 460
 461 # TestParser is a subclass of ShellParser which, beyond parsing shell script
 462 # code, is also imbued with semantic knowledge of test construction, and checks
 463 # tests for common problems (such as broken &&-chains) which might hide bugs in
 464 # the tests themselves or in behaviors being exercised by the tests. As such,
 465 # TestParser is only called upon to parse test bodies, not the top-level
 466 # scripts in which the tests are defined.
 467 package TestParser;
 468
 469 use base 'ShellParser';
 470
 471 sub new {
 472         my $class = shift @_;
 473         my $self = $class->SUPER::new(@_);
 474         $self->{problems} = [];
 475         return $self;
 476 }
 477
 478 sub find_non_nl {
 479         my $tokens = shift @_;
 480         my $n = shift @_;
 481         $n = $#$tokens if !defined($n);
 482         $n-- while $n >= 0 && $$tokens[$n]->[0] eq "\n";
 483         return $n;
 484 }
 485
 486 sub ends_with {
 487         my ($tokens, $needles) = @_;
 488         my $n = find_non_nl($tokens);
 489         for my $needle (reverse(@$needles)) {
 490                 return undef if $n < 0;
 491                 $n = find_non_nl($tokens, $n), next if $needle eq "\n";
 492                 return undef if $$tokens[$n]->[0] !~ $needle;
 493                 $n--;
 494         }
 495         return 1;
 496 }
 497
 498 sub match_ending {
 499         my ($tokens, $endings) = @_;
 500         for my $needles (@$endings) {
 501                 next if @$tokens < scalar(grep {$_ ne "\n"} @$needles);
 502                 return 1 if ends_with($tokens, $needles);
 503         }
 504         return undef;
 505 }
 506
 507 sub parse_loop_body {
 508         my $self = shift @_;
 509         my @tokens = $self->SUPER::parse_loop_body(@_);
 510         # did loop signal failure via "|| return" or "|| exit"?
 511         return @tokens if !@tokens || grep {$_->[0] =~ /^(?:return|exit|\$\?)$/} @tokens;
 512         # did loop upstream of a pipe signal failure via "|| echo 'impossible
 513         # text'" as the final command in the loop body?
 514         return @tokens if ends_with(\@tokens, [qr/^\|\|$/, "\n", qr/^echo$/, qr/^.+$/]);
 515         # flag missing "return/exit" handling explicit failure in loop body
 516         my $n = find_non_nl(\@tokens);
 517         push(@{$self->{problems}}, ['LOOP', $tokens[$n]]);
 518         return @tokens;
 519 }
 520
 521 my @safe_endings = (
 522         [qr/^(?:&&|\|\||\||&)$/],
 523         [qr/^(?:exit|return)$/, qr/^(?:\d+|\$\?)$/],
 524         [qr/^(?:exit|return)$/, qr/^(?:\d+|\$\?)$/, qr/^;$/],
 525         [qr/^(?:exit|return|continue)$/],
 526         [qr/^(?:exit|return|continue)$/, qr/^;$/]);
 527
 528 sub accumulate {
 529         my ($self, $tokens, $cmd) = @_;
 530         my $problems = $self->{problems};
 531
 532         # no previous command to check for missing "&&"
 533         goto DONE unless @$tokens;
 534
 535         # new command is empty line; can't yet check if previous is missing "&&"
 536         goto DONE if @$cmd == 1 && $$cmd[0]->[0] eq "\n";
 537
 538         # did previous command end with "&&", "|", "|| return" or similar?
 539         goto DONE if match_ending($tokens, \@safe_endings);
 540
 541         # if this command handles "$?" specially, then okay for previous
 542         # command to be missing "&&"
 543         for my $token (@$cmd) {
 544                 goto DONE if $token->[0] =~ /\$\?/;
 545         }
 546
 547         # if this command is "false", "return 1", or "exit 1" (which signal
 548         # failure explicitly), then okay for all preceding commands to be
 549         # missing "&&"
 550         if ($$cmd[0]->[0] =~ /^(?:false|return|exit)$/) {
 551                 @$problems = grep {$_->[0] ne 'AMP'} @$problems;
 552                 goto DONE;
 553         }
 554
 555         # flag missing "&&" at end of previous command
 556         my $n = find_non_nl($tokens);
 557         push(@$problems, ['AMP', $tokens->[$n]]) unless $n < 0;
 558
 559 DONE:
 560         $self->SUPER::accumulate($tokens, $cmd);
 561 }
 562
 563 # ScriptParser is a subclass of ShellParser which identifies individual test
 564 # definitions within test scripts, and passes each test body through TestParser
 565 # to identify possible problems. ShellParser detects test definitions not only
 566 # at the top-level of test scripts but also within compound commands such as
 567 # loops and function definitions.
 568 package ScriptParser;
 569
 570 use base 'ShellParser';
 571
 572 sub new {
 573         my $class = shift @_;
 574         my $self = $class->SUPER::new(@_);
 575         $self->{ntests} = 0;
 576         return $self;
 577 }
 578
 579 # extract the raw content of a token, which may be a single string or a
 580 # composition of multiple strings and non-string character runs; for instance,
 581 # `"test body"` unwraps to `test body`; `word"a b"42'c d'` to `worda b42c d`
 582 sub unwrap {
 583         my $token = (@_ ? shift @_ : $_)->[0];
 584         # simple case: 'sqstring' or "dqstring"
 585         return $token if $token =~ s/^'([^']*)'$/$1/;
 586         return $token if $token =~ s/^"([^"]*)"$/$1/;
 587
 588         # composite case
 589         my ($s, $q, $escaped);
 590         while (1) {
 591                 # slurp up non-special characters
 592                 $s .= $1 if $token =~ /\G([^\\'"]*)/gc;
 593                 # handle special characters
 594                 last unless $token =~ /\G(.)/sgc;
 595                 my $c = $1;
 596                 $q = undef, next if defined($q) && $c eq $q;
 597                 $q = $c, next if !defined($q) && $c =~ /^['"]$/;
 598                 if ($c eq '\\') {
 599                         last unless $token =~ /\G(.)/sgc;
 600                         $c = $1;
 601                         $s .= '\\' if $c eq "\n"; # preserve line splice
 602                 }
 603                 $s .= $c;
 604         }
 605         return $s
 606 }
 607
 608 sub check_test {
 609         my $self = shift @_;
 610         my ($title, $body) = map(unwrap, @_);
 611         $self->{ntests}++;
 612         my $parser = TestParser->new(\$body);
 613         my @tokens = $parser->parse();
 614         my $problems = $parser->{problems};
 615         return unless $emit_all || @$problems;
 616         my $c = main::fd_colors(1);
 617         my $start = 0;
 618         my $checked = '';
 619         for (sort {$a->[1]->[2] <=> $b->[1]->[2]} @$problems) {
 620                 my ($label, $token) = @$_;
 621                 my $pos = $token->[2];
 622                 $checked .= substr($body, $start, $pos - $start) . " ?!$label?! ";
 623                 $start = $pos;
 624         }
 625         $checked .= substr($body, $start);
 626         $checked =~ s/^\n//;
 627         $checked =~ s/(\s) \?!/$1?!/mg;
 628         $checked =~ s/\?! (\s)/?!$1/mg;
 629         $checked =~ s/(\?![^?]+\?!)/$c->{rev}$c->{red}$1$c->{reset}/mg;
 630         $checked .= "\n" unless $checked =~ /\n$/;
 631         push(@{$self->{output}}, "$c->{blue}# chainlint: $title$c->{reset}\n$checked");
 632 }
 633
 634 sub parse_cmd {
 635         my $self = shift @_;
 636         my @tokens = $self->SUPER::parse_cmd();
 637         return @tokens unless @tokens && $tokens[0]->[0] =~ /^test_expect_(?:success|failure)$/;
 638         my $n = $#tokens;
 639         $n-- while $n >= 0 && $tokens[$n]->[0] =~ /^(?:[;&\n|]|&&|\|\|)$/;
 640         $self->check_test($tokens[1], $tokens[2]) if $n == 2; # title body
 641         $self->check_test($tokens[2], $tokens[3]) if $n > 2;  # prereq title body
 642         return @tokens;
 643 }
 644
 645 # main contains high-level functionality for processing command-line switches,
 646 # feeding input test scripts to ScriptParser, and reporting results.
 647 package main;
 648
 649 my $getnow = sub { return time(); };
 650 my $interval = sub { return time() - shift; };
 651 if (eval {require Time::HiRes; Time::HiRes->import(); 1;}) {
 652         $getnow = sub { return [Time::HiRes::gettimeofday()]; };
 653         $interval = sub { return Time::HiRes::tv_interval(shift); };
 654 }
 655
 656 # Restore TERM if test framework set it to "dumb" so 'tput' will work; do this
 657 # outside of get_colors() since under 'ithreads' all threads use %ENV of main
 658 # thread and ignore %ENV changes in subthreads.
 659 $ENV{TERM} = $ENV{USER_TERM} if $ENV{USER_TERM};
 660
 661 my @NOCOLORS = (bold => '', rev => '', reset => '', blue => '', green => '', red => '');
 662 my %COLORS = ();
 663 sub get_colors {
 664         return \%COLORS if %COLORS;
 665         if (exists($ENV{NO_COLOR})) {
 666                 %COLORS = @NOCOLORS;
 667                 return \%COLORS;
 668         }
 669         if ($ENV{TERM} =~ /xterm|xterm-\d+color|xterm-new|xterm-direct|nsterm|nsterm-\d+color|nsterm-direct/) {
 670                 %COLORS = (bold  => "\e[1m",
 671                            rev   => "\e[7m",
 672                            reset => "\e[0m",
 673                            blue  => "\e[34m",
 674                            green => "\e[32m",
 675                            red   => "\e[31m");
 676                 return \%COLORS;
 677         }
 678         if (system("tput sgr0 >/dev/null 2>&1") == 0 &&
 679             system("tput bold >/dev/null 2>&1") == 0 &&
 680             system("tput rev  >/dev/null 2>&1") == 0 &&
 681             system("tput setaf 1 >/dev/null 2>&1") == 0) {
 682                 %COLORS = (bold  => `tput bold`,
 683                            rev   => `tput rev`,
 684                            reset => `tput sgr0`,
 685                            blue  => `tput setaf 4`,
 686                            green => `tput setaf 2`,
 687                            red   => `tput setaf 1`);
 688                 return \%COLORS;
 689         }
 690         %COLORS = @NOCOLORS;
 691         return \%COLORS;
 692 }
 693
 694 my %FD_COLORS = ();
 695 sub fd_colors {
 696         my $fd = shift;
 697         return $FD_COLORS{$fd} if exists($FD_COLORS{$fd});
 698         $FD_COLORS{$fd} = -t $fd ? get_colors() : {@NOCOLORS};
 699         return $FD_COLORS{$fd};
 700 }
 701
 702 sub ncores {
 703         # Windows
 704         return $ENV{NUMBER_OF_PROCESSORS} if exists($ENV{NUMBER_OF_PROCESSORS});
 705         # Linux / MSYS2 / Cygwin / WSL
 706         do { local @ARGV='/proc/cpuinfo'; return scalar(grep(/^processor\s*:/, <>)); } if -r '/proc/cpuinfo';
 707         # macOS & BSD
 708         return qx/sysctl -n hw.ncpu/ if $^O =~ /(?:^darwin$|bsd)/;
 709         return 1;
 710 }
 711
 712 sub show_stats {
 713         my ($start_time, $stats) = @_;
 714         my $walltime = $interval->($start_time);
 715         my ($usertime) = times();
 716         my ($total_workers, $total_scripts, $total_tests, $total_errs) = (0, 0, 0, 0);
 717         my $c = fd_colors(2);
 718         print(STDERR $c->{green});
 719         for (@$stats) {
 720                 my ($worker, $nscripts, $ntests, $nerrs) = @$_;
 721                 print(STDERR "worker $worker: $nscripts scripts, $ntests tests, $nerrs errors\n");
 722                 $total_workers++;
 723                 $total_scripts += $nscripts;
 724                 $total_tests += $ntests;
 725                 $total_errs += $nerrs;
 726         }
 727         printf(STDERR "total: %d workers, %d scripts, %d tests, %d errors, %.2fs/%.2fs (wall/user)$c->{reset}\n", $total_workers, $total_scripts, $total_tests, $total_errs, $walltime, $usertime);
 728 }
 729
 730 sub check_script {
 731         my ($id, $next_script, $emit) = @_;
 732         my ($nscripts, $ntests, $nerrs) = (0, 0, 0);
 733         while (my $path = $next_script->()) {
 734                 $nscripts++;
 735                 my $fh;
 736                 unless (open($fh, "<", $path)) {
 737                         $emit->("?!ERR?! $path: $!\n");
 738                         next;
 739                 }
 740                 my $s = do { local $/; <$fh> };
 741                 close($fh);
 742                 my $parser = ScriptParser->new(\$s);
 743                 1 while $parser->parse_cmd();
 744                 if (@{$parser->{output}}) {
 745                         my $c = fd_colors(1);
 746                         my $s = join('', @{$parser->{output}});
 747                         $emit->("$c->{bold}$c->{blue}# chainlint: $path$c->{reset}\n" . $s);
 748                         $nerrs += () = $s =~ /\?![^?]+\?!/g;
 749                 }
 750                 $ntests += $parser->{ntests};
 751         }
 752         return [$id, $nscripts, $ntests, $nerrs];
 753 }
 754
 755 sub exit_code {
 756         my $stats = shift @_;
 757         for (@$stats) {
 758                 my ($worker, $nscripts, $ntests, $nerrs) = @$_;
 759                 return 1 if $nerrs;
 760         }
 761         return 0;
 762 }
 763
 764 Getopt::Long::Configure(qw{bundling});
 765 GetOptions(
 766         "emit-all!" => \$emit_all,
 767         "jobs|j=i" => \$jobs,
 768         "stats|show-stats!" => \$show_stats) or die("option error\n");
 769 $jobs = ncores() if $jobs < 1;
 770
 771 my $start_time = $getnow->();
 772 my @stats;
 773
 774 my @scripts;
 775 push(@scripts, File::Glob::bsd_glob($_)) for (@ARGV);
 776 unless (@scripts) {
 777         show_stats($start_time, \@stats) if $show_stats;
 778         exit;
 779 }
 780
 781 unless ($Config{useithreads} && eval {
 782         require threads; threads->import();
 783         require Thread::Queue; Thread::Queue->import();
 784         1;
 785         }) {
 786         push(@stats, check_script(1, sub { shift(@scripts); }, sub { print(@_); }));
 787         show_stats($start_time, \@stats) if $show_stats;
 788         exit(exit_code(\@stats));
 789 }
 790
 791 my $script_queue = Thread::Queue->new();
 792 my $output_queue = Thread::Queue->new();
 793
 794 sub next_script { return $script_queue->dequeue(); }
 795 sub emit { $output_queue->enqueue(@_); }
 796
 797 sub monitor {
 798         while (my $s = $output_queue->dequeue()) {
 799                 print($s);
 800         }
 801 }
 802
 803 my $mon = threads->create({'context' => 'void'}, \&monitor);
 804 threads->create({'context' => 'list'}, \&check_script, $_, \&next_script, \&emit) for 1..$jobs;
 805
 806 $script_queue->enqueue(@scripts);
 807 $script_queue->end();
 808
 809 for (threads->list()) {
 810         push(@stats, $_->join()) unless $_ == $mon;
 811 }
 812
 813 $output_queue->end();
 814 $mon->join();
 815
 816 show_stats($start_time, \@stats) if $show_stats;
 817 exit(exit_code(\@stats));