tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copywrite 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $#ARGV >= 0 || die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
  17
  18 $| = 1;
  19
  20 my %opt;
  21 my %repeat_tests;
  22 my %repeats;
  23 my %default;
  24
  25 #default opts
  26 $default{"NUM_TESTS"}           = 1;
  27 $default{"REBOOT_TYPE"}         = "grub";
  28 $default{"TEST_TYPE"}           = "test";
  29 $default{"BUILD_TYPE"}          = "randconfig";
  30 $default{"MAKE_CMD"}            = "make";
  31 $default{"TIMEOUT"}             = 120;
  32 $default{"TMP_DIR"}             = "/tmp/ktest";
  33 $default{"SLEEP_TIME"}          = 60;   # sleep time between tests
  34 $default{"BUILD_NOCLEAN"}       = 0;
  35 $default{"REBOOT_ON_ERROR"}     = 0;
  36 $default{"POWEROFF_ON_ERROR"}   = 0;
  37 $default{"REBOOT_ON_SUCCESS"}   = 1;
  38 $default{"POWEROFF_ON_SUCCESS"} = 0;
  39 $default{"BUILD_OPTIONS"}       = "";
  40 $default{"BISECT_SLEEP_TIME"}   = 60;   # sleep time between bisects
  41 $default{"CLEAR_LOG"}           = 0;
  42 $default{"SUCCESS_LINE"}        = "login:";
  43 $default{"BOOTED_TIMEOUT"}      = 1;
  44 $default{"DIE_ON_FAILURE"}      = 1;
  45 $default{"SSH_EXEC"}            = "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND";
  46 $default{"SCP_TO_TARGET"}       = "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE";
  47 $default{"REBOOT"}              = "ssh \$SSH_USER\@\$MACHINE reboot";
  48
  49 my $version;
  50 my $machine;
  51 my $ssh_user;
  52 my $tmpdir;
  53 my $builddir;
  54 my $outputdir;
  55 my $output_config;
  56 my $test_type;
  57 my $build_type;
  58 my $build_options;
  59 my $reboot_type;
  60 my $reboot_script;
  61 my $power_cycle;
  62 my $reboot;
  63 my $reboot_on_error;
  64 my $poweroff_on_error;
  65 my $die_on_failure;
  66 my $powercycle_after_reboot;
  67 my $poweroff_after_halt;
  68 my $ssh_exec;
  69 my $scp_to_target;
  70 my $power_off;
  71 my $grub_menu;
  72 my $grub_number;
  73 my $target;
  74 my $make;
  75 my $post_install;
  76 my $noclean;
  77 my $minconfig;
  78 my $addconfig;
  79 my $in_bisect = 0;
  80 my $bisect_bad = "";
  81 my $reverse_bisect;
  82 my $in_patchcheck = 0;
  83 my $run_test;
  84 my $redirect;
  85 my $buildlog;
  86 my $dmesg;
  87 my $monitor_fp;
  88 my $monitor_pid;
  89 my $monitor_cnt = 0;
  90 my $sleep_time;
  91 my $bisect_sleep_time;
  92 my $store_failures;
  93 my $timeout;
  94 my $booted_timeout;
  95 my $console;
  96 my $success_line;
  97 my $build_target;
  98 my $target_image;
  99 my $localversion;
 100 my $iteration = 0;
 101 my $successes = 0;
 102
 103 sub set_value {
 104     my ($lvalue, $rvalue) = @_;
 105
 106     if (defined($opt{$lvalue})) {
 107         die "Error: Option $lvalue defined more than once!\n";
 108     }
 109     $opt{$lvalue} = $rvalue;
 110     if ($rvalue =~ /^\s*$/) {
 111         delete $opt{$lvalue};
 112     } else {
 113         $opt{$lvalue} = $rvalue;
 114     }
 115 }
 116
 117 sub read_config {
 118     my ($config) = @_;
 119
 120     open(IN, $config) || die "can't read file $config";
 121
 122     my $name = $config;
 123     $name =~ s,.*/(.*),$1,;
 124
 125     my $test_num = 0;
 126     my $default = 1;
 127     my $repeat = 1;
 128     my $num_tests_set = 0;
 129     my $skip = 0;
 130     my $rest;
 131
 132     while (<IN>) {
 133
 134         # ignore blank lines and comments
 135         next if (/^\s*$/ || /\s*\#/);
 136
 137         if (/^\s*TEST_START(.*)/) {
 138
 139             $rest = $1;
 140
 141             if ($num_tests_set) {
 142                 die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 143             }
 144
 145             my $old_test_num = $test_num;
 146             my $old_repeat = $repeat;
 147
 148             $test_num += $repeat;
 149             $default = 0;
 150             $repeat = 1;
 151
 152             if ($rest =~ /\s+SKIP(.*)/) {
 153                 $rest = $1;
 154                 $skip = 1;
 155             } else {
 156                 $skip = 0;
 157             }
 158
 159             if ($rest =~ /\s+ITERATE\s+(\d+)(.*)$/) {
 160                 $repeat = $1;
 161                 $rest = $2;
 162                 $repeat_tests{"$test_num"} = $repeat;
 163             }
 164
 165             if ($rest =~ /\s+SKIP(.*)/) {
 166                 $rest = $1;
 167                 $skip = 1;
 168             }
 169
 170             if ($rest !~ /^\s*$/) {
 171                 die "$name: $.: Gargbage found after TEST_START\n$_";
 172             }
 173
 174             if ($skip) {
 175                 $test_num = $old_test_num;
 176                 $repeat = $old_repeat;
 177             }
 178
 179         } elsif (/^\s*DEFAULTS(.*)$/) {
 180             $default = 1;
 181
 182             $rest = $1;
 183
 184             if ($rest =~ /\s+SKIP(.*)/) {
 185                 $rest = $1;
 186                 $skip = 1;
 187             } else {
 188                 $skip = 0;
 189             }
 190
 191             if ($rest !~ /^\s*$/) {
 192                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 193             }
 194
 195         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 196
 197             next if ($skip);
 198
 199             my $lvalue = $1;
 200             my $rvalue = $2;
 201
 202             if (!$default &&
 203                 ($lvalue eq "NUM_TESTS" ||
 204                  $lvalue eq "LOG_FILE" ||
 205                  $lvalue eq "CLEAR_LOG")) {
 206                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 207             }
 208
 209             if ($lvalue eq "NUM_TESTS") {
 210                 if ($test_num) {
 211                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 212                 }
 213                 if (!$default) {
 214                     die "$name: $.: NUM_TESTS must be set in default section\n";
 215                 }
 216                 $num_tests_set = 1;
 217             }
 218
 219             if ($default || $lvalue =~ /\[\d+\]$/) {
 220                 set_value($lvalue, $rvalue);
 221             } else {
 222                 my $val = "$lvalue\[$test_num\]";
 223                 set_value($val, $rvalue);
 224
 225                 if ($repeat > 1) {
 226                     $repeats{$val} = $repeat;
 227                 }
 228             }
 229         } else {
 230             die "$name: $.: Garbage found in config\n$_";
 231         }
 232     }
 233
 234     close(IN);
 235
 236     if ($test_num) {
 237         $test_num += $repeat - 1;
 238         $opt{"NUM_TESTS"} = $test_num;
 239     }
 240
 241     # set any defaults
 242
 243     foreach my $default (keys %default) {
 244         if (!defined($opt{$default})) {
 245             $opt{$default} = $default{$default};
 246         }
 247     }
 248 }
 249
 250 sub _logit {
 251     if (defined($opt{"LOG_FILE"})) {
 252         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
 253         print OUT @_;
 254         close(OUT);
 255     }
 256 }
 257
 258 sub logit {
 259     if (defined($opt{"LOG_FILE"})) {
 260         _logit @_;
 261     } else {
 262         print @_;
 263     }
 264 }
 265
 266 sub doprint {
 267     print @_;
 268     _logit @_;
 269 }
 270
 271 sub run_command;
 272
 273 sub reboot {
 274     # try to reboot normally
 275     if (run_command $reboot) {
 276         if (defined($powercycle_after_reboot)) {
 277             sleep $powercycle_after_reboot;
 278             run_command "$power_cycle";
 279         }
 280     } else {
 281         # nope? power cycle it.
 282         run_command "$power_cycle";
 283     }
 284 }
 285
 286 sub do_not_reboot {
 287     my $i = $iteration;
 288
 289     return $test_type eq "build" ||
 290         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
 291         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
 292 }
 293
 294 sub dodie {
 295     doprint "CRITICAL FAILURE... ", @_, "\n";
 296
 297     my $i = $iteration;
 298
 299     if ($reboot_on_error && !do_not_reboot) {
 300
 301         doprint "REBOOTING\n";
 302         reboot;
 303
 304     } elsif ($poweroff_on_error && defined($power_off)) {
 305         doprint "POWERING OFF\n";
 306         `$power_off`;
 307     }
 308
 309     die @_, "\n";
 310 }
 311
 312 sub open_console {
 313     my ($fp) = @_;
 314
 315     my $flags;
 316
 317     my $pid = open($fp, "$console|") or
 318         dodie "Can't open console $console";
 319
 320     $flags = fcntl($fp, F_GETFL, 0) or
 321         dodie "Can't get flags for the socket: $!";
 322     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 323         dodie "Can't set flags for the socket: $!";
 324
 325     return $pid;
 326 }
 327
 328 sub close_console {
 329     my ($fp, $pid) = @_;
 330
 331     doprint "kill child process $pid\n";
 332     kill 2, $pid;
 333
 334     print "closing!\n";
 335     close($fp);
 336 }
 337
 338 sub start_monitor {
 339     if ($monitor_cnt++) {
 340         return;
 341     }
 342     $monitor_fp = \*MONFD;
 343     $monitor_pid = open_console $monitor_fp;
 344
 345     return;
 346
 347     open(MONFD, "Stop perl from warning about single use of MONFD");
 348 }
 349
 350 sub end_monitor {
 351     if (--$monitor_cnt) {
 352         return;
 353     }
 354     close_console($monitor_fp, $monitor_pid);
 355 }
 356
 357 sub wait_for_monitor {
 358     my ($time) = @_;
 359     my $line;
 360
 361     doprint "** Wait for monitor to settle down **\n";
 362
 363     # read the monitor and wait for the system to calm down
 364     do {
 365         $line = wait_for_input($monitor_fp, $time);
 366         print "$line" if (defined($line));
 367     } while (defined($line));
 368     print "** Monitor flushed **\n";
 369 }
 370
 371 sub fail {
 372
 373         if ($die_on_failure) {
 374                 dodie @_;
 375         }
 376
 377         doprint "FAILED\n";
 378
 379         my $i = $iteration;
 380
 381         # no need to reboot for just building.
 382         if (!do_not_reboot) {
 383             doprint "REBOOTING\n";
 384             reboot;
 385             start_monitor;
 386             wait_for_monitor $sleep_time;
 387             end_monitor;
 388         }
 389
 390         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 391         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 392         doprint "KTEST RESULT: TEST $i Failed: ", @_, "\n";
 393         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 394         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 395
 396         return 1 if (!defined($store_failures));
 397
 398         my @t = localtime;
 399         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
 400                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
 401
 402         my $dir = "$machine-$test_type-$build_type-fail-$date";
 403         my $faildir = "$store_failures/$dir";
 404
 405         if (!-d $faildir) {
 406             mkpath($faildir) or
 407                 die "can't create $faildir";
 408         }
 409         if (-f "$output_config") {
 410             cp "$output_config", "$faildir/config" or
 411                 die "failed to copy .config";
 412         }
 413         if (-f $buildlog) {
 414             cp $buildlog, "$faildir/buildlog" or
 415                 die "failed to move $buildlog";
 416         }
 417         if (-f $dmesg) {
 418             cp $dmesg, "$faildir/dmesg" or
 419                 die "failed to move $dmesg";
 420         }
 421
 422         doprint "*** Saved info to $faildir ***\n";
 423
 424         return 1;
 425 }
 426
 427 sub run_command {
 428     my ($command) = @_;
 429     my $dolog = 0;
 430     my $dord = 0;
 431     my $pid;
 432
 433     $command =~ s/\$SSH_USER/$ssh_user/g;
 434     $command =~ s/\$MACHINE/$machine/g;
 435
 436     doprint("$command ... ");
 437
 438     $pid = open(CMD, "$command 2>&1 |") or
 439         (fail "unable to exec $command" and return 0);
 440
 441     if (defined($opt{"LOG_FILE"})) {
 442         open(LOG, ">>$opt{LOG_FILE}") or
 443             dodie "failed to write to log";
 444         $dolog = 1;
 445     }
 446
 447     if (defined($redirect)) {
 448         open (RD, ">$redirect") or
 449             dodie "failed to write to redirect $redirect";
 450         $dord = 1;
 451     }
 452
 453     while (<CMD>) {
 454         print LOG if ($dolog);
 455         print RD  if ($dord);
 456     }
 457
 458     waitpid($pid, 0);
 459     my $failed = $?;
 460
 461     close(CMD);
 462     close(LOG) if ($dolog);
 463     close(RD)  if ($dord);
 464
 465     if ($failed) {
 466         doprint "FAILED!\n";
 467     } else {
 468         doprint "SUCCESS\n";
 469     }
 470
 471     return !$failed;
 472 }
 473
 474 sub run_ssh {
 475     my ($cmd) = @_;
 476     my $cp_exec = $ssh_exec;
 477
 478     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
 479     return run_command "$cp_exec";
 480 }
 481
 482 sub run_scp {
 483     my ($src, $dst) = @_;
 484     my $cp_scp = $scp_to_target;
 485
 486     $cp_scp =~ s/\$SRC_FILE/$src/g;
 487     $cp_scp =~ s/\$DST_FILE/$dst/g;
 488
 489     return run_command "$cp_scp";
 490 }
 491
 492 sub get_grub_index {
 493
 494     if ($reboot_type ne "grub") {
 495         return;
 496     }
 497     return if (defined($grub_number));
 498
 499     doprint "Find grub menu ... ";
 500     $grub_number = -1;
 501
 502     my $ssh_grub = $ssh_exec;
 503     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
 504
 505     open(IN, "$ssh_grub |")
 506         or die "unable to get menu.lst";
 507
 508     while (<IN>) {
 509         if (/^\s*title\s+$grub_menu\s*$/) {
 510             $grub_number++;
 511             last;
 512         } elsif (/^\s*title\s/) {
 513             $grub_number++;
 514         }
 515     }
 516     close(IN);
 517
 518     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
 519         if ($grub_number < 0);
 520     doprint "$grub_number\n";
 521 }
 522
 523 sub wait_for_input
 524 {
 525     my ($fp, $time) = @_;
 526     my $rin;
 527     my $ready;
 528     my $line;
 529     my $ch;
 530
 531     if (!defined($time)) {
 532         $time = $timeout;
 533     }
 534
 535     $rin = '';
 536     vec($rin, fileno($fp), 1) = 1;
 537     $ready = select($rin, undef, undef, $time);
 538
 539     $line = "";
 540
 541     # try to read one char at a time
 542     while (sysread $fp, $ch, 1) {
 543         $line .= $ch;
 544         last if ($ch eq "\n");
 545     }
 546
 547     if (!length($line)) {
 548         return undef;
 549     }
 550
 551     return $line;
 552 }
 553
 554 sub reboot_to {
 555     if ($reboot_type eq "grub") {
 556         run_command "$ssh_exec '(echo \"savedefault --default=$grub_number --once\" | grub --batch; reboot)'";
 557         return;
 558     }
 559
 560     run_command "$reboot_script";
 561 }
 562
 563 sub get_sha1 {
 564     my ($commit) = @_;
 565
 566     doprint "git rev-list --max-count=1 $commit ... ";
 567     my $sha1 = `git rev-list --max-count=1 $commit`;
 568     my $ret = $?;
 569
 570     logit $sha1;
 571
 572     if ($ret) {
 573         doprint "FAILED\n";
 574         dodie "Failed to get git $commit";
 575     }
 576
 577     print "SUCCESS\n";
 578
 579     chomp $sha1;
 580
 581     return $sha1;
 582 }
 583
 584 sub monitor {
 585     my $booted = 0;
 586     my $bug = 0;
 587     my $skip_call_trace = 0;
 588     my $loops;
 589
 590     wait_for_monitor 5;
 591
 592     my $line;
 593     my $full_line = "";
 594
 595     open(DMESG, "> $dmesg") or
 596         die "unable to write to $dmesg";
 597
 598     reboot_to;
 599
 600     for (;;) {
 601
 602         if ($booted) {
 603             $line = wait_for_input($monitor_fp, $booted_timeout);
 604         } else {
 605             $line = wait_for_input($monitor_fp);
 606         }
 607
 608         last if (!defined($line));
 609
 610         doprint $line;
 611         print DMESG $line;
 612
 613         # we are not guaranteed to get a full line
 614         $full_line .= $line;
 615
 616         if ($full_line =~ /$success_line/) {
 617             $booted = 1;
 618         }
 619
 620         if ($full_line =~ /\[ backtrace testing \]/) {
 621             $skip_call_trace = 1;
 622         }
 623
 624         if ($full_line =~ /call trace:/i) {
 625             $bug = 1 if (!$skip_call_trace);
 626         }
 627
 628         if ($full_line =~ /\[ end of backtrace testing \]/) {
 629             $skip_call_trace = 0;
 630         }
 631
 632         if ($full_line =~ /Kernel panic -/) {
 633             $bug = 1;
 634         }
 635
 636         if ($line =~ /\n/) {
 637             $full_line = "";
 638         }
 639     }
 640
 641     close(DMESG);
 642
 643     if ($bug) {
 644         return 0 if ($in_bisect);
 645         fail "failed - got a bug report" and return 0;
 646     }
 647
 648     if (!$booted) {
 649         return 0 if ($in_bisect);
 650         fail "failed - never got a boot prompt." and return 0;
 651     }
 652
 653     return 1;
 654 }
 655
 656 sub install {
 657
 658     run_scp "$outputdir/$build_target", "$target_image" or
 659         dodie "failed to copy image";
 660
 661     my $install_mods = 0;
 662
 663     # should we process modules?
 664     $install_mods = 0;
 665     open(IN, "$output_config") or dodie("Can't read config file");
 666     while (<IN>) {
 667         if (/CONFIG_MODULES(=y)?/) {
 668             $install_mods = 1 if (defined($1));
 669             last;
 670         }
 671     }
 672     close(IN);
 673
 674     if (!$install_mods) {
 675         doprint "No modules needed\n";
 676         return;
 677     }
 678
 679     run_command "$make INSTALL_MOD_PATH=$tmpdir modules_install" or
 680         dodie "Failed to install modules";
 681
 682     my $modlib = "/lib/modules/$version";
 683     my $modtar = "ktest-mods.tar.bz2";
 684
 685     run_ssh "rm -rf $modlib" or
 686         dodie "failed to remove old mods: $modlib";
 687
 688     # would be nice if scp -r did not follow symbolic links
 689     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
 690         dodie "making tarball";
 691
 692     run_scp "$tmpdir/$modtar", "/tmp" or
 693         dodie "failed to copy modules";
 694
 695     unlink "$tmpdir/$modtar";
 696
 697     run_ssh "'(cd / && tar xf /tmp/$modtar)'" or
 698         dodie "failed to tar modules";
 699
 700     run_ssh "rm -f /tmp/$modtar";
 701
 702     return if (!defined($post_install));
 703
 704     my $cp_post_install = $post_install;
 705     $cp_post_install = s/\$KERNEL_VERSION/$version/g;
 706     run_command "$cp_post_install" or
 707         dodie "Failed to run post install";
 708 }
 709
 710 sub check_buildlog {
 711     my ($patch) = @_;
 712
 713     my @files = `git show $patch | diffstat -l`;
 714
 715     open(IN, "git show $patch |") or
 716         dodie "failed to show $patch";
 717     while (<IN>) {
 718         if (m,^--- a/(.*),) {
 719             chomp $1;
 720             $files[$#files] = $1;
 721         }
 722     }
 723     close(IN);
 724
 725     open(IN, $buildlog) or dodie "Can't open $buildlog";
 726     while (<IN>) {
 727         if (/^\s*(.*?):.*(warning|error)/) {
 728             my $err = $1;
 729             foreach my $file (@files) {
 730                 my $fullpath = "$builddir/$file";
 731                 if ($file eq $err || $fullpath eq $err) {
 732                     fail "$file built with warnings" and return 0;
 733                 }
 734             }
 735         }
 736     }
 737     close(IN);
 738
 739     return 1;
 740 }
 741
 742 sub build {
 743     my ($type) = @_;
 744     my $defconfig = "";
 745
 746     unlink $buildlog;
 747
 748     if ($type =~ /^useconfig:(.*)/) {
 749         run_command "cp $1 $output_config" or
 750             dodie "could not copy $1 to .config";
 751
 752         $type = "oldconfig";
 753     }
 754
 755     # old config can ask questions
 756     if ($type eq "oldconfig") {
 757         $type = "oldnoconfig";
 758
 759         # allow for empty configs
 760         run_command "touch $output_config";
 761
 762         run_command "mv $output_config $outputdir/config_temp" or
 763             dodie "moving .config";
 764
 765         if (!$noclean && !run_command "$make mrproper") {
 766             dodie "make mrproper";
 767         }
 768
 769         run_command "mv $outputdir/config_temp $output_config" or
 770             dodie "moving config_temp";
 771
 772     } elsif (!$noclean) {
 773         unlink "$output_config";
 774         run_command "$make mrproper" or
 775             dodie "make mrproper";
 776     }
 777
 778     # add something to distinguish this build
 779     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
 780     print OUT "$localversion\n";
 781     close(OUT);
 782
 783     if (defined($minconfig)) {
 784         $defconfig = "KCONFIG_ALLCONFIG=$minconfig";
 785     }
 786
 787     run_command "$defconfig $make $type" or
 788         dodie "failed make config";
 789
 790     $redirect = "$buildlog";
 791     if (!run_command "$make $build_options") {
 792         undef $redirect;
 793         # bisect may need this to pass
 794         return 0 if ($in_bisect);
 795         fail "failed build" and return 0;
 796     }
 797     undef $redirect;
 798
 799     return 1;
 800 }
 801
 802 sub halt {
 803     if (!run_ssh "halt" or defined($power_off)) {
 804         if (defined($poweroff_after_halt)) {
 805             sleep $poweroff_after_halt;
 806             run_command "$power_off";
 807         }
 808     } else {
 809         # nope? the zap it!
 810         run_command "$power_off";
 811     }
 812 }
 813
 814 sub success {
 815     my ($i) = @_;
 816
 817     $successes++;
 818
 819     doprint "\n\n*******************************************\n";
 820     doprint     "*******************************************\n";
 821     doprint     "KTEST RESULT: TEST $i SUCCESS!!!!         **\n";
 822     doprint     "*******************************************\n";
 823     doprint     "*******************************************\n";
 824
 825     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
 826         doprint "Reboot and wait $sleep_time seconds\n";
 827         reboot;
 828         start_monitor;
 829         wait_for_monitor $sleep_time;
 830         end_monitor;
 831     }
 832 }
 833
 834 sub get_version {
 835     # get the release name
 836     doprint "$make kernelrelease ... ";
 837     $version = `$make kernelrelease | tail -1`;
 838     chomp($version);
 839     doprint "$version\n";
 840 }
 841
 842 sub child_run_test {
 843     my $failed = 0;
 844
 845     # child should have no power
 846     $reboot_on_error = 0;
 847     $poweroff_on_error = 0;
 848     $die_on_failure = 1;
 849
 850     run_command $run_test or $failed = 1;
 851     exit $failed;
 852 }
 853
 854 my $child_done;
 855
 856 sub child_finished {
 857     $child_done = 1;
 858 }
 859
 860 sub do_run_test {
 861     my $child_pid;
 862     my $child_exit;
 863     my $line;
 864     my $full_line;
 865     my $bug = 0;
 866
 867     wait_for_monitor 1;
 868
 869     doprint "run test $run_test\n";
 870
 871     $child_done = 0;
 872
 873     $SIG{CHLD} = qw(child_finished);
 874
 875     $child_pid = fork;
 876
 877     child_run_test if (!$child_pid);
 878
 879     $full_line = "";
 880
 881     do {
 882         $line = wait_for_input($monitor_fp, 1);
 883         if (defined($line)) {
 884
 885             # we are not guaranteed to get a full line
 886             $full_line .= $line;
 887
 888             if ($full_line =~ /call trace:/i) {
 889                 $bug = 1;
 890             }
 891
 892             if ($full_line =~ /Kernel panic -/) {
 893                 $bug = 1;
 894             }
 895
 896             if ($line =~ /\n/) {
 897                 $full_line = "";
 898             }
 899         }
 900     } while (!$child_done && !$bug);
 901
 902     if ($bug) {
 903         doprint "Detected kernel crash!\n";
 904         # kill the child with extreme prejudice
 905         kill 9, $child_pid;
 906     }
 907
 908     waitpid $child_pid, 0;
 909     $child_exit = $?;
 910
 911     if ($bug || $child_exit) {
 912         return 0 if $in_bisect;
 913         fail "test failed" and return 0;
 914     }
 915     return 1;
 916 }
 917
 918 sub run_git_bisect {
 919     my ($command) = @_;
 920
 921     doprint "$command ... ";
 922
 923     my $output = `$command 2>&1`;
 924     my $ret = $?;
 925
 926     logit $output;
 927
 928     if ($ret) {
 929         doprint "FAILED\n";
 930         dodie "Failed to git bisect";
 931     }
 932
 933     doprint "SUCCESS\n";
 934     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
 935         doprint "$1 [$2]\n";
 936     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
 937         $bisect_bad = $1;
 938         doprint "Found bad commit... $1\n";
 939         return 0;
 940     } else {
 941         # we already logged it, just print it now.
 942         print $output;
 943     }
 944
 945     return 1;
 946 }
 947
 948 sub run_bisect {
 949     my ($type) = @_;
 950
 951     my $failed = 0;
 952     my $result;
 953     my $output;
 954     my $ret;
 955
 956     if (defined($minconfig)) {
 957         build "useconfig:$minconfig" or $failed = 1;
 958     } else {
 959         # ?? no config to use?
 960         build "oldconfig" or $failed = 1;
 961     }
 962
 963     if ($type ne "build") {
 964         dodie "Failed on build" if $failed;
 965
 966         # Now boot the box
 967         get_grub_index;
 968         get_version;
 969         install;
 970
 971         start_monitor;
 972         monitor or $failed = 1;
 973
 974         if ($type ne "boot") {
 975             dodie "Failed on boot" if $failed;
 976
 977             do_run_test or $failed = 1;
 978         }
 979         end_monitor;
 980     }
 981
 982     if ($failed) {
 983         $result = "bad";
 984
 985         # reboot the box to a good kernel
 986         if ($type ne "build") {
 987             doprint "Reboot and sleep $bisect_sleep_time seconds\n";
 988             reboot;
 989             start_monitor;
 990             wait_for_monitor $bisect_sleep_time;
 991             end_monitor;
 992         }
 993     } else {
 994         $result = "good";
 995     }
 996
 997     # Are we looking for where it worked, not failed?
 998     if ($reverse_bisect) {
 999         if ($failed) {
1000             $result = "good";
1001         } else {
1002             $result = "bad";
1003         }
1004     }
1005
1006     return $result;
1007 }
1008
1009 sub bisect {
1010     my ($i) = @_;
1011
1012     my $result;
1013
1014     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
1015     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
1016     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
1017
1018     my $good = $opt{"BISECT_GOOD[$i]"};
1019     my $bad = $opt{"BISECT_BAD[$i]"};
1020     my $type = $opt{"BISECT_TYPE[$i]"};
1021     my $start = $opt{"BISECT_START[$i]"};
1022     my $replay = $opt{"BISECT_REPLAY[$i]"};
1023
1024     # convert to true sha1's
1025     $good = get_sha1($good);
1026     $bad = get_sha1($bad);
1027
1028     if (defined($opt{"BISECT_REVERSE[$i]"}) &&
1029         $opt{"BISECT_REVERSE[$i]"} == 1) {
1030         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
1031         $reverse_bisect = 1;
1032     } else {
1033         $reverse_bisect = 0;
1034     }
1035
1036     $in_bisect = 1;
1037
1038     # Can't have a test without having a test to run
1039     if ($type eq "test" && !defined($run_test)) {
1040         $type = "boot";
1041     }
1042
1043     my $check = $opt{"BISECT_CHECK[$i]"};
1044     if (defined($check) && $check ne "0") {
1045
1046         # get current HEAD
1047         my $head = get_sha1("HEAD");
1048
1049         if ($check ne "good") {
1050             doprint "TESTING BISECT BAD [$bad]\n";
1051             run_command "git checkout $bad" or
1052                 die "Failed to checkout $bad";
1053
1054             $result = run_bisect $type;
1055
1056             if ($result ne "bad") {
1057                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
1058             }
1059         }
1060
1061         if ($check ne "bad") {
1062             doprint "TESTING BISECT GOOD [$good]\n";
1063             run_command "git checkout $good" or
1064                 die "Failed to checkout $good";
1065
1066             $result = run_bisect $type;
1067
1068             if ($result ne "good") {
1069                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
1070             }
1071         }
1072
1073         # checkout where we started
1074         run_command "git checkout $head" or
1075             die "Failed to checkout $head";
1076     }
1077
1078     run_command "git bisect start" or
1079         dodie "could not start bisect";
1080
1081     run_command "git bisect good $good" or
1082         dodie "could not set bisect good to $good";
1083
1084     run_git_bisect "git bisect bad $bad" or
1085         dodie "could not set bisect bad to $bad";
1086
1087     if (defined($replay)) {
1088         run_command "git bisect replay $replay" or
1089             dodie "failed to run replay";
1090     }
1091
1092     if (defined($start)) {
1093         run_command "git checkout $start" or
1094             dodie "failed to checkout $start";
1095     }
1096
1097     my $test;
1098     do {
1099         $result = run_bisect $type;
1100         $test = run_git_bisect "git bisect $result";
1101     } while ($test);
1102
1103     run_command "git bisect log" or
1104         dodie "could not capture git bisect log";
1105
1106     run_command "git bisect reset" or
1107         dodie "could not reset git bisect";
1108
1109     doprint "Bad commit was [$bisect_bad]\n";
1110
1111     $in_bisect = 0;
1112
1113     success $i;
1114 }
1115
1116 sub patchcheck {
1117     my ($i) = @_;
1118
1119     die "PATCHCHECK_START[$i] not defined\n"
1120         if (!defined($opt{"PATCHCHECK_START[$i]"}));
1121     die "PATCHCHECK_TYPE[$i] not defined\n"
1122         if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
1123
1124     my $start = $opt{"PATCHCHECK_START[$i]"};
1125
1126     my $end = "HEAD";
1127     if (defined($opt{"PATCHCHECK_END[$i]"})) {
1128         $end = $opt{"PATCHCHECK_END[$i]"};
1129     }
1130
1131     # Get the true sha1's since we can use things like HEAD~3
1132     $start = get_sha1($start);
1133     $end = get_sha1($end);
1134
1135     my $type = $opt{"PATCHCHECK_TYPE[$i]"};
1136
1137     # Can't have a test without having a test to run
1138     if ($type eq "test" && !defined($run_test)) {
1139         $type = "boot";
1140     }
1141
1142     open (IN, "git log --pretty=oneline $end|") or
1143         dodie "could not get git list";
1144
1145     my @list;
1146
1147     while (<IN>) {
1148         chomp;
1149         $list[$#list+1] = $_;
1150         last if (/^$start/);
1151     }
1152     close(IN);
1153
1154     if ($list[$#list] !~ /^$start/) {
1155         fail "SHA1 $start not found";
1156     }
1157
1158     # go backwards in the list
1159     @list = reverse @list;
1160
1161     my $save_clean = $noclean;
1162
1163     $in_patchcheck = 1;
1164     foreach my $item (@list) {
1165         my $sha1 = $item;
1166         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
1167
1168         doprint "\nProcessing commit $item\n\n";
1169
1170         run_command "git checkout $sha1" or
1171             die "Failed to checkout $sha1";
1172
1173         # only clean on the first and last patch
1174         if ($item eq $list[0] ||
1175             $item eq $list[$#list]) {
1176             $noclean = $save_clean;
1177         } else {
1178             $noclean = 1;
1179         }
1180
1181         if (defined($minconfig)) {
1182             build "useconfig:$minconfig" or return 0;
1183         } else {
1184             # ?? no config to use?
1185             build "oldconfig" or return 0;
1186         }
1187
1188         check_buildlog $sha1 or return 0;
1189
1190         next if ($type eq "build");
1191
1192         get_grub_index;
1193         get_version;
1194         install;
1195
1196         my $failed = 0;
1197
1198         start_monitor;
1199         monitor or $failed = 1;
1200
1201         if (!$failed && $type ne "boot"){
1202             do_run_test or $failed = 1;
1203         }
1204         end_monitor;
1205         return 0 if ($failed);
1206
1207     }
1208     $in_patchcheck = 0;
1209     success $i;
1210
1211     return 1;
1212 }
1213
1214 read_config $ARGV[0];
1215
1216 # mandatory configs
1217 die "MACHINE not defined\n"             if (!defined($opt{"MACHINE"}));
1218 die "SSH_USER not defined\n"            if (!defined($opt{"SSH_USER"}));
1219 die "BUILD_DIR not defined\n"           if (!defined($opt{"BUILD_DIR"}));
1220 die "OUTPUT_DIR not defined\n"          if (!defined($opt{"OUTPUT_DIR"}));
1221 die "BUILD_TARGET not defined\n"        if (!defined($opt{"BUILD_TARGET"}));
1222 die "TARGET_IMAGE not defined\n"        if (!defined($opt{"TARGET_IMAGE"}));
1223 die "POWER_CYCLE not defined\n"         if (!defined($opt{"POWER_CYCLE"}));
1224 die "CONSOLE not defined\n"             if (!defined($opt{"CONSOLE"}));
1225 die "LOCALVERSION not defined\n"        if (!defined($opt{"LOCALVERSION"}));
1226
1227 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
1228     unlink $opt{"LOG_FILE"};
1229 }
1230
1231 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
1232
1233 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
1234
1235     if (!$i) {
1236         doprint "DEFAULT OPTIONS:\n";
1237     } else {
1238         doprint "\nTEST $i OPTIONS";
1239         if (defined($repeat_tests{$i})) {
1240             $repeat = $repeat_tests{$i};
1241             doprint " ITERATE $repeat";
1242         }
1243         doprint "\n";
1244     }
1245
1246     foreach my $option (sort keys %opt) {
1247
1248         if ($option =~ /\[(\d+)\]$/) {
1249             next if ($i != $1);
1250         } else {
1251             next if ($i);
1252         }
1253
1254         doprint "$option = $opt{$option}\n";
1255     }
1256 }
1257
1258 sub set_test_option {
1259     my ($name, $i) = @_;
1260
1261     my $option = "$name\[$i\]";
1262
1263     if (defined($opt{$option})) {
1264         return $opt{$option};
1265     }
1266
1267     foreach my $test (keys %repeat_tests) {
1268         if ($i >= $test &&
1269             $i < $test + $repeat_tests{$test}) {
1270             $option = "$name\[$test\]";
1271             if (defined($opt{$option})) {
1272                 return $opt{$option};
1273             }
1274         }
1275     }
1276
1277     if (defined($opt{$name})) {
1278         return $opt{$name};
1279     }
1280
1281     return undef;
1282 }
1283
1284 # First we need to do is the builds
1285 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
1286
1287     $iteration = $i;
1288
1289     my $makecmd = set_test_option("MAKE_CMD", $i);
1290
1291     $machine = set_test_option("MACHINE", $i);
1292     $ssh_user = set_test_option("SSH_USER", $i);
1293     $tmpdir = set_test_option("TMP_DIR", $i);
1294     $outputdir = set_test_option("OUTPUT_DIR", $i);
1295     $builddir = set_test_option("BUILD_DIR", $i);
1296     $test_type = set_test_option("TEST_TYPE", $i);
1297     $build_type = set_test_option("BUILD_TYPE", $i);
1298     $build_options = set_test_option("BUILD_OPTIONS", $i);
1299     $power_cycle = set_test_option("POWER_CYCLE", $i);
1300     $reboot = set_test_option("REBOOT", $i);
1301     $noclean = set_test_option("BUILD_NOCLEAN", $i);
1302     $minconfig = set_test_option("MIN_CONFIG", $i);
1303     $run_test = set_test_option("TEST", $i);
1304     $addconfig = set_test_option("ADD_CONFIG", $i);
1305     $reboot_type = set_test_option("REBOOT_TYPE", $i);
1306     $grub_menu = set_test_option("GRUB_MENU", $i);
1307     $post_install = set_test_option("POST_INSTALL", $i);
1308     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
1309     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
1310     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
1311     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
1312     $power_off = set_test_option("POWER_OFF", $i);
1313     $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
1314     $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
1315     $sleep_time = set_test_option("SLEEP_TIME", $i);
1316     $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
1317     $store_failures = set_test_option("STORE_FAILURES", $i);
1318     $timeout = set_test_option("TIMEOUT", $i);
1319     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
1320     $console = set_test_option("CONSOLE", $i);
1321     $success_line = set_test_option("SUCCESS_LINE", $i);
1322     $build_target = set_test_option("BUILD_TARGET", $i);
1323     $ssh_exec = set_test_option("SSH_EXEC", $i);
1324     $scp_to_target = set_test_option("SCP_TO_TARGET", $i);
1325     $target_image = set_test_option("TARGET_IMAGE", $i);
1326     $localversion = set_test_option("LOCALVERSION", $i);
1327
1328     chdir $builddir || die "can't change directory to $builddir";
1329
1330     if (!-d $tmpdir) {
1331         mkpath($tmpdir) or
1332             die "can't create $tmpdir";
1333     }
1334
1335     $ENV{"SSH_USER"} = $ssh_user;
1336     $ENV{"MACHINE"} = $machine;
1337
1338     $target = "$ssh_user\@$machine";
1339
1340     $buildlog = "$tmpdir/buildlog-$machine";
1341     $dmesg = "$tmpdir/dmesg-$machine";
1342     $make = "$makecmd O=$outputdir";
1343     $output_config = "$outputdir/.config";
1344     $output_config = "$outputdir/.config";
1345
1346     if ($reboot_type eq "grub") {
1347         dodie "GRUB_MENU not defined" if (!defined($grub_menu));
1348     } elsif (!defined($reboot_script)) {
1349         dodie "REBOOT_SCRIPT not defined"
1350     }
1351
1352     my $run_type = $build_type;
1353     if ($test_type eq "patchcheck") {
1354         $run_type = $opt{"PATCHCHECK_TYPE[$i]"};
1355     } elsif ($test_type eq "bisect") {
1356         $run_type = $opt{"BISECT_TYPE[$i]"};
1357     }
1358
1359     # mistake in config file?
1360     if (!defined($run_type)) {
1361         $run_type = "ERROR";
1362     }
1363
1364     doprint "\n\n";
1365     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type\n\n";
1366
1367     unlink $dmesg;
1368     unlink $buildlog;
1369
1370     if (!defined($minconfig)) {
1371         $minconfig = $addconfig;
1372
1373     } elsif (defined($addconfig)) {
1374         run_command "cat $addconfig $minconfig > $tmpdir/use_config" or
1375             dodie "Failed to create temp config";
1376         $minconfig = "$tmpdir/use_config";
1377     }
1378
1379     my $checkout = $opt{"CHECKOUT[$i]"};
1380     if (defined($checkout)) {
1381         run_command "git checkout $checkout" or
1382             die "failed to checkout $checkout";
1383     }
1384
1385     if ($test_type eq "bisect") {
1386         bisect $i;
1387         next;
1388     } elsif ($test_type eq "patchcheck") {
1389         patchcheck $i;
1390         next;
1391     }
1392
1393     if ($build_type ne "nobuild") {
1394         build $build_type or next;
1395     }
1396
1397     if ($test_type ne "build") {
1398         get_grub_index;
1399         get_version;
1400         install;
1401
1402         my $failed = 0;
1403         start_monitor;
1404         monitor or $failed = 1;;
1405
1406         if (!$failed && $test_type ne "boot" && defined($run_test)) {
1407             do_run_test or $failed = 1;
1408         }
1409         end_monitor;
1410         next if ($failed);
1411     }
1412
1413     success $i;
1414 }
1415
1416 if ($opt{"POWEROFF_ON_SUCCESS"}) {
1417     halt;
1418 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
1419     reboot;
1420 }
1421
1422 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
1423
1424 exit 0;