tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copywrite 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 $#ARGV >= 0 || die "usage: ktest.pl config-file\n";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21 my %default;
  22
  23 #default opts
  24 $default{"NUM_TESTS"}           = 1;
  25 $default{"REBOOT_TYPE"}         = "grub";
  26 $default{"TEST_TYPE"}           = "test";
  27 $default{"BUILD_TYPE"}          = "randconfig";
  28 $default{"MAKE_CMD"}            = "make";
  29 $default{"TIMEOUT"}             = 120;
  30 $default{"TMP_DIR"}             = "/tmp/ktest";
  31 $default{"SLEEP_TIME"}          = 60;   # sleep time between tests
  32 $default{"BUILD_NOCLEAN"}       = 0;
  33 $default{"REBOOT_ON_ERROR"}     = 0;
  34 $default{"POWEROFF_ON_ERROR"}   = 0;
  35 $default{"REBOOT_ON_SUCCESS"}   = 1;
  36 $default{"POWEROFF_ON_SUCCESS"} = 0;
  37 $default{"BUILD_OPTIONS"}       = "";
  38 $default{"BISECT_SLEEP_TIME"}   = 60;   # sleep time between bisects
  39 $default{"CLEAR_LOG"}           = 0;
  40 $default{"SUCCESS_LINE"}        = "login:";
  41 $default{"BOOTED_TIMEOUT"}      = 1;
  42 $default{"DIE_ON_FAILURE"}      = 1;
  43
  44 my $version;
  45 my $machine;
  46 my $tmpdir;
  47 my $builddir;
  48 my $outputdir;
  49 my $test_type;
  50 my $build_type;
  51 my $build_options;
  52 my $reboot_type;
  53 my $reboot_script;
  54 my $power_cycle;
  55 my $reboot_on_error;
  56 my $poweroff_on_error;
  57 my $die_on_failure;
  58 my $powercycle_after_reboot;
  59 my $poweroff_after_halt;
  60 my $power_off;
  61 my $grub_menu;
  62 my $grub_number;
  63 my $target;
  64 my $make;
  65 my $post_install;
  66 my $noclean;
  67 my $minconfig;
  68 my $addconfig;
  69 my $in_bisect = 0;
  70 my $bisect_bad = "";
  71 my $reverse_bisect;
  72 my $in_patchcheck = 0;
  73 my $run_test;
  74 my $redirect;
  75 my $buildlog;
  76 my $dmesg;
  77 my $monitor_fp;
  78 my $monitor_pid;
  79 my $monitor_cnt = 0;
  80 my $sleep_time;
  81 my $bisect_sleep_time;
  82 my $store_failures;
  83 my $timeout;
  84 my $booted_timeout;
  85 my $console;
  86 my $success_line;
  87 my $build_target;
  88 my $target_image;
  89 my $localversion;
  90 my $iteration = 0;
  91
  92 sub set_value {
  93     my ($lvalue, $rvalue) = @_;
  94
  95     if (defined($opt{$lvalue})) {
  96         die "Error: Option $lvalue defined more than once!\n";
  97     }
  98     $opt{$lvalue} = $rvalue;
  99 }
 100
 101 sub read_config {
 102     my ($config) = @_;
 103
 104     open(IN, $config) || die "can't read file $config";
 105
 106     my $name = $config;
 107     $name =~ s,.*/(.*),$1,;
 108
 109     my $test_num = 0;
 110     my $default = 1;
 111     my $repeat = 1;
 112     my $num_tests_set = 0;
 113     my $skip = 0;
 114     my $rest;
 115
 116     while (<IN>) {
 117
 118         # ignore blank lines and comments
 119         next if (/^\s*$/ || /\s*\#/);
 120
 121         if (/^\s*TEST_START(.*)/) {
 122
 123             $rest = $1;
 124
 125             if ($num_tests_set) {
 126                 die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 127             }
 128
 129             my $old_test_num = $test_num;
 130
 131             $test_num += $repeat;
 132             $default = 0;
 133             $repeat = 1;
 134
 135             if ($rest =~ /\s+SKIP(.*)/) {
 136                 $rest = $1;
 137                 $skip = 1;
 138             } else {
 139                 $skip = 0;
 140             }
 141
 142             if ($rest =~ /\s+ITERATE\s+(\d+)(.*)$/) {
 143                 $repeat = $1;
 144                 $rest = $2;
 145                 $repeat_tests{"$test_num"} = $repeat;
 146             }
 147
 148             if ($rest =~ /\s+SKIP(.*)/) {
 149                 $rest = $1;
 150                 $skip = 1;
 151             }
 152
 153             if ($rest !~ /^\s*$/) {
 154                 die "$name: $.: Gargbage found after TEST_START\n$_";
 155             }
 156
 157             if ($skip) {
 158                 $test_num = $old_test_num;
 159                 $repeat = 1;
 160             }
 161
 162         } elsif (/^\s*DEFAULTS(.*)$/) {
 163             $default = 1;
 164
 165             $rest = $1;
 166
 167             if ($rest =~ /\s+SKIP(.*)/) {
 168                 $rest = $1;
 169                 $skip = 1;
 170             } else {
 171                 $skip = 0;
 172             }
 173
 174             if ($rest !~ /^\s*$/) {
 175                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 176             }
 177
 178         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 179
 180             next if ($skip);
 181
 182             my $lvalue = $1;
 183             my $rvalue = $2;
 184
 185             if (!$default &&
 186                 ($lvalue eq "NUM_TESTS" ||
 187                  $lvalue eq "LOG_FILE" ||
 188                  $lvalue eq "CLEAR_LOG")) {
 189                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 190             }
 191
 192             if ($lvalue eq "NUM_TESTS") {
 193                 if ($test_num) {
 194                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 195                 }
 196                 if (!$default) {
 197                     die "$name: $.: NUM_TESTS must be set in default section\n";
 198                 }
 199                 $num_tests_set = 1;
 200             }
 201
 202             if ($default || $lvalue =~ /\[\d+\]$/) {
 203                 set_value($lvalue, $rvalue);
 204             } else {
 205                 my $val = "$lvalue\[$test_num\]";
 206                 set_value($val, $rvalue);
 207
 208                 if ($repeat > 1) {
 209                     $repeats{$val} = $repeat;
 210                 }
 211             }
 212         } else {
 213             die "$name: $.: Garbage found in config\n$_";
 214         }
 215     }
 216
 217     close(IN);
 218
 219     if ($test_num) {
 220         $test_num += $repeat - 1;
 221         $opt{"NUM_TESTS"} = $test_num;
 222     }
 223
 224     # set any defaults
 225
 226     foreach my $default (keys %default) {
 227         if (!defined($opt{$default})) {
 228             $opt{$default} = $default{$default};
 229         }
 230     }
 231 }
 232
 233 sub _logit {
 234     if (defined($opt{"LOG_FILE"})) {
 235         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
 236         print OUT @_;
 237         close(OUT);
 238     }
 239 }
 240
 241 sub logit {
 242     if (defined($opt{"LOG_FILE"})) {
 243         _logit @_;
 244     } else {
 245         print @_;
 246     }
 247 }
 248
 249 sub doprint {
 250     print @_;
 251     _logit @_;
 252 }
 253
 254 sub run_command;
 255
 256 sub reboot {
 257     # try to reboot normally
 258     if (run_command "ssh $target reboot") {
 259         if (defined($powercycle_after_reboot)) {
 260             sleep $powercycle_after_reboot;
 261             run_command "$power_cycle";
 262         }
 263     } else {
 264         # nope? power cycle it.
 265         run_command "$power_cycle";
 266     }
 267 }
 268
 269 sub do_not_reboot {
 270     my $i = $iteration;
 271
 272     return $test_type eq "build" ||
 273         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
 274         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
 275 }
 276
 277 sub dodie {
 278     doprint "CRITICAL FAILURE... ", @_, "\n";
 279
 280     my $i = $iteration;
 281
 282     if ($reboot_on_error && !do_not_reboot) {
 283
 284         doprint "REBOOTING\n";
 285         reboot;
 286
 287     } elsif ($poweroff_on_error && defined($power_off)) {
 288         doprint "POWERING OFF\n";
 289         `$power_off`;
 290     }
 291
 292     die @_, "\n";
 293 }
 294
 295 sub open_console {
 296     my ($fp) = @_;
 297
 298     my $flags;
 299
 300     my $pid = open($fp, "$console|") or
 301         dodie "Can't open console $console";
 302
 303     $flags = fcntl($fp, F_GETFL, 0) or
 304         dodie "Can't get flags for the socket: $!";
 305     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 306         dodie "Can't set flags for the socket: $!";
 307
 308     return $pid;
 309 }
 310
 311 sub close_console {
 312     my ($fp, $pid) = @_;
 313
 314     doprint "kill child process $pid\n";
 315     kill 2, $pid;
 316
 317     print "closing!\n";
 318     close($fp);
 319 }
 320
 321 sub start_monitor {
 322     if ($monitor_cnt++) {
 323         return;
 324     }
 325     $monitor_fp = \*MONFD;
 326     $monitor_pid = open_console $monitor_fp;
 327
 328     return;
 329
 330     open(MONFD, "Stop perl from warning about single use of MONFD");
 331 }
 332
 333 sub end_monitor {
 334     if (--$monitor_cnt) {
 335         return;
 336     }
 337     close_console($monitor_fp, $monitor_pid);
 338 }
 339
 340 sub wait_for_monitor {
 341     my ($time) = @_;
 342     my $line;
 343
 344     doprint "** Wait for monitor to settle down **\n";
 345
 346     # read the monitor and wait for the system to calm down
 347     do {
 348         $line = wait_for_input($monitor_fp, $time);
 349         print "$line" if (defined($line));
 350     } while (defined($line));
 351     print "** Monitor flushed **\n";
 352 }
 353
 354 sub fail {
 355
 356         if ($die_on_failure) {
 357                 dodie @_;
 358         }
 359
 360         doprint "FAILED\n";
 361
 362         my $i = $iteration;
 363
 364         # no need to reboot for just building.
 365         if (!do_not_reboot) {
 366             doprint "REBOOTING\n";
 367             reboot;
 368             start_monitor;
 369             wait_for_monitor $sleep_time;
 370             end_monitor;
 371         }
 372
 373         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 374         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 375         doprint "**** Failed: ", @_, " ****\n";
 376         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 377         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 378
 379         return 1 if (!defined($store_failures));
 380
 381         my @t = localtime;
 382         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
 383                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
 384
 385         my $dir = "$machine-$test_type-$build_type-fail-$date";
 386         my $faildir = "$store_failures/$dir";
 387
 388         if (!-d $faildir) {
 389             mkpath($faildir) or
 390                 die "can't create $faildir";
 391         }
 392         if (-f "$outputdir/.config") {
 393             cp "$outputdir/.config", "$faildir/config" or
 394                 die "failed to copy .config";
 395         }
 396         if (-f $buildlog) {
 397             cp $buildlog, "$faildir/buildlog" or
 398                 die "failed to move $buildlog";
 399         }
 400         if (-f $dmesg) {
 401             cp $dmesg, "$faildir/dmesg" or
 402                 die "failed to move $dmesg";
 403         }
 404
 405         doprint "*** Saved info to $faildir ***\n";
 406
 407         return 1;
 408 }
 409
 410 sub run_command {
 411     my ($command) = @_;
 412     my $dolog = 0;
 413     my $dord = 0;
 414     my $pid;
 415
 416     doprint("$command ... ");
 417
 418     $pid = open(CMD, "$command 2>&1 |") or
 419         (fail "unable to exec $command" and return 0);
 420
 421     if (defined($opt{"LOG_FILE"})) {
 422         open(LOG, ">>$opt{LOG_FILE}") or
 423             dodie "failed to write to log";
 424         $dolog = 1;
 425     }
 426
 427     if (defined($redirect)) {
 428         open (RD, ">$redirect") or
 429             dodie "failed to write to redirect $redirect";
 430         $dord = 1;
 431     }
 432
 433     while (<CMD>) {
 434         print LOG if ($dolog);
 435         print RD  if ($dord);
 436     }
 437
 438     waitpid($pid, 0);
 439     my $failed = $?;
 440
 441     close(CMD);
 442     close(LOG) if ($dolog);
 443     close(RD)  if ($dord);
 444
 445     if ($failed) {
 446         doprint "FAILED!\n";
 447     } else {
 448         doprint "SUCCESS\n";
 449     }
 450
 451     return !$failed;
 452 }
 453
 454 sub get_grub_index {
 455
 456     if ($reboot_type ne "grub") {
 457         return;
 458     }
 459     return if (defined($grub_number));
 460
 461     doprint "Find grub menu ... ";
 462     $grub_number = -1;
 463     open(IN, "ssh $target cat /boot/grub/menu.lst |")
 464         or die "unable to get menu.lst";
 465     while (<IN>) {
 466         if (/^\s*title\s+$grub_menu\s*$/) {
 467             $grub_number++;
 468             last;
 469         } elsif (/^\s*title\s/) {
 470             $grub_number++;
 471         }
 472     }
 473     close(IN);
 474
 475     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
 476         if ($grub_number < 0);
 477     doprint "$grub_number\n";
 478 }
 479
 480 sub wait_for_input
 481 {
 482     my ($fp, $time) = @_;
 483     my $rin;
 484     my $ready;
 485     my $line;
 486     my $ch;
 487
 488     if (!defined($time)) {
 489         $time = $timeout;
 490     }
 491
 492     $rin = '';
 493     vec($rin, fileno($fp), 1) = 1;
 494     $ready = select($rin, undef, undef, $time);
 495
 496     $line = "";
 497
 498     # try to read one char at a time
 499     while (sysread $fp, $ch, 1) {
 500         $line .= $ch;
 501         last if ($ch eq "\n");
 502     }
 503
 504     if (!length($line)) {
 505         return undef;
 506     }
 507
 508     return $line;
 509 }
 510
 511 sub reboot_to {
 512     if ($reboot_type eq "grub") {
 513         run_command "ssh $target '(echo \"savedefault --default=$grub_number --once\" | grub --batch; reboot)'";
 514         return;
 515     }
 516
 517     run_command "$reboot_script";
 518 }
 519
 520 sub get_sha1 {
 521     my ($commit) = @_;
 522
 523     doprint "git rev-list --max-count=1 $commit ... ";
 524     my $sha1 = `git rev-list --max-count=1 $commit`;
 525     my $ret = $?;
 526
 527     logit $sha1;
 528
 529     if ($ret) {
 530         doprint "FAILED\n";
 531         dodie "Failed to get git $commit";
 532     }
 533
 534     print "SUCCESS\n";
 535
 536     chomp $sha1;
 537
 538     return $sha1;
 539 }
 540
 541 sub monitor {
 542     my $booted = 0;
 543     my $bug = 0;
 544     my $skip_call_trace = 0;
 545     my $loops;
 546
 547     wait_for_monitor 5;
 548
 549     my $line;
 550     my $full_line = "";
 551
 552     open(DMESG, "> $dmesg") or
 553         die "unable to write to $dmesg";
 554
 555     reboot_to;
 556
 557     for (;;) {
 558
 559         if ($booted) {
 560             $line = wait_for_input($monitor_fp, $booted_timeout);
 561         } else {
 562             $line = wait_for_input($monitor_fp);
 563         }
 564
 565         last if (!defined($line));
 566
 567         doprint $line;
 568         print DMESG $line;
 569
 570         # we are not guaranteed to get a full line
 571         $full_line .= $line;
 572
 573         if ($full_line =~ /$success_line/) {
 574             $booted = 1;
 575         }
 576
 577         if ($full_line =~ /\[ backtrace testing \]/) {
 578             $skip_call_trace = 1;
 579         }
 580
 581         if ($full_line =~ /call trace:/i) {
 582             $bug = 1 if (!$skip_call_trace);
 583         }
 584
 585         if ($full_line =~ /\[ end of backtrace testing \]/) {
 586             $skip_call_trace = 0;
 587         }
 588
 589         if ($full_line =~ /Kernel panic -/) {
 590             $bug = 1;
 591         }
 592
 593         if ($line =~ /\n/) {
 594             $full_line = "";
 595         }
 596     }
 597
 598     close(DMESG);
 599
 600     if ($bug) {
 601         return 0 if ($in_bisect);
 602         fail "failed - got a bug report" and return 0;
 603     }
 604
 605     if (!$booted) {
 606         return 0 if ($in_bisect);
 607         fail "failed - never got a boot prompt." and return 0;
 608     }
 609
 610     return 1;
 611 }
 612
 613 sub install {
 614
 615     run_command "scp $outputdir/$build_target $target:$target_image" or
 616         dodie "failed to copy image";
 617
 618     my $install_mods = 0;
 619
 620     # should we process modules?
 621     $install_mods = 0;
 622     open(IN, "$outputdir/.config") or dodie("Can't read config file");
 623     while (<IN>) {
 624         if (/CONFIG_MODULES(=y)?/) {
 625             $install_mods = 1 if (defined($1));
 626             last;
 627         }
 628     }
 629     close(IN);
 630
 631     if (!$install_mods) {
 632         doprint "No modules needed\n";
 633         return;
 634     }
 635
 636     run_command "$make INSTALL_MOD_PATH=$tmpdir modules_install" or
 637         dodie "Failed to install modules";
 638
 639     my $modlib = "/lib/modules/$version";
 640     my $modtar = "ktest-mods.tar.bz2";
 641
 642     run_command "ssh $target rm -rf $modlib" or
 643         dodie "failed to remove old mods: $modlib";
 644
 645     # would be nice if scp -r did not follow symbolic links
 646     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
 647         dodie "making tarball";
 648
 649     run_command "scp $tmpdir/$modtar $target:/tmp" or
 650         dodie "failed to copy modules";
 651
 652     unlink "$tmpdir/$modtar";
 653
 654     run_command "ssh $target '(cd / && tar xf /tmp/$modtar)'" or
 655         dodie "failed to tar modules";
 656
 657     run_command "ssh $target rm -f /tmp/$modtar";
 658
 659     return if (!defined($post_install));
 660
 661     my $save_env = $ENV{KERNEL_VERSION};
 662
 663     $ENV{KERNEL_VERSION} = $version;
 664     run_command "$post_install" or
 665         dodie "Failed to run post install";
 666
 667     $ENV{KERNEL_VERSION} = $save_env;
 668 }
 669
 670 sub check_buildlog {
 671     my ($patch) = @_;
 672
 673     my @files = `git show $patch | diffstat -l`;
 674
 675     open(IN, "git show $patch |") or
 676         dodie "failed to show $patch";
 677     while (<IN>) {
 678         if (m,^--- a/(.*),) {
 679             chomp $1;
 680             $files[$#files] = $1;
 681         }
 682     }
 683     close(IN);
 684
 685     open(IN, $buildlog) or dodie "Can't open $buildlog";
 686     while (<IN>) {
 687         if (/^\s*(.*?):.*(warning|error)/) {
 688             my $err = $1;
 689             foreach my $file (@files) {
 690                 my $fullpath = "$builddir/$file";
 691                 if ($file eq $err || $fullpath eq $err) {
 692                     fail "$file built with warnings" and return 0;
 693                 }
 694             }
 695         }
 696     }
 697     close(IN);
 698
 699     return 1;
 700 }
 701
 702 sub build {
 703     my ($type) = @_;
 704     my $defconfig = "";
 705
 706     unlink $buildlog;
 707
 708     if ($type =~ /^useconfig:(.*)/) {
 709         run_command "cp $1 $outputdir/.config" or
 710             dodie "could not copy $1 to .config";
 711
 712         $type = "oldconfig";
 713     }
 714
 715     # old config can ask questions
 716     if ($type eq "oldconfig") {
 717         $type = "oldnoconfig";
 718
 719         # allow for empty configs
 720         run_command "touch $outputdir/.config";
 721
 722         run_command "mv $outputdir/.config $outputdir/config_temp" or
 723             dodie "moving .config";
 724
 725         if (!$noclean && !run_command "$make mrproper") {
 726             dodie "make mrproper";
 727         }
 728
 729         run_command "mv $outputdir/config_temp $outputdir/.config" or
 730             dodie "moving config_temp";
 731
 732     } elsif (!$noclean) {
 733         unlink "$outputdir/.config";
 734         run_command "$make mrproper" or
 735             dodie "make mrproper";
 736     }
 737
 738     # add something to distinguish this build
 739     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
 740     print OUT "$localversion\n";
 741     close(OUT);
 742
 743     if (defined($minconfig)) {
 744         $defconfig = "KCONFIG_ALLCONFIG=$minconfig";
 745     }
 746
 747     run_command "$defconfig $make $type" or
 748         dodie "failed make config";
 749
 750     $redirect = "$buildlog";
 751     if (!run_command "$make $build_options") {
 752         undef $redirect;
 753         # bisect may need this to pass
 754         return 0 if ($in_bisect);
 755         fail "failed build" and return 0;
 756     }
 757     undef $redirect;
 758
 759     return 1;
 760 }
 761
 762 sub halt {
 763     if (!run_command "ssh $target halt" or defined($power_off)) {
 764         if (defined($poweroff_after_halt)) {
 765             sleep $poweroff_after_halt;
 766             run_command "$power_off";
 767         }
 768     } else {
 769         # nope? the zap it!
 770         run_command "$power_off";
 771     }
 772 }
 773
 774 sub success {
 775     my ($i) = @_;
 776
 777     doprint "\n\n*******************************************\n";
 778     doprint     "*******************************************\n";
 779     doprint     "**           TEST $i SUCCESS!!!!         **\n";
 780     doprint     "*******************************************\n";
 781     doprint     "*******************************************\n";
 782
 783     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
 784         doprint "Reboot and wait $sleep_time seconds\n";
 785         reboot;
 786         start_monitor;
 787         wait_for_monitor $sleep_time;
 788         end_monitor;
 789     }
 790 }
 791
 792 sub get_version {
 793     # get the release name
 794     doprint "$make kernelrelease ... ";
 795     $version = `$make kernelrelease | tail -1`;
 796     chomp($version);
 797     doprint "$version\n";
 798 }
 799
 800 sub child_run_test {
 801     my $failed = 0;
 802
 803     # child should have no power
 804     $reboot_on_error = 0;
 805     $poweroff_on_error = 0;
 806     $die_on_failure = 1;
 807
 808     run_command $run_test or $failed = 1;
 809     exit $failed;
 810 }
 811
 812 my $child_done;
 813
 814 sub child_finished {
 815     $child_done = 1;
 816 }
 817
 818 sub do_run_test {
 819     my $child_pid;
 820     my $child_exit;
 821     my $line;
 822     my $full_line;
 823     my $bug = 0;
 824
 825     wait_for_monitor 1;
 826
 827     doprint "run test $run_test\n";
 828
 829     $child_done = 0;
 830
 831     $SIG{CHLD} = qw(child_finished);
 832
 833     $child_pid = fork;
 834
 835     child_run_test if (!$child_pid);
 836
 837     $full_line = "";
 838
 839     do {
 840         $line = wait_for_input($monitor_fp, 1);
 841         if (defined($line)) {
 842
 843             # we are not guaranteed to get a full line
 844             $full_line .= $line;
 845
 846             if ($full_line =~ /call trace:/i) {
 847                 $bug = 1;
 848             }
 849
 850             if ($full_line =~ /Kernel panic -/) {
 851                 $bug = 1;
 852             }
 853
 854             if ($line =~ /\n/) {
 855                 $full_line = "";
 856             }
 857         }
 858     } while (!$child_done && !$bug);
 859
 860     if ($bug) {
 861         doprint "Detected kernel crash!\n";
 862         # kill the child with extreme prejudice
 863         kill 9, $child_pid;
 864     }
 865
 866     waitpid $child_pid, 0;
 867     $child_exit = $?;
 868
 869     if ($bug || $child_exit) {
 870         return 0 if $in_bisect;
 871         fail "test failed" and return 0;
 872     }
 873     return 1;
 874 }
 875
 876 sub run_git_bisect {
 877     my ($command) = @_;
 878
 879     doprint "$command ... ";
 880
 881     my $output = `$command 2>&1`;
 882     my $ret = $?;
 883
 884     logit $output;
 885
 886     if ($ret) {
 887         doprint "FAILED\n";
 888         dodie "Failed to git bisect";
 889     }
 890
 891     doprint "SUCCESS\n";
 892     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
 893         doprint "$1 [$2]\n";
 894     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
 895         $bisect_bad = $1;
 896         doprint "Found bad commit... $1\n";
 897         return 0;
 898     } else {
 899         # we already logged it, just print it now.
 900         print $output;
 901     }
 902
 903     return 1;
 904 }
 905
 906 sub run_bisect {
 907     my ($type) = @_;
 908
 909     my $failed = 0;
 910     my $result;
 911     my $output;
 912     my $ret;
 913
 914     if (defined($minconfig)) {
 915         build "useconfig:$minconfig" or $failed = 1;
 916     } else {
 917         # ?? no config to use?
 918         build "oldconfig" or $failed = 1;
 919     }
 920
 921     if ($type ne "build") {
 922         dodie "Failed on build" if $failed;
 923
 924         # Now boot the box
 925         get_grub_index;
 926         get_version;
 927         install;
 928
 929         start_monitor;
 930         monitor or $failed = 1;
 931
 932         if ($type ne "boot") {
 933             dodie "Failed on boot" if $failed;
 934
 935             do_run_test or $failed = 1;
 936         }
 937         end_monitor;
 938     }
 939
 940     if ($failed) {
 941         $result = "bad";
 942
 943         # reboot the box to a good kernel
 944         if ($type ne "build") {
 945             doprint "Reboot and sleep $bisect_sleep_time seconds\n";
 946             reboot;
 947             start_monitor;
 948             wait_for_monitor $bisect_sleep_time;
 949             end_monitor;
 950         }
 951     } else {
 952         $result = "good";
 953     }
 954
 955     # Are we looking for where it worked, not failed?
 956     if ($reverse_bisect) {
 957         if ($failed) {
 958             $result = "good";
 959         } else {
 960             $result = "bad";
 961         }
 962     }
 963
 964     return $result;
 965 }
 966
 967 sub bisect {
 968     my ($i) = @_;
 969
 970     my $result;
 971
 972     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
 973     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
 974     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
 975
 976     my $good = $opt{"BISECT_GOOD[$i]"};
 977     my $bad = $opt{"BISECT_BAD[$i]"};
 978     my $type = $opt{"BISECT_TYPE[$i]"};
 979     my $start = $opt{"BISECT_START[$i]"};
 980     my $replay = $opt{"BISECT_REPLAY[$i]"};
 981
 982     # convert to true sha1's
 983     $good = get_sha1($good);
 984     $bad = get_sha1($bad);
 985
 986     if (defined($opt{"BISECT_REVERSE[$i]"}) &&
 987         $opt{"BISECT_REVERSE[$i]"} == 1) {
 988         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
 989         $reverse_bisect = 1;
 990     } else {
 991         $reverse_bisect = 0;
 992     }
 993
 994     $in_bisect = 1;
 995
 996     # Can't have a test without having a test to run
 997     if ($type eq "test" && !defined($run_test)) {
 998         $type = "boot";
 999     }
1000
1001     my $check = $opt{"BISECT_CHECK[$i]"};
1002     if (defined($check) && $check ne "0") {
1003
1004         # get current HEAD
1005         my $head = get_sha1("HEAD");
1006
1007         if ($check ne "good") {
1008             doprint "TESTING BISECT BAD [$bad]\n";
1009             run_command "git checkout $bad" or
1010                 die "Failed to checkout $bad";
1011
1012             $result = run_bisect $type;
1013
1014             if ($result ne "bad") {
1015                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
1016             }
1017         }
1018
1019         if ($check ne "bad") {
1020             doprint "TESTING BISECT GOOD [$good]\n";
1021             run_command "git checkout $good" or
1022                 die "Failed to checkout $good";
1023
1024             $result = run_bisect $type;
1025
1026             if ($result ne "good") {
1027                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
1028             }
1029         }
1030
1031         # checkout where we started
1032         run_command "git checkout $head" or
1033             die "Failed to checkout $head";
1034     }
1035
1036     run_command "git bisect start" or
1037         dodie "could not start bisect";
1038
1039     run_command "git bisect good $good" or
1040         dodie "could not set bisect good to $good";
1041
1042     run_git_bisect "git bisect bad $bad" or
1043         dodie "could not set bisect bad to $bad";
1044
1045     if (defined($replay)) {
1046         run_command "git bisect replay $replay" or
1047             dodie "failed to run replay";
1048     }
1049
1050     if (defined($start)) {
1051         run_command "git checkout $start" or
1052             dodie "failed to checkout $start";
1053     }
1054
1055     my $test;
1056     do {
1057         $result = run_bisect $type;
1058         $test = run_git_bisect "git bisect $result";
1059     } while ($test);
1060
1061     run_command "git bisect log" or
1062         dodie "could not capture git bisect log";
1063
1064     run_command "git bisect reset" or
1065         dodie "could not reset git bisect";
1066
1067     doprint "Bad commit was [$bisect_bad]\n";
1068
1069     $in_bisect = 0;
1070
1071     success $i;
1072 }
1073
1074 sub patchcheck {
1075     my ($i) = @_;
1076
1077     die "PATCHCHECK_START[$i] not defined\n"
1078         if (!defined($opt{"PATCHCHECK_START[$i]"}));
1079     die "PATCHCHECK_TYPE[$i] not defined\n"
1080         if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
1081
1082     my $start = $opt{"PATCHCHECK_START[$i]"};
1083
1084     my $end = "HEAD";
1085     if (defined($opt{"PATCHCHECK_END[$i]"})) {
1086         $end = $opt{"PATCHCHECK_END[$i]"};
1087     }
1088
1089     # Get the true sha1's since we can use things like HEAD~3
1090     $start = get_sha1($start);
1091     $end = get_sha1($end);
1092
1093     my $type = $opt{"PATCHCHECK_TYPE[$i]"};
1094
1095     # Can't have a test without having a test to run
1096     if ($type eq "test" && !defined($run_test)) {
1097         $type = "boot";
1098     }
1099
1100     open (IN, "git log --pretty=oneline $end|") or
1101         dodie "could not get git list";
1102
1103     my @list;
1104
1105     while (<IN>) {
1106         chomp;
1107         $list[$#list+1] = $_;
1108         last if (/^$start/);
1109     }
1110     close(IN);
1111
1112     if ($list[$#list] !~ /^$start/) {
1113         fail "SHA1 $start not found";
1114     }
1115
1116     # go backwards in the list
1117     @list = reverse @list;
1118
1119     my $save_clean = $noclean;
1120
1121     $in_patchcheck = 1;
1122     foreach my $item (@list) {
1123         my $sha1 = $item;
1124         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
1125
1126         doprint "\nProcessing commit $item\n\n";
1127
1128         run_command "git checkout $sha1" or
1129             die "Failed to checkout $sha1";
1130
1131         # only clean on the first and last patch
1132         if ($item eq $list[0] ||
1133             $item eq $list[$#list]) {
1134             $noclean = $save_clean;
1135         } else {
1136             $noclean = 1;
1137         }
1138
1139         if (defined($minconfig)) {
1140             build "useconfig:$minconfig" or return 0;
1141         } else {
1142             # ?? no config to use?
1143             build "oldconfig" or return 0;
1144         }
1145
1146         check_buildlog $sha1 or return 0;
1147
1148         next if ($type eq "build");
1149
1150         get_grub_index;
1151         get_version;
1152         install;
1153
1154         my $failed = 0;
1155
1156         start_monitor;
1157         monitor or $failed = 1;
1158
1159         if (!$failed && $type ne "boot"){
1160             do_run_test or $failed = 1;
1161         }
1162         end_monitor;
1163         return 0 if ($failed);
1164
1165     }
1166     $in_patchcheck = 0;
1167     success $i;
1168
1169     return 1;
1170 }
1171
1172 read_config $ARGV[0];
1173
1174 # mandatory configs
1175 die "MACHINE not defined\n"             if (!defined($opt{"MACHINE"}));
1176 die "SSH_USER not defined\n"            if (!defined($opt{"SSH_USER"}));
1177 die "BUILD_DIR not defined\n"           if (!defined($opt{"BUILD_DIR"}));
1178 die "OUTPUT_DIR not defined\n"          if (!defined($opt{"OUTPUT_DIR"}));
1179 die "BUILD_TARGET not defined\n"        if (!defined($opt{"BUILD_TARGET"}));
1180 die "TARGET_IMAGE not defined\n"        if (!defined($opt{"TARGET_IMAGE"}));
1181 die "POWER_CYCLE not defined\n"         if (!defined($opt{"POWER_CYCLE"}));
1182 die "CONSOLE not defined\n"             if (!defined($opt{"CONSOLE"}));
1183 die "LOCALVERSION not defined\n"        if (!defined($opt{"LOCALVERSION"}));
1184
1185 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
1186     unlink $opt{"LOG_FILE"};
1187 }
1188
1189 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
1190
1191 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
1192
1193     if (!$i) {
1194         doprint "DEFAULT OPTIONS:\n";
1195     } else {
1196         doprint "\nTEST $i OPTIONS";
1197         if (defined($repeat_tests{$i})) {
1198             $repeat = $repeat_tests{$i};
1199             doprint " ITERATE $repeat";
1200         }
1201         doprint "\n";
1202     }
1203
1204     foreach my $option (sort keys %opt) {
1205
1206         if ($option =~ /\[(\d+)\]$/) {
1207             next if ($i != $1);
1208         } else {
1209             next if ($i);
1210         }
1211
1212         doprint "$option = $opt{$option}\n";
1213     }
1214 }
1215
1216 sub set_test_option {
1217     my ($name, $i) = @_;
1218
1219     my $option = "$name\[$i\]";
1220
1221     if (defined($opt{$option})) {
1222         return $opt{$option};
1223     }
1224
1225     foreach my $test (keys %repeat_tests) {
1226         if ($i >= $test &&
1227             $i < $test + $repeat_tests{$test}) {
1228             $option = "$name\[$test\]";
1229             if (defined($opt{$option})) {
1230                 return $opt{$option};
1231             }
1232         }
1233     }
1234
1235     if (defined($opt{$name})) {
1236         return $opt{$name};
1237     }
1238
1239     return undef;
1240 }
1241
1242 # First we need to do is the builds
1243 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
1244
1245     $iteration = $i;
1246
1247     my $ssh_user = set_test_option("SSH_USER", $i);
1248     my $makecmd = set_test_option("MAKE_CMD", $i);
1249
1250     $machine = set_test_option("MACHINE", $i);
1251     $tmpdir = set_test_option("TMP_DIR", $i);
1252     $outputdir = set_test_option("OUTPUT_DIR", $i);
1253     $builddir = set_test_option("BUILD_DIR", $i);
1254     $test_type = set_test_option("TEST_TYPE", $i);
1255     $build_type = set_test_option("BUILD_TYPE", $i);
1256     $build_options = set_test_option("BUILD_OPTIONS", $i);
1257     $power_cycle = set_test_option("POWER_CYCLE", $i);
1258     $noclean = set_test_option("BUILD_NOCLEAN", $i);
1259     $minconfig = set_test_option("MIN_CONFIG", $i);
1260     $run_test = set_test_option("TEST", $i);
1261     $addconfig = set_test_option("ADD_CONFIG", $i);
1262     $reboot_type = set_test_option("REBOOT_TYPE", $i);
1263     $grub_menu = set_test_option("GRUB_MENU", $i);
1264     $post_install = set_test_option("POST_INSTALL", $i);
1265     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
1266     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
1267     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
1268     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
1269     $power_off = set_test_option("POWER_OFF", $i);
1270     $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
1271     $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
1272     $sleep_time = set_test_option("SLEEP_TIME", $i);
1273     $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
1274     $store_failures = set_test_option("STORE_FAILURES", $i);
1275     $timeout = set_test_option("TIMEOUT", $i);
1276     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
1277     $console = set_test_option("CONSOLE", $i);
1278     $success_line = set_test_option("SUCCESS_LINE", $i);
1279     $build_target = set_test_option("BUILD_TARGET", $i);
1280     $target_image = set_test_option("TARGET_IMAGE", $i);
1281     $localversion = set_test_option("LOCALVERSION", $i);
1282
1283     chdir $builddir || die "can't change directory to $builddir";
1284
1285     if (!-d $tmpdir) {
1286         mkpath($tmpdir) or
1287             die "can't create $tmpdir";
1288     }
1289
1290     $target = "$ssh_user\@$machine";
1291
1292     $buildlog = "$tmpdir/buildlog-$machine";
1293     $dmesg = "$tmpdir/dmesg-$machine";
1294     $make = "$makecmd O=$outputdir";
1295
1296     if ($reboot_type eq "grub") {
1297         dodie "GRUB_MENU not defined" if (!defined($grub_menu));
1298     } elsif (!defined($reboot_script)) {
1299         dodie "REBOOT_SCRIPT not defined"
1300     }
1301
1302     my $run_type = $build_type;
1303     if ($test_type eq "patchcheck") {
1304         $run_type = $opt{"PATCHCHECK_TYPE[$i]"};
1305     } elsif ($test_type eq "bisect") {
1306         $run_type = $opt{"BISECT_TYPE[$i]"};
1307     }
1308
1309     # mistake in config file?
1310     if (!defined($run_type)) {
1311         $run_type = "ERROR";
1312     }
1313
1314     doprint "\n\n";
1315     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type\n\n";
1316
1317     unlink $dmesg;
1318     unlink $buildlog;
1319
1320     if (!defined($minconfig)) {
1321         $minconfig = $addconfig;
1322
1323     } elsif (defined($addconfig)) {
1324         run_command "cat $addconfig $minconfig > $tmpdir/use_config" or
1325             dodie "Failed to create temp config";
1326         $minconfig = "$tmpdir/use_config";
1327     }
1328
1329     my $checkout = $opt{"CHECKOUT[$i]"};
1330     if (defined($checkout)) {
1331         run_command "git checkout $checkout" or
1332             die "failed to checkout $checkout";
1333     }
1334
1335     if ($test_type eq "bisect") {
1336         bisect $i;
1337         next;
1338     } elsif ($test_type eq "patchcheck") {
1339         patchcheck $i;
1340         next;
1341     }
1342
1343     if ($build_type ne "nobuild") {
1344         build $build_type or next;
1345     }
1346
1347     if ($test_type ne "build") {
1348         get_grub_index;
1349         get_version;
1350         install;
1351
1352         my $failed = 0;
1353         start_monitor;
1354         monitor or $failed = 1;;
1355
1356         if (!$failed && $test_type ne "boot" && defined($run_test)) {
1357             do_run_test or $failed = 1;
1358         }
1359         end_monitor;
1360         next if ($failed);
1361     }
1362
1363     success $i;
1364 }
1365
1366 if ($opt{"POWEROFF_ON_SUCCESS"}) {
1367     halt;
1368 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
1369     reboot;
1370 }
1371
1372 exit 0;