tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copyright 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21 my %default;
  22
  23 #default opts
  24 $default{"NUM_TESTS"}           = 1;
  25 $default{"REBOOT_TYPE"}         = "grub";
  26 $default{"TEST_TYPE"}           = "test";
  27 $default{"BUILD_TYPE"}          = "randconfig";
  28 $default{"MAKE_CMD"}            = "make";
  29 $default{"TIMEOUT"}             = 120;
  30 $default{"TMP_DIR"}             = "/tmp/ktest";
  31 $default{"SLEEP_TIME"}          = 60;   # sleep time between tests
  32 $default{"BUILD_NOCLEAN"}       = 0;
  33 $default{"REBOOT_ON_ERROR"}     = 0;
  34 $default{"POWEROFF_ON_ERROR"}   = 0;
  35 $default{"REBOOT_ON_SUCCESS"}   = 1;
  36 $default{"POWEROFF_ON_SUCCESS"} = 0;
  37 $default{"BUILD_OPTIONS"}       = "";
  38 $default{"BISECT_SLEEP_TIME"}   = 60;   # sleep time between bisects
  39 $default{"CLEAR_LOG"}           = 0;
  40 $default{"BISECT_MANUAL"}       = 0;
  41 $default{"BISECT_SKIP"}         = 1;
  42 $default{"SUCCESS_LINE"}        = "login:";
  43 $default{"BOOTED_TIMEOUT"}      = 1;
  44 $default{"DIE_ON_FAILURE"}      = 1;
  45 $default{"SSH_EXEC"}            = "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND";
  46 $default{"SCP_TO_TARGET"}       = "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE";
  47 $default{"REBOOT"}              = "ssh \$SSH_USER\@\$MACHINE reboot";
  48 $default{"STOP_AFTER_SUCCESS"}  = 10;
  49 $default{"STOP_AFTER_FAILURE"}  = 60;
  50 $default{"STOP_TEST_AFTER"}     = 600;
  51 $default{"LOCALVERSION"}        = "-test";
  52
  53 my $ktest_config;
  54 my $version;
  55 my $machine;
  56 my $ssh_user;
  57 my $tmpdir;
  58 my $builddir;
  59 my $outputdir;
  60 my $output_config;
  61 my $test_type;
  62 my $build_type;
  63 my $build_options;
  64 my $reboot_type;
  65 my $reboot_script;
  66 my $power_cycle;
  67 my $reboot;
  68 my $reboot_on_error;
  69 my $poweroff_on_error;
  70 my $die_on_failure;
  71 my $powercycle_after_reboot;
  72 my $poweroff_after_halt;
  73 my $ssh_exec;
  74 my $scp_to_target;
  75 my $power_off;
  76 my $grub_menu;
  77 my $grub_number;
  78 my $target;
  79 my $make;
  80 my $post_install;
  81 my $noclean;
  82 my $minconfig;
  83 my $addconfig;
  84 my $in_bisect = 0;
  85 my $bisect_bad = "";
  86 my $reverse_bisect;
  87 my $bisect_manual;
  88 my $bisect_skip;
  89 my $in_patchcheck = 0;
  90 my $run_test;
  91 my $redirect;
  92 my $buildlog;
  93 my $dmesg;
  94 my $monitor_fp;
  95 my $monitor_pid;
  96 my $monitor_cnt = 0;
  97 my $sleep_time;
  98 my $bisect_sleep_time;
  99 my $store_failures;
 100 my $timeout;
 101 my $booted_timeout;
 102 my $console;
 103 my $success_line;
 104 my $stop_after_success;
 105 my $stop_after_failure;
 106 my $stop_test_after;
 107 my $build_target;
 108 my $target_image;
 109 my $localversion;
 110 my $iteration = 0;
 111 my $successes = 0;
 112
 113 my %entered_configs;
 114 my %config_help;
 115
 116 $config_help{"MACHINE"} = << "EOF"
 117  The machine hostname that you will test.
 118 EOF
 119     ;
 120 $config_help{"SSH_USER"} = << "EOF"
 121  The box is expected to have ssh on normal bootup, provide the user
 122   (most likely root, since you need privileged operations)
 123 EOF
 124     ;
 125 $config_help{"BUILD_DIR"} = << "EOF"
 126  The directory that contains the Linux source code (full path).
 127 EOF
 128     ;
 129 $config_help{"OUTPUT_DIR"} = << "EOF"
 130  The directory that the objects will be built (full path).
 131  (can not be same as BUILD_DIR)
 132 EOF
 133     ;
 134 $config_help{"BUILD_TARGET"} = << "EOF"
 135  The location of the compiled file to copy to the target.
 136  (relative to OUTPUT_DIR)
 137 EOF
 138     ;
 139 $config_help{"TARGET_IMAGE"} = << "EOF"
 140  The place to put your image on the test machine.
 141 EOF
 142     ;
 143 $config_help{"POWER_CYCLE"} = << "EOF"
 144  A script or command to reboot the box.
 145
 146  Here is a digital loggers power switch example
 147  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 148
 149  Here is an example to reboot a virtual box on the current host
 150  with the name "Guest".
 151  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 152 EOF
 153     ;
 154 $config_help{"CONSOLE"} = << "EOF"
 155  The script or command that reads the console
 156
 157   If you use ttywatch server, something like the following would work.
 158 CONSOLE = nc -d localhost 3001
 159
 160  For a virtual machine with guest name "Guest".
 161 CONSOLE =  virsh console Guest
 162 EOF
 163     ;
 164 $config_help{"LOCALVERSION"} = << "EOF"
 165  Required version ending to differentiate the test
 166  from other linux builds on the system.
 167 EOF
 168     ;
 169 $config_help{"REBOOT_TYPE"} = << "EOF"
 170  Way to reboot the box to the test kernel.
 171  Only valid options so far are "grub" and "script".
 172
 173  If you specify grub, it will assume grub version 1
 174  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 175  and select that target to reboot to the kernel. If this is not
 176  your setup, then specify "script" and have a command or script
 177  specified in REBOOT_SCRIPT to boot to the target.
 178
 179  The entry in /boot/grub/menu.lst must be entered in manually.
 180  The test will not modify that file.
 181 EOF
 182     ;
 183 $config_help{"GRUB_MENU"} = << "EOF"
 184  The grub title name for the test kernel to boot
 185  (Only mandatory if REBOOT_TYPE = grub)
 186
 187  Note, ktest.pl will not update the grub menu.lst, you need to
 188  manually add an option for the test. ktest.pl will search
 189  the grub menu.lst for this option to find what kernel to
 190  reboot into.
 191
 192  For example, if in the /boot/grub/menu.lst the test kernel title has:
 193  title Test Kernel
 194  kernel vmlinuz-test
 195  GRUB_MENU = Test Kernel
 196 EOF
 197     ;
 198 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 199  A script to reboot the target into the test kernel
 200  (Only mandatory if REBOOT_TYPE = script)
 201 EOF
 202     ;
 203
 204
 205 sub get_ktest_config {
 206     my ($config) = @_;
 207
 208     return if (defined($opt{$config}));
 209
 210     if (defined($config_help{$config})) {
 211         print "\n";
 212         print $config_help{$config};
 213     }
 214
 215     for (;;) {
 216         print "$config = ";
 217         if (defined($default{$config})) {
 218             print "\[$default{$config}\] ";
 219         }
 220         $entered_configs{$config} = <STDIN>;
 221         $entered_configs{$config} =~ s/^\s*(.*\S)\s*$/$1/;
 222         if ($entered_configs{$config} =~ /^\s*$/) {
 223             if ($default{$config}) {
 224                 $entered_configs{$config} = $default{$config};
 225             } else {
 226                 print "Your answer can not be blank\n";
 227                 next;
 228             }
 229         }
 230         last;
 231     }
 232 }
 233
 234 sub get_ktest_configs {
 235     get_ktest_config("MACHINE");
 236     get_ktest_config("SSH_USER");
 237     get_ktest_config("BUILD_DIR");
 238     get_ktest_config("OUTPUT_DIR");
 239     get_ktest_config("BUILD_TARGET");
 240     get_ktest_config("TARGET_IMAGE");
 241     get_ktest_config("POWER_CYCLE");
 242     get_ktest_config("CONSOLE");
 243     get_ktest_config("LOCALVERSION");
 244
 245     my $rtype = $opt{"REBOOT_TYPE"};
 246
 247     if (!defined($rtype)) {
 248         if (!defined($opt{"GRUB_MENU"})) {
 249             get_ktest_config("REBOOT_TYPE");
 250             $rtype = $entered_configs{"REBOOT_TYPE"};
 251         } else {
 252             $rtype = "grub";
 253         }
 254     }
 255
 256     if ($rtype eq "grub") {
 257         get_ktest_config("GRUB_MENU");
 258     } else {
 259         get_ktest_config("REBOOT_SCRIPT");
 260     }
 261 }
 262
 263 sub set_value {
 264     my ($lvalue, $rvalue) = @_;
 265
 266     if (defined($opt{$lvalue})) {
 267         die "Error: Option $lvalue defined more than once!\n";
 268     }
 269     if ($rvalue =~ /^\s*$/) {
 270         delete $opt{$lvalue};
 271     } else {
 272         $opt{$lvalue} = $rvalue;
 273     }
 274 }
 275
 276 sub read_config {
 277     my ($config) = @_;
 278
 279     open(IN, $config) || die "can't read file $config";
 280
 281     my $name = $config;
 282     $name =~ s,.*/(.*),$1,;
 283
 284     my $test_num = 0;
 285     my $default = 1;
 286     my $repeat = 1;
 287     my $num_tests_set = 0;
 288     my $skip = 0;
 289     my $rest;
 290
 291     while (<IN>) {
 292
 293         # ignore blank lines and comments
 294         next if (/^\s*$/ || /\s*\#/);
 295
 296         if (/^\s*TEST_START(.*)/) {
 297
 298             $rest = $1;
 299
 300             if ($num_tests_set) {
 301                 die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 302             }
 303
 304             my $old_test_num = $test_num;
 305             my $old_repeat = $repeat;
 306
 307             $test_num += $repeat;
 308             $default = 0;
 309             $repeat = 1;
 310
 311             if ($rest =~ /\s+SKIP(.*)/) {
 312                 $rest = $1;
 313                 $skip = 1;
 314             } else {
 315                 $skip = 0;
 316             }
 317
 318             if ($rest =~ /\s+ITERATE\s+(\d+)(.*)$/) {
 319                 $repeat = $1;
 320                 $rest = $2;
 321                 $repeat_tests{"$test_num"} = $repeat;
 322             }
 323
 324             if ($rest =~ /\s+SKIP(.*)/) {
 325                 $rest = $1;
 326                 $skip = 1;
 327             }
 328
 329             if ($rest !~ /^\s*$/) {
 330                 die "$name: $.: Gargbage found after TEST_START\n$_";
 331             }
 332
 333             if ($skip) {
 334                 $test_num = $old_test_num;
 335                 $repeat = $old_repeat;
 336             }
 337
 338         } elsif (/^\s*DEFAULTS(.*)$/) {
 339             $default = 1;
 340
 341             $rest = $1;
 342
 343             if ($rest =~ /\s+SKIP(.*)/) {
 344                 $rest = $1;
 345                 $skip = 1;
 346             } else {
 347                 $skip = 0;
 348             }
 349
 350             if ($rest !~ /^\s*$/) {
 351                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 352             }
 353
 354         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 355
 356             next if ($skip);
 357
 358             my $lvalue = $1;
 359             my $rvalue = $2;
 360
 361             if (!$default &&
 362                 ($lvalue eq "NUM_TESTS" ||
 363                  $lvalue eq "LOG_FILE" ||
 364                  $lvalue eq "CLEAR_LOG")) {
 365                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 366             }
 367
 368             if ($lvalue eq "NUM_TESTS") {
 369                 if ($test_num) {
 370                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 371                 }
 372                 if (!$default) {
 373                     die "$name: $.: NUM_TESTS must be set in default section\n";
 374                 }
 375                 $num_tests_set = 1;
 376             }
 377
 378             if ($default || $lvalue =~ /\[\d+\]$/) {
 379                 set_value($lvalue, $rvalue);
 380             } else {
 381                 my $val = "$lvalue\[$test_num\]";
 382                 set_value($val, $rvalue);
 383
 384                 if ($repeat > 1) {
 385                     $repeats{$val} = $repeat;
 386                 }
 387             }
 388         } else {
 389             die "$name: $.: Garbage found in config\n$_";
 390         }
 391     }
 392
 393     close(IN);
 394
 395     if ($test_num) {
 396         $test_num += $repeat - 1;
 397         $opt{"NUM_TESTS"} = $test_num;
 398     }
 399
 400     # make sure we have all mandatory configs
 401     get_ktest_configs;
 402
 403     # set any defaults
 404
 405     foreach my $default (keys %default) {
 406         if (!defined($opt{$default})) {
 407             $opt{$default} = $default{$default};
 408         }
 409     }
 410 }
 411
 412 sub _logit {
 413     if (defined($opt{"LOG_FILE"})) {
 414         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
 415         print OUT @_;
 416         close(OUT);
 417     }
 418 }
 419
 420 sub logit {
 421     if (defined($opt{"LOG_FILE"})) {
 422         _logit @_;
 423     } else {
 424         print @_;
 425     }
 426 }
 427
 428 sub doprint {
 429     print @_;
 430     _logit @_;
 431 }
 432
 433 sub run_command;
 434
 435 sub reboot {
 436     # try to reboot normally
 437     if (run_command $reboot) {
 438         if (defined($powercycle_after_reboot)) {
 439             sleep $powercycle_after_reboot;
 440             run_command "$power_cycle";
 441         }
 442     } else {
 443         # nope? power cycle it.
 444         run_command "$power_cycle";
 445     }
 446 }
 447
 448 sub do_not_reboot {
 449     my $i = $iteration;
 450
 451     return $test_type eq "build" ||
 452         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
 453         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
 454 }
 455
 456 sub dodie {
 457     doprint "CRITICAL FAILURE... ", @_, "\n";
 458
 459     my $i = $iteration;
 460
 461     if ($reboot_on_error && !do_not_reboot) {
 462
 463         doprint "REBOOTING\n";
 464         reboot;
 465
 466     } elsif ($poweroff_on_error && defined($power_off)) {
 467         doprint "POWERING OFF\n";
 468         `$power_off`;
 469     }
 470
 471     if (defined($opt{"LOG_FILE"})) {
 472         print " See $opt{LOG_FILE} for more info.\n";
 473     }
 474
 475     die @_, "\n";
 476 }
 477
 478 sub open_console {
 479     my ($fp) = @_;
 480
 481     my $flags;
 482
 483     my $pid = open($fp, "$console|") or
 484         dodie "Can't open console $console";
 485
 486     $flags = fcntl($fp, F_GETFL, 0) or
 487         dodie "Can't get flags for the socket: $!";
 488     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 489         dodie "Can't set flags for the socket: $!";
 490
 491     return $pid;
 492 }
 493
 494 sub close_console {
 495     my ($fp, $pid) = @_;
 496
 497     doprint "kill child process $pid\n";
 498     kill 2, $pid;
 499
 500     print "closing!\n";
 501     close($fp);
 502 }
 503
 504 sub start_monitor {
 505     if ($monitor_cnt++) {
 506         return;
 507     }
 508     $monitor_fp = \*MONFD;
 509     $monitor_pid = open_console $monitor_fp;
 510
 511     return;
 512
 513     open(MONFD, "Stop perl from warning about single use of MONFD");
 514 }
 515
 516 sub end_monitor {
 517     if (--$monitor_cnt) {
 518         return;
 519     }
 520     close_console($monitor_fp, $monitor_pid);
 521 }
 522
 523 sub wait_for_monitor {
 524     my ($time) = @_;
 525     my $line;
 526
 527     doprint "** Wait for monitor to settle down **\n";
 528
 529     # read the monitor and wait for the system to calm down
 530     do {
 531         $line = wait_for_input($monitor_fp, $time);
 532         print "$line" if (defined($line));
 533     } while (defined($line));
 534     print "** Monitor flushed **\n";
 535 }
 536
 537 sub fail {
 538
 539         if ($die_on_failure) {
 540                 dodie @_;
 541         }
 542
 543         doprint "FAILED\n";
 544
 545         my $i = $iteration;
 546
 547         # no need to reboot for just building.
 548         if (!do_not_reboot) {
 549             doprint "REBOOTING\n";
 550             reboot;
 551             start_monitor;
 552             wait_for_monitor $sleep_time;
 553             end_monitor;
 554         }
 555
 556         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 557         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 558         doprint "KTEST RESULT: TEST $i Failed: ", @_, "\n";
 559         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 560         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 561
 562         return 1 if (!defined($store_failures));
 563
 564         my @t = localtime;
 565         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
 566                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
 567
 568         my $type = $build_type;
 569         if ($type =~ /useconfig/) {
 570             $type = "useconfig";
 571         }
 572
 573         my $dir = "$machine-$test_type-$type-fail-$date";
 574         my $faildir = "$store_failures/$dir";
 575
 576         if (!-d $faildir) {
 577             mkpath($faildir) or
 578                 die "can't create $faildir";
 579         }
 580         if (-f "$output_config") {
 581             cp "$output_config", "$faildir/config" or
 582                 die "failed to copy .config";
 583         }
 584         if (-f $buildlog) {
 585             cp $buildlog, "$faildir/buildlog" or
 586                 die "failed to move $buildlog";
 587         }
 588         if (-f $dmesg) {
 589             cp $dmesg, "$faildir/dmesg" or
 590                 die "failed to move $dmesg";
 591         }
 592
 593         doprint "*** Saved info to $faildir ***\n";
 594
 595         return 1;
 596 }
 597
 598 sub run_command {
 599     my ($command) = @_;
 600     my $dolog = 0;
 601     my $dord = 0;
 602     my $pid;
 603
 604     $command =~ s/\$SSH_USER/$ssh_user/g;
 605     $command =~ s/\$MACHINE/$machine/g;
 606
 607     doprint("$command ... ");
 608
 609     $pid = open(CMD, "$command 2>&1 |") or
 610         (fail "unable to exec $command" and return 0);
 611
 612     if (defined($opt{"LOG_FILE"})) {
 613         open(LOG, ">>$opt{LOG_FILE}") or
 614             dodie "failed to write to log";
 615         $dolog = 1;
 616     }
 617
 618     if (defined($redirect)) {
 619         open (RD, ">$redirect") or
 620             dodie "failed to write to redirect $redirect";
 621         $dord = 1;
 622     }
 623
 624     while (<CMD>) {
 625         print LOG if ($dolog);
 626         print RD  if ($dord);
 627     }
 628
 629     waitpid($pid, 0);
 630     my $failed = $?;
 631
 632     close(CMD);
 633     close(LOG) if ($dolog);
 634     close(RD)  if ($dord);
 635
 636     if ($failed) {
 637         doprint "FAILED!\n";
 638     } else {
 639         doprint "SUCCESS\n";
 640     }
 641
 642     return !$failed;
 643 }
 644
 645 sub run_ssh {
 646     my ($cmd) = @_;
 647     my $cp_exec = $ssh_exec;
 648
 649     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
 650     return run_command "$cp_exec";
 651 }
 652
 653 sub run_scp {
 654     my ($src, $dst) = @_;
 655     my $cp_scp = $scp_to_target;
 656
 657     $cp_scp =~ s/\$SRC_FILE/$src/g;
 658     $cp_scp =~ s/\$DST_FILE/$dst/g;
 659
 660     return run_command "$cp_scp";
 661 }
 662
 663 sub get_grub_index {
 664
 665     if ($reboot_type ne "grub") {
 666         return;
 667     }
 668     return if (defined($grub_number));
 669
 670     doprint "Find grub menu ... ";
 671     $grub_number = -1;
 672
 673     my $ssh_grub = $ssh_exec;
 674     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
 675
 676     open(IN, "$ssh_grub |")
 677         or die "unable to get menu.lst";
 678
 679     while (<IN>) {
 680         if (/^\s*title\s+$grub_menu\s*$/) {
 681             $grub_number++;
 682             last;
 683         } elsif (/^\s*title\s/) {
 684             $grub_number++;
 685         }
 686     }
 687     close(IN);
 688
 689     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
 690         if ($grub_number < 0);
 691     doprint "$grub_number\n";
 692 }
 693
 694 sub wait_for_input
 695 {
 696     my ($fp, $time) = @_;
 697     my $rin;
 698     my $ready;
 699     my $line;
 700     my $ch;
 701
 702     if (!defined($time)) {
 703         $time = $timeout;
 704     }
 705
 706     $rin = '';
 707     vec($rin, fileno($fp), 1) = 1;
 708     $ready = select($rin, undef, undef, $time);
 709
 710     $line = "";
 711
 712     # try to read one char at a time
 713     while (sysread $fp, $ch, 1) {
 714         $line .= $ch;
 715         last if ($ch eq "\n");
 716     }
 717
 718     if (!length($line)) {
 719         return undef;
 720     }
 721
 722     return $line;
 723 }
 724
 725 sub reboot_to {
 726     if ($reboot_type eq "grub") {
 727         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch; reboot)'";
 728         return;
 729     }
 730
 731     run_command "$reboot_script";
 732 }
 733
 734 sub get_sha1 {
 735     my ($commit) = @_;
 736
 737     doprint "git rev-list --max-count=1 $commit ... ";
 738     my $sha1 = `git rev-list --max-count=1 $commit`;
 739     my $ret = $?;
 740
 741     logit $sha1;
 742
 743     if ($ret) {
 744         doprint "FAILED\n";
 745         dodie "Failed to get git $commit";
 746     }
 747
 748     print "SUCCESS\n";
 749
 750     chomp $sha1;
 751
 752     return $sha1;
 753 }
 754
 755 sub monitor {
 756     my $booted = 0;
 757     my $bug = 0;
 758     my $skip_call_trace = 0;
 759     my $loops;
 760
 761     wait_for_monitor 5;
 762
 763     my $line;
 764     my $full_line = "";
 765
 766     open(DMESG, "> $dmesg") or
 767         die "unable to write to $dmesg";
 768
 769     reboot_to;
 770
 771     my $success_start;
 772     my $failure_start;
 773     my $monitor_start = time;
 774     my $done = 0;
 775
 776     while (!$done) {
 777
 778         if ($booted) {
 779             $line = wait_for_input($monitor_fp, $booted_timeout);
 780         } else {
 781             $line = wait_for_input($monitor_fp);
 782         }
 783
 784         last if (!defined($line));
 785
 786         doprint $line;
 787         print DMESG $line;
 788
 789         # we are not guaranteed to get a full line
 790         $full_line .= $line;
 791
 792         if ($full_line =~ /$success_line/) {
 793             $booted = 1;
 794             $success_start = time;
 795         }
 796
 797         if ($booted && defined($stop_after_success) &&
 798             $stop_after_success >= 0) {
 799             my $now = time;
 800             if ($now - $success_start >= $stop_after_success) {
 801                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
 802                 last;
 803             }
 804         }
 805
 806         if ($full_line =~ /\[ backtrace testing \]/) {
 807             $skip_call_trace = 1;
 808         }
 809
 810         if ($full_line =~ /call trace:/i) {
 811             if (!$bug && !$skip_call_trace) {
 812                 $bug = 1;
 813                 $failure_start = time;
 814             }
 815         }
 816
 817         if ($bug && defined($stop_after_failure) &&
 818             $stop_after_failure >= 0) {
 819             my $now = time;
 820             if ($now - $failure_start >= $stop_after_failure) {
 821                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
 822                 last;
 823             }
 824         }
 825
 826         if ($full_line =~ /\[ end of backtrace testing \]/) {
 827             $skip_call_trace = 0;
 828         }
 829
 830         if ($full_line =~ /Kernel panic -/) {
 831             $failure_start = time;
 832             $bug = 1;
 833         }
 834
 835         if ($line =~ /\n/) {
 836             $full_line = "";
 837         }
 838
 839         if ($stop_test_after > 0 && !$booted && !$bug) {
 840             if (time - $monitor_start > $stop_test_after) {
 841                 $done = 1;
 842             }
 843         }
 844     }
 845
 846     close(DMESG);
 847
 848     if ($bug) {
 849         return 0 if ($in_bisect);
 850         fail "failed - got a bug report" and return 0;
 851     }
 852
 853     if (!$booted) {
 854         return 0 if ($in_bisect);
 855         fail "failed - never got a boot prompt." and return 0;
 856     }
 857
 858     return 1;
 859 }
 860
 861 sub install {
 862
 863     run_scp "$outputdir/$build_target", "$target_image" or
 864         dodie "failed to copy image";
 865
 866     my $install_mods = 0;
 867
 868     # should we process modules?
 869     $install_mods = 0;
 870     open(IN, "$output_config") or dodie("Can't read config file");
 871     while (<IN>) {
 872         if (/CONFIG_MODULES(=y)?/) {
 873             $install_mods = 1 if (defined($1));
 874             last;
 875         }
 876     }
 877     close(IN);
 878
 879     if (!$install_mods) {
 880         doprint "No modules needed\n";
 881         return;
 882     }
 883
 884     run_command "$make INSTALL_MOD_PATH=$tmpdir modules_install" or
 885         dodie "Failed to install modules";
 886
 887     my $modlib = "/lib/modules/$version";
 888     my $modtar = "ktest-mods.tar.bz2";
 889
 890     run_ssh "rm -rf $modlib" or
 891         dodie "failed to remove old mods: $modlib";
 892
 893     # would be nice if scp -r did not follow symbolic links
 894     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
 895         dodie "making tarball";
 896
 897     run_scp "$tmpdir/$modtar", "/tmp" or
 898         dodie "failed to copy modules";
 899
 900     unlink "$tmpdir/$modtar";
 901
 902     run_ssh "'(cd / && tar xf /tmp/$modtar)'" or
 903         dodie "failed to tar modules";
 904
 905     run_ssh "rm -f /tmp/$modtar";
 906
 907     return if (!defined($post_install));
 908
 909     my $cp_post_install = $post_install;
 910     $cp_post_install = s/\$KERNEL_VERSION/$version/g;
 911     run_command "$cp_post_install" or
 912         dodie "Failed to run post install";
 913 }
 914
 915 sub check_buildlog {
 916     my ($patch) = @_;
 917
 918     my @files = `git show $patch | diffstat -l`;
 919
 920     open(IN, "git show $patch |") or
 921         dodie "failed to show $patch";
 922     while (<IN>) {
 923         if (m,^--- a/(.*),) {
 924             chomp $1;
 925             $files[$#files] = $1;
 926         }
 927     }
 928     close(IN);
 929
 930     open(IN, $buildlog) or dodie "Can't open $buildlog";
 931     while (<IN>) {
 932         if (/^\s*(.*?):.*(warning|error)/) {
 933             my $err = $1;
 934             foreach my $file (@files) {
 935                 my $fullpath = "$builddir/$file";
 936                 if ($file eq $err || $fullpath eq $err) {
 937                     fail "$file built with warnings" and return 0;
 938                 }
 939             }
 940         }
 941     }
 942     close(IN);
 943
 944     return 1;
 945 }
 946
 947 sub make_oldconfig {
 948     my ($defconfig) = @_;
 949
 950     if (!run_command "$defconfig $make oldnoconfig") {
 951         # Perhaps oldnoconfig doesn't exist in this version of the kernel
 952         # try a yes '' | oldconfig
 953         doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
 954         run_command "yes '' | $defconfig $make oldconfig" or
 955             dodie "failed make config oldconfig";
 956     }
 957 }
 958
 959 sub build {
 960     my ($type) = @_;
 961     my $defconfig = "";
 962
 963     unlink $buildlog;
 964
 965     if ($type =~ /^useconfig:(.*)/) {
 966         run_command "cp $1 $output_config" or
 967             dodie "could not copy $1 to .config";
 968
 969         $type = "oldconfig";
 970     }
 971
 972     # old config can ask questions
 973     if ($type eq "oldconfig") {
 974         $type = "oldnoconfig";
 975
 976         # allow for empty configs
 977         run_command "touch $output_config";
 978
 979         run_command "mv $output_config $outputdir/config_temp" or
 980             dodie "moving .config";
 981
 982         if (!$noclean && !run_command "$make mrproper") {
 983             dodie "make mrproper";
 984         }
 985
 986         run_command "mv $outputdir/config_temp $output_config" or
 987             dodie "moving config_temp";
 988
 989     } elsif (!$noclean) {
 990         unlink "$output_config";
 991         run_command "$make mrproper" or
 992             dodie "make mrproper";
 993     }
 994
 995     # add something to distinguish this build
 996     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
 997     print OUT "$localversion\n";
 998     close(OUT);
 999
1000     if (defined($minconfig)) {
1001         $defconfig = "KCONFIG_ALLCONFIG=$minconfig";
1002     }
1003
1004     if ($type eq "oldnoconfig") {
1005         make_oldconfig $defconfig;
1006     } else {
1007         run_command "$defconfig $make $type" or
1008             dodie "failed make config";
1009     }
1010
1011     $redirect = "$buildlog";
1012     if (!run_command "$make $build_options") {
1013         undef $redirect;
1014         # bisect may need this to pass
1015         return 0 if ($in_bisect);
1016         fail "failed build" and return 0;
1017     }
1018     undef $redirect;
1019
1020     return 1;
1021 }
1022
1023 sub halt {
1024     if (!run_ssh "halt" or defined($power_off)) {
1025         if (defined($poweroff_after_halt)) {
1026             sleep $poweroff_after_halt;
1027             run_command "$power_off";
1028         }
1029     } else {
1030         # nope? the zap it!
1031         run_command "$power_off";
1032     }
1033 }
1034
1035 sub success {
1036     my ($i) = @_;
1037
1038     $successes++;
1039
1040     doprint "\n\n*******************************************\n";
1041     doprint     "*******************************************\n";
1042     doprint     "KTEST RESULT: TEST $i SUCCESS!!!!         **\n";
1043     doprint     "*******************************************\n";
1044     doprint     "*******************************************\n";
1045
1046     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
1047         doprint "Reboot and wait $sleep_time seconds\n";
1048         reboot;
1049         start_monitor;
1050         wait_for_monitor $sleep_time;
1051         end_monitor;
1052     }
1053 }
1054
1055 sub get_version {
1056     # get the release name
1057     doprint "$make kernelrelease ... ";
1058     $version = `$make kernelrelease | tail -1`;
1059     chomp($version);
1060     doprint "$version\n";
1061 }
1062
1063 sub answer_bisect {
1064     for (;;) {
1065         doprint "Pass or fail? [p/f]";
1066         my $ans = <STDIN>;
1067         chomp $ans;
1068         if ($ans eq "p" || $ans eq "P") {
1069             return 1;
1070         } elsif ($ans eq "f" || $ans eq "F") {
1071             return 0;
1072         } else {
1073             print "Please answer 'P' or 'F'\n";
1074         }
1075     }
1076 }
1077
1078 sub child_run_test {
1079     my $failed = 0;
1080
1081     # child should have no power
1082     $reboot_on_error = 0;
1083     $poweroff_on_error = 0;
1084     $die_on_failure = 1;
1085
1086     run_command $run_test or $failed = 1;
1087     exit $failed;
1088 }
1089
1090 my $child_done;
1091
1092 sub child_finished {
1093     $child_done = 1;
1094 }
1095
1096 sub do_run_test {
1097     my $child_pid;
1098     my $child_exit;
1099     my $line;
1100     my $full_line;
1101     my $bug = 0;
1102
1103     wait_for_monitor 1;
1104
1105     doprint "run test $run_test\n";
1106
1107     $child_done = 0;
1108
1109     $SIG{CHLD} = qw(child_finished);
1110
1111     $child_pid = fork;
1112
1113     child_run_test if (!$child_pid);
1114
1115     $full_line = "";
1116
1117     do {
1118         $line = wait_for_input($monitor_fp, 1);
1119         if (defined($line)) {
1120
1121             # we are not guaranteed to get a full line
1122             $full_line .= $line;
1123             doprint $line;
1124
1125             if ($full_line =~ /call trace:/i) {
1126                 $bug = 1;
1127             }
1128
1129             if ($full_line =~ /Kernel panic -/) {
1130                 $bug = 1;
1131             }
1132
1133             if ($line =~ /\n/) {
1134                 $full_line = "";
1135             }
1136         }
1137     } while (!$child_done && !$bug);
1138
1139     if ($bug) {
1140         my $failure_start = time;
1141         my $now;
1142         do {
1143             $line = wait_for_input($monitor_fp, 1);
1144             if (defined($line)) {
1145                 doprint $line;
1146             }
1147             $now = time;
1148             if ($now - $failure_start >= $stop_after_failure) {
1149                 last;
1150             }
1151         } while (defined($line));
1152
1153         doprint "Detected kernel crash!\n";
1154         # kill the child with extreme prejudice
1155         kill 9, $child_pid;
1156     }
1157
1158     waitpid $child_pid, 0;
1159     $child_exit = $?;
1160
1161     if ($bug || $child_exit) {
1162         return 0 if $in_bisect;
1163         fail "test failed" and return 0;
1164     }
1165     return 1;
1166 }
1167
1168 sub run_git_bisect {
1169     my ($command) = @_;
1170
1171     doprint "$command ... ";
1172
1173     my $output = `$command 2>&1`;
1174     my $ret = $?;
1175
1176     logit $output;
1177
1178     if ($ret) {
1179         doprint "FAILED\n";
1180         dodie "Failed to git bisect";
1181     }
1182
1183     doprint "SUCCESS\n";
1184     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
1185         doprint "$1 [$2]\n";
1186     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
1187         $bisect_bad = $1;
1188         doprint "Found bad commit... $1\n";
1189         return 0;
1190     } else {
1191         # we already logged it, just print it now.
1192         print $output;
1193     }
1194
1195     return 1;
1196 }
1197
1198 sub bisect_reboot {
1199     doprint "Reboot and sleep $bisect_sleep_time seconds\n";
1200     reboot;
1201     start_monitor;
1202     wait_for_monitor $bisect_sleep_time;
1203     end_monitor;
1204 }
1205
1206 # returns 1 on success, 0 on failure, -1 on skip
1207 sub run_bisect_test {
1208     my ($type, $buildtype) = @_;
1209
1210     my $failed = 0;
1211     my $result;
1212     my $output;
1213     my $ret;
1214
1215     $in_bisect = 1;
1216
1217     build $buildtype or $failed = 1;
1218
1219     if ($type ne "build") {
1220         if ($failed && $bisect_skip) {
1221             $in_bisect = 0;
1222             return -1;
1223         }
1224         dodie "Failed on build" if $failed;
1225
1226         # Now boot the box
1227         get_grub_index;
1228         get_version;
1229         install;
1230
1231         start_monitor;
1232         monitor or $failed = 1;
1233
1234         if ($type ne "boot") {
1235             if ($failed && $bisect_skip) {
1236                 end_monitor;
1237                 bisect_reboot;
1238                 $in_bisect = 0;
1239                 return -1;
1240             }
1241             dodie "Failed on boot" if $failed;
1242
1243             do_run_test or $failed = 1;
1244         }
1245         end_monitor;
1246     }
1247
1248     if ($failed) {
1249         $result = 0;
1250
1251         # reboot the box to a good kernel
1252         if ($type ne "build") {
1253             bisect_reboot;
1254         }
1255     } else {
1256         $result = 1;
1257     }
1258     $in_bisect = 0;
1259
1260     return $result;
1261 }
1262
1263 sub run_bisect {
1264     my ($type) = @_;
1265     my $buildtype = "oldconfig";
1266
1267     # We should have a minconfig to use?
1268     if (defined($minconfig)) {
1269         $buildtype = "useconfig:$minconfig";
1270     }
1271
1272     my $ret = run_bisect_test $type, $buildtype;
1273
1274     if ($bisect_manual) {
1275         $ret = answer_bisect;
1276     }
1277
1278     # Are we looking for where it worked, not failed?
1279     if ($reverse_bisect) {
1280         $ret = !$ret;
1281     }
1282
1283     if ($ret > 0) {
1284         return "good";
1285     } elsif ($ret == 0) {
1286         return  "bad";
1287     } elsif ($bisect_skip) {
1288         doprint "HIT A BAD COMMIT ... SKIPPING\n";
1289         return "skip";
1290     }
1291 }
1292
1293 sub bisect {
1294     my ($i) = @_;
1295
1296     my $result;
1297
1298     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
1299     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
1300     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
1301
1302     my $good = $opt{"BISECT_GOOD[$i]"};
1303     my $bad = $opt{"BISECT_BAD[$i]"};
1304     my $type = $opt{"BISECT_TYPE[$i]"};
1305     my $start = $opt{"BISECT_START[$i]"};
1306     my $replay = $opt{"BISECT_REPLAY[$i]"};
1307     my $start_files = $opt{"BISECT_FILES[$i]"};
1308
1309     if (defined($start_files)) {
1310         $start_files = " -- " . $start_files;
1311     } else {
1312         $start_files = "";
1313     }
1314
1315     # convert to true sha1's
1316     $good = get_sha1($good);
1317     $bad = get_sha1($bad);
1318
1319     if (defined($opt{"BISECT_REVERSE[$i]"}) &&
1320         $opt{"BISECT_REVERSE[$i]"} == 1) {
1321         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
1322         $reverse_bisect = 1;
1323     } else {
1324         $reverse_bisect = 0;
1325     }
1326
1327     # Can't have a test without having a test to run
1328     if ($type eq "test" && !defined($run_test)) {
1329         $type = "boot";
1330     }
1331
1332     my $check = $opt{"BISECT_CHECK[$i]"};
1333     if (defined($check) && $check ne "0") {
1334
1335         # get current HEAD
1336         my $head = get_sha1("HEAD");
1337
1338         if ($check ne "good") {
1339             doprint "TESTING BISECT BAD [$bad]\n";
1340             run_command "git checkout $bad" or
1341                 die "Failed to checkout $bad";
1342
1343             $result = run_bisect $type;
1344
1345             if ($result ne "bad") {
1346                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
1347             }
1348         }
1349
1350         if ($check ne "bad") {
1351             doprint "TESTING BISECT GOOD [$good]\n";
1352             run_command "git checkout $good" or
1353                 die "Failed to checkout $good";
1354
1355             $result = run_bisect $type;
1356
1357             if ($result ne "good") {
1358                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
1359             }
1360         }
1361
1362         # checkout where we started
1363         run_command "git checkout $head" or
1364             die "Failed to checkout $head";
1365     }
1366
1367     run_command "git bisect start$start_files" or
1368         dodie "could not start bisect";
1369
1370     run_command "git bisect good $good" or
1371         dodie "could not set bisect good to $good";
1372
1373     run_git_bisect "git bisect bad $bad" or
1374         dodie "could not set bisect bad to $bad";
1375
1376     if (defined($replay)) {
1377         run_command "git bisect replay $replay" or
1378             dodie "failed to run replay";
1379     }
1380
1381     if (defined($start)) {
1382         run_command "git checkout $start" or
1383             dodie "failed to checkout $start";
1384     }
1385
1386     my $test;
1387     do {
1388         $result = run_bisect $type;
1389         $test = run_git_bisect "git bisect $result";
1390     } while ($test);
1391
1392     run_command "git bisect log" or
1393         dodie "could not capture git bisect log";
1394
1395     run_command "git bisect reset" or
1396         dodie "could not reset git bisect";
1397
1398     doprint "Bad commit was [$bisect_bad]\n";
1399
1400     success $i;
1401 }
1402
1403 my %config_ignore;
1404 my %config_set;
1405
1406 my %config_list;
1407 my %null_config;
1408
1409 my %dependency;
1410
1411 sub process_config_ignore {
1412     my ($config) = @_;
1413
1414     open (IN, $config)
1415         or dodie "Failed to read $config";
1416
1417     while (<IN>) {
1418         if (/^(.*?(CONFIG\S*)(=.*| is not set))/) {
1419             $config_ignore{$2} = $1;
1420         }
1421     }
1422
1423     close(IN);
1424 }
1425
1426 sub read_current_config {
1427     my ($config_ref) = @_;
1428
1429     %{$config_ref} = ();
1430     undef %{$config_ref};
1431
1432     my @key = keys %{$config_ref};
1433     if ($#key >= 0) {
1434         print "did not delete!\n";
1435         exit;
1436     }
1437     open (IN, "$output_config");
1438
1439     while (<IN>) {
1440         if (/^(CONFIG\S+)=(.*)/) {
1441             ${$config_ref}{$1} = $2;
1442         }
1443     }
1444     close(IN);
1445 }
1446
1447 sub get_dependencies {
1448     my ($config) = @_;
1449
1450     my $arr = $dependency{$config};
1451     if (!defined($arr)) {
1452         return ();
1453     }
1454
1455     my @deps = @{$arr};
1456
1457     foreach my $dep (@{$arr}) {
1458         print "ADD DEP $dep\n";
1459         @deps = (@deps, get_dependencies $dep);
1460     }
1461
1462     return @deps;
1463 }
1464
1465 sub create_config {
1466     my @configs = @_;
1467
1468     open(OUT, ">$output_config") or dodie "Can not write to $output_config";
1469
1470     foreach my $config (@configs) {
1471         print OUT "$config_set{$config}\n";
1472         my @deps = get_dependencies $config;
1473         foreach my $dep (@deps) {
1474             print OUT "$config_set{$dep}\n";
1475         }
1476     }
1477
1478     foreach my $config (keys %config_ignore) {
1479         print OUT "$config_ignore{$config}\n";
1480     }
1481     close(OUT);
1482
1483 #    exit;
1484     make_oldconfig "";
1485 }
1486
1487 sub compare_configs {
1488     my (%a, %b) = @_;
1489
1490     foreach my $item (keys %a) {
1491         if (!defined($b{$item})) {
1492             print "diff $item\n";
1493             return 1;
1494         }
1495         delete $b{$item};
1496     }
1497
1498     my @keys = keys %b;
1499     if ($#keys) {
1500         print "diff2 $keys[0]\n";
1501     }
1502     return -1 if ($#keys >= 0);
1503
1504     return 0;
1505 }
1506
1507 sub run_config_bisect_test {
1508     my ($type) = @_;
1509
1510     return run_bisect_test $type, "oldconfig";
1511 }
1512
1513 sub process_passed {
1514     my (%configs) = @_;
1515
1516     doprint "These configs had no failure: (Enabling them for further compiles)\n";
1517     # Passed! All these configs are part of a good compile.
1518     # Add them to the min options.
1519     foreach my $config (keys %configs) {
1520         if (defined($config_list{$config})) {
1521             doprint " removing $config\n";
1522             $config_ignore{$config} = $config_list{$config};
1523             delete $config_list{$config};
1524         }
1525     }
1526     doprint "config copied to $outputdir/config_good\n";
1527     run_command "cp -f $output_config $outputdir/config_good";
1528 }
1529
1530 sub process_failed {
1531     my ($config) = @_;
1532
1533     doprint "\n\n***************************************\n";
1534     doprint "Found bad config: $config\n";
1535     doprint "***************************************\n\n";
1536 }
1537
1538 sub run_config_bisect {
1539
1540     my @start_list = keys %config_list;
1541
1542     if ($#start_list < 0) {
1543         doprint "No more configs to test!!!\n";
1544         return -1;
1545     }
1546
1547     doprint "***** RUN TEST ***\n";
1548     my $type = $opt{"CONFIG_BISECT_TYPE[$iteration]"};
1549     my $ret;
1550     my %current_config;
1551
1552     my $count = $#start_list + 1;
1553     doprint "  $count configs to test\n";
1554
1555     my $half = int($#start_list / 2);
1556
1557     do {
1558         my @tophalf = @start_list[0 .. $half];
1559
1560         create_config @tophalf;
1561         read_current_config \%current_config;
1562
1563         $count = $#tophalf + 1;
1564         doprint "Testing $count configs\n";
1565         my $found = 0;
1566         # make sure we test something
1567         foreach my $config (@tophalf) {
1568             if (defined($current_config{$config})) {
1569                 logit " $config\n";
1570                 $found = 1;
1571             }
1572         }
1573         if (!$found) {
1574             # try the other half
1575             doprint "Top half produced no set configs, trying bottom half\n";
1576             @tophalf = @start_list[$half .. $#start_list];
1577             create_config @tophalf;
1578             read_current_config \%current_config;
1579             foreach my $config (@tophalf) {
1580                 if (defined($current_config{$config})) {
1581                     logit " $config\n";
1582                     $found = 1;
1583                 }
1584             }
1585             if (!$found) {
1586                 doprint "Failed: Can't make new config with current configs\n";
1587                 foreach my $config (@start_list) {
1588                     doprint "  CONFIG: $config\n";
1589                 }
1590                 return -1;
1591             }
1592             $count = $#tophalf + 1;
1593             doprint "Testing $count configs\n";
1594         }
1595
1596         $ret = run_config_bisect_test $type;
1597         if ($bisect_manual) {
1598             $ret = answer_bisect;
1599         }
1600         if ($ret) {
1601             process_passed %current_config;
1602             return 0;
1603         }
1604
1605         doprint "This config had a failure.\n";
1606         doprint "Removing these configs that were not set in this config:\n";
1607         doprint "config copied to $outputdir/config_bad\n";
1608         run_command "cp -f $output_config $outputdir/config_bad";
1609
1610         # A config exists in this group that was bad.
1611         foreach my $config (keys %config_list) {
1612             if (!defined($current_config{$config})) {
1613                 doprint " removing $config\n";
1614                 delete $config_list{$config};
1615             }
1616         }
1617
1618         @start_list = @tophalf;
1619
1620         if ($#start_list == 0) {
1621             process_failed $start_list[0];
1622             return 1;
1623         }
1624
1625         # remove half the configs we are looking at and see if
1626         # they are good.
1627         $half = int($#start_list / 2);
1628     } while ($half > 0);
1629
1630     # we found a single config, try it again unless we are running manually
1631
1632     if ($bisect_manual) {
1633         process_failed $start_list[0];
1634         return 1;
1635     }
1636
1637     my @tophalf = @start_list[0 .. 0];
1638
1639     $ret = run_config_bisect_test $type;
1640     if ($ret) {
1641         process_passed %current_config;
1642         return 0;
1643     }
1644
1645     process_failed $start_list[0];
1646     return 1;
1647 }
1648
1649 sub config_bisect {
1650     my ($i) = @_;
1651
1652     my $start_config = $opt{"CONFIG_BISECT[$i]"};
1653
1654     my $tmpconfig = "$tmpdir/use_config";
1655
1656     # Make the file with the bad config and the min config
1657     if (defined($minconfig)) {
1658         # read the min config for things to ignore
1659         run_command "cp $minconfig $tmpconfig" or
1660             dodie "failed to copy $minconfig to $tmpconfig";
1661     } else {
1662         unlink $tmpconfig;
1663     }
1664
1665     # Add other configs
1666     if (defined($addconfig)) {
1667         run_command "cat $addconfig >> $tmpconfig" or
1668             dodie "failed to append $addconfig";
1669     }
1670
1671     my $defconfig = "";
1672     if (-f $tmpconfig) {
1673         $defconfig = "KCONFIG_ALLCONFIG=$tmpconfig";
1674         process_config_ignore $tmpconfig;
1675     }
1676
1677     # now process the start config
1678     run_command "cp $start_config $output_config" or
1679         dodie "failed to copy $start_config to $output_config";
1680
1681     # read directly what we want to check
1682     my %config_check;
1683     open (IN, $output_config)
1684         or dodie "faied to open $output_config";
1685
1686     while (<IN>) {
1687         if (/^((CONFIG\S*)=.*)/) {
1688             $config_check{$2} = $1;
1689         }
1690     }
1691     close(IN);
1692
1693     # Now run oldconfig with the minconfig (and addconfigs)
1694     make_oldconfig $defconfig;
1695
1696     # check to see what we lost (or gained)
1697     open (IN, $output_config)
1698         or dodie "Failed to read $start_config";
1699
1700     my %removed_configs;
1701     my %added_configs;
1702
1703     while (<IN>) {
1704         if (/^((CONFIG\S*)=.*)/) {
1705             # save off all options
1706             $config_set{$2} = $1;
1707             if (defined($config_check{$2})) {
1708                 if (defined($config_ignore{$2})) {
1709                     $removed_configs{$2} = $1;
1710                 } else {
1711                     $config_list{$2} = $1;
1712                 }
1713             } elsif (!defined($config_ignore{$2})) {
1714                 $added_configs{$2} = $1;
1715                 $config_list{$2} = $1;
1716             }
1717         }
1718     }
1719     close(IN);
1720
1721     my @confs = keys %removed_configs;
1722     if ($#confs >= 0) {
1723         doprint "Configs overridden by default configs and removed from check:\n";
1724         foreach my $config (@confs) {
1725             doprint " $config\n";
1726         }
1727     }
1728     @confs = keys %added_configs;
1729     if ($#confs >= 0) {
1730         doprint "Configs appearing in make oldconfig and added:\n";
1731         foreach my $config (@confs) {
1732             doprint " $config\n";
1733         }
1734     }
1735
1736     my %config_test;
1737     my $once = 0;
1738
1739     # Sometimes kconfig does weird things. We must make sure
1740     # that the config we autocreate has everything we need
1741     # to test, otherwise we may miss testing configs, or
1742     # may not be able to create a new config.
1743     # Here we create a config with everything set.
1744     create_config (keys %config_list);
1745     read_current_config \%config_test;
1746     foreach my $config (keys %config_list) {
1747         if (!defined($config_test{$config})) {
1748             if (!$once) {
1749                 $once = 1;
1750                 doprint "Configs not produced by kconfig (will not be checked):\n";
1751             }
1752             doprint "  $config\n";
1753             delete $config_list{$config};
1754         }
1755     }
1756     my $ret;
1757     do {
1758         $ret = run_config_bisect;
1759     } while (!$ret);
1760
1761     return $ret if ($ret < 0);
1762
1763     success $i;
1764 }
1765
1766 sub patchcheck {
1767     my ($i) = @_;
1768
1769     die "PATCHCHECK_START[$i] not defined\n"
1770         if (!defined($opt{"PATCHCHECK_START[$i]"}));
1771     die "PATCHCHECK_TYPE[$i] not defined\n"
1772         if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
1773
1774     my $start = $opt{"PATCHCHECK_START[$i]"};
1775
1776     my $end = "HEAD";
1777     if (defined($opt{"PATCHCHECK_END[$i]"})) {
1778         $end = $opt{"PATCHCHECK_END[$i]"};
1779     }
1780
1781     # Get the true sha1's since we can use things like HEAD~3
1782     $start = get_sha1($start);
1783     $end = get_sha1($end);
1784
1785     my $type = $opt{"PATCHCHECK_TYPE[$i]"};
1786
1787     # Can't have a test without having a test to run
1788     if ($type eq "test" && !defined($run_test)) {
1789         $type = "boot";
1790     }
1791
1792     open (IN, "git log --pretty=oneline $end|") or
1793         dodie "could not get git list";
1794
1795     my @list;
1796
1797     while (<IN>) {
1798         chomp;
1799         $list[$#list+1] = $_;
1800         last if (/^$start/);
1801     }
1802     close(IN);
1803
1804     if ($list[$#list] !~ /^$start/) {
1805         fail "SHA1 $start not found";
1806     }
1807
1808     # go backwards in the list
1809     @list = reverse @list;
1810
1811     my $save_clean = $noclean;
1812
1813     $in_patchcheck = 1;
1814     foreach my $item (@list) {
1815         my $sha1 = $item;
1816         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
1817
1818         doprint "\nProcessing commit $item\n\n";
1819
1820         run_command "git checkout $sha1" or
1821             die "Failed to checkout $sha1";
1822
1823         # only clean on the first and last patch
1824         if ($item eq $list[0] ||
1825             $item eq $list[$#list]) {
1826             $noclean = $save_clean;
1827         } else {
1828             $noclean = 1;
1829         }
1830
1831         if (defined($minconfig)) {
1832             build "useconfig:$minconfig" or return 0;
1833         } else {
1834             # ?? no config to use?
1835             build "oldconfig" or return 0;
1836         }
1837
1838         check_buildlog $sha1 or return 0;
1839
1840         next if ($type eq "build");
1841
1842         get_grub_index;
1843         get_version;
1844         install;
1845
1846         my $failed = 0;
1847
1848         start_monitor;
1849         monitor or $failed = 1;
1850
1851         if (!$failed && $type ne "boot"){
1852             do_run_test or $failed = 1;
1853         }
1854         end_monitor;
1855         return 0 if ($failed);
1856
1857     }
1858     $in_patchcheck = 0;
1859     success $i;
1860
1861     return 1;
1862 }
1863
1864 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
1865
1866 if ($#ARGV == 0) {
1867     $ktest_config = $ARGV[0];
1868     if (! -f $ktest_config) {
1869         print "$ktest_config does not exist.\n";
1870         my $ans;
1871         for (;;) {
1872             print "Create it? [Y/n] ";
1873             $ans = <STDIN>;
1874             chomp $ans;
1875             if ($ans =~ /^\s*$/) {
1876                 $ans = "y";
1877             }
1878             last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
1879             print "Please answer either 'y' or 'n'.\n";
1880         }
1881         if ($ans !~ /^y$/i) {
1882             exit 0;
1883         }
1884     }
1885 } else {
1886     $ktest_config = "ktest.conf";
1887 }
1888
1889 if (! -f $ktest_config) {
1890     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
1891     print OUT << "EOF"
1892 # Generated by ktest.pl
1893 #
1894 # Define each test with TEST_START
1895 # The config options below it will override the defaults
1896 TEST_START
1897
1898 DEFAULTS
1899 EOF
1900 ;
1901     close(OUT);
1902 }
1903 read_config $ktest_config;
1904
1905 # Append any configs entered in manually to the config file.
1906 my @new_configs = keys %entered_configs;
1907 if ($#new_configs >= 0) {
1908     print "\nAppending entered in configs to $ktest_config\n";
1909     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
1910     foreach my $config (@new_configs) {
1911         print OUT "$config = $entered_configs{$config}\n";
1912         $opt{$config} = $entered_configs{$config};
1913     }
1914 }
1915
1916 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
1917     unlink $opt{"LOG_FILE"};
1918 }
1919
1920 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
1921
1922 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
1923
1924     if (!$i) {
1925         doprint "DEFAULT OPTIONS:\n";
1926     } else {
1927         doprint "\nTEST $i OPTIONS";
1928         if (defined($repeat_tests{$i})) {
1929             $repeat = $repeat_tests{$i};
1930             doprint " ITERATE $repeat";
1931         }
1932         doprint "\n";
1933     }
1934
1935     foreach my $option (sort keys %opt) {
1936
1937         if ($option =~ /\[(\d+)\]$/) {
1938             next if ($i != $1);
1939         } else {
1940             next if ($i);
1941         }
1942
1943         doprint "$option = $opt{$option}\n";
1944     }
1945 }
1946
1947 sub set_test_option {
1948     my ($name, $i) = @_;
1949
1950     my $option = "$name\[$i\]";
1951
1952     if (defined($opt{$option})) {
1953         return $opt{$option};
1954     }
1955
1956     foreach my $test (keys %repeat_tests) {
1957         if ($i >= $test &&
1958             $i < $test + $repeat_tests{$test}) {
1959             $option = "$name\[$test\]";
1960             if (defined($opt{$option})) {
1961                 return $opt{$option};
1962             }
1963         }
1964     }
1965
1966     if (defined($opt{$name})) {
1967         return $opt{$name};
1968     }
1969
1970     return undef;
1971 }
1972
1973 # First we need to do is the builds
1974 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
1975
1976     $iteration = $i;
1977
1978     my $makecmd = set_test_option("MAKE_CMD", $i);
1979
1980     $machine = set_test_option("MACHINE", $i);
1981     $ssh_user = set_test_option("SSH_USER", $i);
1982     $tmpdir = set_test_option("TMP_DIR", $i);
1983     $outputdir = set_test_option("OUTPUT_DIR", $i);
1984     $builddir = set_test_option("BUILD_DIR", $i);
1985     $test_type = set_test_option("TEST_TYPE", $i);
1986     $build_type = set_test_option("BUILD_TYPE", $i);
1987     $build_options = set_test_option("BUILD_OPTIONS", $i);
1988     $power_cycle = set_test_option("POWER_CYCLE", $i);
1989     $reboot = set_test_option("REBOOT", $i);
1990     $noclean = set_test_option("BUILD_NOCLEAN", $i);
1991     $minconfig = set_test_option("MIN_CONFIG", $i);
1992     $run_test = set_test_option("TEST", $i);
1993     $addconfig = set_test_option("ADD_CONFIG", $i);
1994     $reboot_type = set_test_option("REBOOT_TYPE", $i);
1995     $grub_menu = set_test_option("GRUB_MENU", $i);
1996     $post_install = set_test_option("POST_INSTALL", $i);
1997     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
1998     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
1999     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
2000     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
2001     $power_off = set_test_option("POWER_OFF", $i);
2002     $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
2003     $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
2004     $sleep_time = set_test_option("SLEEP_TIME", $i);
2005     $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
2006     $bisect_manual = set_test_option("BISECT_MANUAL", $i);
2007     $bisect_skip = set_test_option("BISECT_SKIP", $i);
2008     $store_failures = set_test_option("STORE_FAILURES", $i);
2009     $timeout = set_test_option("TIMEOUT", $i);
2010     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
2011     $console = set_test_option("CONSOLE", $i);
2012     $success_line = set_test_option("SUCCESS_LINE", $i);
2013     $stop_after_success = set_test_option("STOP_AFTER_SUCCESS", $i);
2014     $stop_after_failure = set_test_option("STOP_AFTER_FAILURE", $i);
2015     $stop_test_after = set_test_option("STOP_TEST_AFTER", $i);
2016     $build_target = set_test_option("BUILD_TARGET", $i);
2017     $ssh_exec = set_test_option("SSH_EXEC", $i);
2018     $scp_to_target = set_test_option("SCP_TO_TARGET", $i);
2019     $target_image = set_test_option("TARGET_IMAGE", $i);
2020     $localversion = set_test_option("LOCALVERSION", $i);
2021
2022     chdir $builddir || die "can't change directory to $builddir";
2023
2024     if (!-d $tmpdir) {
2025         mkpath($tmpdir) or
2026             die "can't create $tmpdir";
2027     }
2028
2029     $ENV{"SSH_USER"} = $ssh_user;
2030     $ENV{"MACHINE"} = $machine;
2031
2032     $target = "$ssh_user\@$machine";
2033
2034     $buildlog = "$tmpdir/buildlog-$machine";
2035     $dmesg = "$tmpdir/dmesg-$machine";
2036     $make = "$makecmd O=$outputdir";
2037     $output_config = "$outputdir/.config";
2038
2039     if ($reboot_type eq "grub") {
2040         dodie "GRUB_MENU not defined" if (!defined($grub_menu));
2041     } elsif (!defined($reboot_script)) {
2042         dodie "REBOOT_SCRIPT not defined"
2043     }
2044
2045     my $run_type = $build_type;
2046     if ($test_type eq "patchcheck") {
2047         $run_type = $opt{"PATCHCHECK_TYPE[$i]"};
2048     } elsif ($test_type eq "bisect") {
2049         $run_type = $opt{"BISECT_TYPE[$i]"};
2050     } elsif ($test_type eq "config_bisect") {
2051         $run_type = $opt{"CONFIG_BISECT_TYPE[$i]"};
2052     }
2053
2054     # mistake in config file?
2055     if (!defined($run_type)) {
2056         $run_type = "ERROR";
2057     }
2058
2059     doprint "\n\n";
2060     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type\n\n";
2061
2062     unlink $dmesg;
2063     unlink $buildlog;
2064
2065     if (!defined($minconfig)) {
2066         $minconfig = $addconfig;
2067
2068     } elsif (defined($addconfig)) {
2069         run_command "cat $addconfig $minconfig > $tmpdir/add_config" or
2070             dodie "Failed to create temp config";
2071         $minconfig = "$tmpdir/add_config";
2072     }
2073
2074     my $checkout = $opt{"CHECKOUT[$i]"};
2075     if (defined($checkout)) {
2076         run_command "git checkout $checkout" or
2077             die "failed to checkout $checkout";
2078     }
2079
2080     if ($test_type eq "bisect") {
2081         bisect $i;
2082         next;
2083     } elsif ($test_type eq "config_bisect") {
2084         config_bisect $i;
2085         next;
2086     } elsif ($test_type eq "patchcheck") {
2087         patchcheck $i;
2088         next;
2089     }
2090
2091     if ($build_type ne "nobuild") {
2092         build $build_type or next;
2093     }
2094
2095     if ($test_type ne "build") {
2096         get_grub_index;
2097         get_version;
2098         install;
2099
2100         my $failed = 0;
2101         start_monitor;
2102         monitor or $failed = 1;;
2103
2104         if (!$failed && $test_type ne "boot" && defined($run_test)) {
2105             do_run_test or $failed = 1;
2106         }
2107         end_monitor;
2108         next if ($failed);
2109     }
2110
2111     success $i;
2112 }
2113
2114 if ($opt{"POWEROFF_ON_SUCCESS"}) {
2115     halt;
2116 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
2117     reboot;
2118 }
2119
2120 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
2121
2122 exit 0;