tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copyright 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21 my %default;
  22
  23 #default opts
  24 $default{"NUM_TESTS"}           = 1;
  25 $default{"REBOOT_TYPE"}         = "grub";
  26 $default{"TEST_TYPE"}           = "test";
  27 $default{"BUILD_TYPE"}          = "randconfig";
  28 $default{"MAKE_CMD"}            = "make";
  29 $default{"TIMEOUT"}             = 120;
  30 $default{"TMP_DIR"}             = "/tmp/ktest/\${MACHINE}";
  31 $default{"SLEEP_TIME"}          = 60;   # sleep time between tests
  32 $default{"BUILD_NOCLEAN"}       = 0;
  33 $default{"REBOOT_ON_ERROR"}     = 0;
  34 $default{"POWEROFF_ON_ERROR"}   = 0;
  35 $default{"REBOOT_ON_SUCCESS"}   = 1;
  36 $default{"POWEROFF_ON_SUCCESS"} = 0;
  37 $default{"BUILD_OPTIONS"}       = "";
  38 $default{"BISECT_SLEEP_TIME"}   = 60;   # sleep time between bisects
  39 $default{"PATCHCHECK_SLEEP_TIME"} = 60; # sleep time between patch checks
  40 $default{"CLEAR_LOG"}           = 0;
  41 $default{"BISECT_MANUAL"}       = 0;
  42 $default{"BISECT_SKIP"}         = 1;
  43 $default{"SUCCESS_LINE"}        = "login:";
  44 $default{"DETECT_TRIPLE_FAULT"} = 1;
  45 $default{"BOOTED_TIMEOUT"}      = 1;
  46 $default{"DIE_ON_FAILURE"}      = 1;
  47 $default{"SSH_EXEC"}            = "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND";
  48 $default{"SCP_TO_TARGET"}       = "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE";
  49 $default{"REBOOT"}              = "ssh \$SSH_USER\@\$MACHINE reboot";
  50 $default{"STOP_AFTER_SUCCESS"}  = 10;
  51 $default{"STOP_AFTER_FAILURE"}  = 60;
  52 $default{"STOP_TEST_AFTER"}     = 600;
  53 $default{"LOCALVERSION"}        = "-test";
  54
  55 my $ktest_config;
  56 my $version;
  57 my $machine;
  58 my $ssh_user;
  59 my $tmpdir;
  60 my $builddir;
  61 my $outputdir;
  62 my $output_config;
  63 my $test_type;
  64 my $build_type;
  65 my $build_options;
  66 my $pre_build;
  67 my $post_build;
  68 my $pre_build_die;
  69 my $post_build_die;
  70 my $reboot_type;
  71 my $reboot_script;
  72 my $power_cycle;
  73 my $reboot;
  74 my $reboot_on_error;
  75 my $poweroff_on_error;
  76 my $die_on_failure;
  77 my $powercycle_after_reboot;
  78 my $poweroff_after_halt;
  79 my $ssh_exec;
  80 my $scp_to_target;
  81 my $power_off;
  82 my $grub_menu;
  83 my $grub_number;
  84 my $target;
  85 my $make;
  86 my $post_install;
  87 my $noclean;
  88 my $minconfig;
  89 my $start_minconfig;
  90 my $start_minconfig_defined;
  91 my $output_minconfig;
  92 my $ignore_config;
  93 my $addconfig;
  94 my $in_bisect = 0;
  95 my $bisect_bad = "";
  96 my $reverse_bisect;
  97 my $bisect_manual;
  98 my $bisect_skip;
  99 my $config_bisect_good;
 100 my $in_patchcheck = 0;
 101 my $run_test;
 102 my $redirect;
 103 my $buildlog;
 104 my $dmesg;
 105 my $monitor_fp;
 106 my $monitor_pid;
 107 my $monitor_cnt = 0;
 108 my $sleep_time;
 109 my $bisect_sleep_time;
 110 my $patchcheck_sleep_time;
 111 my $ignore_warnings;
 112 my $store_failures;
 113 my $test_name;
 114 my $timeout;
 115 my $booted_timeout;
 116 my $detect_triplefault;
 117 my $console;
 118 my $success_line;
 119 my $stop_after_success;
 120 my $stop_after_failure;
 121 my $stop_test_after;
 122 my $build_target;
 123 my $target_image;
 124 my $localversion;
 125 my $iteration = 0;
 126 my $successes = 0;
 127
 128 my %entered_configs;
 129 my %config_help;
 130 my %variable;
 131 my %force_config;
 132
 133 $config_help{"MACHINE"} = << "EOF"
 134  The machine hostname that you will test.
 135 EOF
 136     ;
 137 $config_help{"SSH_USER"} = << "EOF"
 138  The box is expected to have ssh on normal bootup, provide the user
 139   (most likely root, since you need privileged operations)
 140 EOF
 141     ;
 142 $config_help{"BUILD_DIR"} = << "EOF"
 143  The directory that contains the Linux source code (full path).
 144 EOF
 145     ;
 146 $config_help{"OUTPUT_DIR"} = << "EOF"
 147  The directory that the objects will be built (full path).
 148  (can not be same as BUILD_DIR)
 149 EOF
 150     ;
 151 $config_help{"BUILD_TARGET"} = << "EOF"
 152  The location of the compiled file to copy to the target.
 153  (relative to OUTPUT_DIR)
 154 EOF
 155     ;
 156 $config_help{"TARGET_IMAGE"} = << "EOF"
 157  The place to put your image on the test machine.
 158 EOF
 159     ;
 160 $config_help{"POWER_CYCLE"} = << "EOF"
 161  A script or command to reboot the box.
 162
 163  Here is a digital loggers power switch example
 164  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 165
 166  Here is an example to reboot a virtual box on the current host
 167  with the name "Guest".
 168  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 169 EOF
 170     ;
 171 $config_help{"CONSOLE"} = << "EOF"
 172  The script or command that reads the console
 173
 174   If you use ttywatch server, something like the following would work.
 175 CONSOLE = nc -d localhost 3001
 176
 177  For a virtual machine with guest name "Guest".
 178 CONSOLE =  virsh console Guest
 179 EOF
 180     ;
 181 $config_help{"LOCALVERSION"} = << "EOF"
 182  Required version ending to differentiate the test
 183  from other linux builds on the system.
 184 EOF
 185     ;
 186 $config_help{"REBOOT_TYPE"} = << "EOF"
 187  Way to reboot the box to the test kernel.
 188  Only valid options so far are "grub" and "script".
 189
 190  If you specify grub, it will assume grub version 1
 191  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 192  and select that target to reboot to the kernel. If this is not
 193  your setup, then specify "script" and have a command or script
 194  specified in REBOOT_SCRIPT to boot to the target.
 195
 196  The entry in /boot/grub/menu.lst must be entered in manually.
 197  The test will not modify that file.
 198 EOF
 199     ;
 200 $config_help{"GRUB_MENU"} = << "EOF"
 201  The grub title name for the test kernel to boot
 202  (Only mandatory if REBOOT_TYPE = grub)
 203
 204  Note, ktest.pl will not update the grub menu.lst, you need to
 205  manually add an option for the test. ktest.pl will search
 206  the grub menu.lst for this option to find what kernel to
 207  reboot into.
 208
 209  For example, if in the /boot/grub/menu.lst the test kernel title has:
 210  title Test Kernel
 211  kernel vmlinuz-test
 212  GRUB_MENU = Test Kernel
 213 EOF
 214     ;
 215 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 216  A script to reboot the target into the test kernel
 217  (Only mandatory if REBOOT_TYPE = script)
 218 EOF
 219     ;
 220
 221 sub read_yn {
 222     my ($prompt) = @_;
 223
 224     my $ans;
 225
 226     for (;;) {
 227         print "$prompt [Y/n] ";
 228         $ans = <STDIN>;
 229         chomp $ans;
 230         if ($ans =~ /^\s*$/) {
 231             $ans = "y";
 232         }
 233         last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
 234         print "Please answer either 'y' or 'n'.\n";
 235     }
 236     if ($ans !~ /^y$/i) {
 237         return 0;
 238     }
 239     return 1;
 240 }
 241
 242 sub get_ktest_config {
 243     my ($config) = @_;
 244
 245     return if (defined($opt{$config}));
 246
 247     if (defined($config_help{$config})) {
 248         print "\n";
 249         print $config_help{$config};
 250     }
 251
 252     for (;;) {
 253         print "$config = ";
 254         if (defined($default{$config})) {
 255             print "\[$default{$config}\] ";
 256         }
 257         $entered_configs{$config} = <STDIN>;
 258         $entered_configs{$config} =~ s/^\s*(.*\S)\s*$/$1/;
 259         if ($entered_configs{$config} =~ /^\s*$/) {
 260             if ($default{$config}) {
 261                 $entered_configs{$config} = $default{$config};
 262             } else {
 263                 print "Your answer can not be blank\n";
 264                 next;
 265             }
 266         }
 267         last;
 268     }
 269 }
 270
 271 sub get_ktest_configs {
 272     get_ktest_config("MACHINE");
 273     get_ktest_config("SSH_USER");
 274     get_ktest_config("BUILD_DIR");
 275     get_ktest_config("OUTPUT_DIR");
 276     get_ktest_config("BUILD_TARGET");
 277     get_ktest_config("TARGET_IMAGE");
 278     get_ktest_config("POWER_CYCLE");
 279     get_ktest_config("CONSOLE");
 280     get_ktest_config("LOCALVERSION");
 281
 282     my $rtype = $opt{"REBOOT_TYPE"};
 283
 284     if (!defined($rtype)) {
 285         if (!defined($opt{"GRUB_MENU"})) {
 286             get_ktest_config("REBOOT_TYPE");
 287             $rtype = $entered_configs{"REBOOT_TYPE"};
 288         } else {
 289             $rtype = "grub";
 290         }
 291     }
 292
 293     if ($rtype eq "grub") {
 294         get_ktest_config("GRUB_MENU");
 295     } else {
 296         get_ktest_config("REBOOT_SCRIPT");
 297     }
 298 }
 299
 300 sub process_variables {
 301     my ($value) = @_;
 302     my $retval = "";
 303
 304     # We want to check for '\', and it is just easier
 305     # to check the previous characet of '$' and not need
 306     # to worry if '$' is the first character. By adding
 307     # a space to $value, we can just check [^\\]\$ and
 308     # it will still work.
 309     $value = " $value";
 310
 311     while ($value =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 312         my $begin = $1;
 313         my $var = $2;
 314         my $end = $3;
 315         # append beginning of value to retval
 316         $retval = "$retval$begin";
 317         if (defined($variable{$var})) {
 318             $retval = "$retval$variable{$var}";
 319         } else {
 320             # put back the origin piece.
 321             $retval = "$retval\$\{$var\}";
 322         }
 323         $value = $end;
 324     }
 325     $retval = "$retval$value";
 326
 327     # remove the space added in the beginning
 328     $retval =~ s/ //;
 329
 330     return "$retval"
 331 }
 332
 333 sub set_value {
 334     my ($lvalue, $rvalue) = @_;
 335
 336     if (defined($opt{$lvalue})) {
 337         die "Error: Option $lvalue defined more than once!\n";
 338     }
 339     if ($rvalue =~ /^\s*$/) {
 340         delete $opt{$lvalue};
 341     } else {
 342         $rvalue = process_variables($rvalue);
 343         $opt{$lvalue} = $rvalue;
 344     }
 345 }
 346
 347 sub set_variable {
 348     my ($lvalue, $rvalue) = @_;
 349
 350     if ($rvalue =~ /^\s*$/) {
 351         delete $variable{$lvalue};
 352     } else {
 353         $rvalue = process_variables($rvalue);
 354         $variable{$lvalue} = $rvalue;
 355     }
 356 }
 357
 358 sub read_config {
 359     my ($config) = @_;
 360
 361     open(IN, $config) || die "can't read file $config";
 362
 363     my $name = $config;
 364     $name =~ s,.*/(.*),$1,;
 365
 366     my $test_num = 0;
 367     my $default = 1;
 368     my $repeat = 1;
 369     my $num_tests_set = 0;
 370     my $skip = 0;
 371     my $rest;
 372     my $test_case = 0;
 373
 374     while (<IN>) {
 375
 376         # ignore blank lines and comments
 377         next if (/^\s*$/ || /\s*\#/);
 378
 379         if (/^\s*TEST_START(.*)/) {
 380
 381             $rest = $1;
 382
 383             if ($num_tests_set) {
 384                 die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 385             }
 386
 387             my $old_test_num = $test_num;
 388             my $old_repeat = $repeat;
 389
 390             $test_num += $repeat;
 391             $default = 0;
 392             $repeat = 1;
 393
 394             if ($rest =~ /\s+SKIP(.*)/) {
 395                 $rest = $1;
 396                 $skip = 1;
 397             } else {
 398                 $test_case = 1;
 399                 $skip = 0;
 400             }
 401
 402             if ($rest =~ /\s+ITERATE\s+(\d+)(.*)$/) {
 403                 $repeat = $1;
 404                 $rest = $2;
 405                 $repeat_tests{"$test_num"} = $repeat;
 406             }
 407
 408             if ($rest =~ /\s+SKIP(.*)/) {
 409                 $rest = $1;
 410                 $skip = 1;
 411             }
 412
 413             if ($rest !~ /^\s*$/) {
 414                 die "$name: $.: Gargbage found after TEST_START\n$_";
 415             }
 416
 417             if ($skip) {
 418                 $test_num = $old_test_num;
 419                 $repeat = $old_repeat;
 420             }
 421
 422         } elsif (/^\s*DEFAULTS(.*)$/) {
 423             $default = 1;
 424
 425             $rest = $1;
 426
 427             if ($rest =~ /\s+SKIP(.*)/) {
 428                 $rest = $1;
 429                 $skip = 1;
 430             } else {
 431                 $skip = 0;
 432             }
 433
 434             if ($rest !~ /^\s*$/) {
 435                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 436             }
 437
 438         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 439
 440             next if ($skip);
 441
 442             my $lvalue = $1;
 443             my $rvalue = $2;
 444
 445             if (!$default &&
 446                 ($lvalue eq "NUM_TESTS" ||
 447                  $lvalue eq "LOG_FILE" ||
 448                  $lvalue eq "CLEAR_LOG")) {
 449                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 450             }
 451
 452             if ($lvalue eq "NUM_TESTS") {
 453                 if ($test_num) {
 454                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 455                 }
 456                 if (!$default) {
 457                     die "$name: $.: NUM_TESTS must be set in default section\n";
 458                 }
 459                 $num_tests_set = 1;
 460             }
 461
 462             if ($default || $lvalue =~ /\[\d+\]$/) {
 463                 set_value($lvalue, $rvalue);
 464             } else {
 465                 my $val = "$lvalue\[$test_num\]";
 466                 set_value($val, $rvalue);
 467
 468                 if ($repeat > 1) {
 469                     $repeats{$val} = $repeat;
 470                 }
 471             }
 472         } elsif (/^\s*([A-Z_\[\]\d]+)\s*:=\s*(.*?)\s*$/) {
 473             next if ($skip);
 474
 475             my $lvalue = $1;
 476             my $rvalue = $2;
 477
 478             # process config variables.
 479             # Config variables are only active while reading the
 480             # config and can be defined anywhere. They also ignore
 481             # TEST_START and DEFAULTS, but are skipped if they are in
 482             # on of these sections that have SKIP defined.
 483             # The save variable can be
 484             # defined multiple times and the new one simply overrides
 485             # the prevous one.
 486             set_variable($lvalue, $rvalue);
 487
 488         } else {
 489             die "$name: $.: Garbage found in config\n$_";
 490         }
 491     }
 492
 493     close(IN);
 494
 495     if ($test_num) {
 496         $test_num += $repeat - 1;
 497         $opt{"NUM_TESTS"} = $test_num;
 498     }
 499
 500     # make sure we have all mandatory configs
 501     get_ktest_configs;
 502
 503     # was a test specified?
 504     if (!$test_case) {
 505         print "No test case specified.\n";
 506         print "What test case would you like to run?\n";
 507         my $ans = <STDIN>;
 508         chomp $ans;
 509         $default{"TEST_TYPE"} = $ans;
 510     }
 511
 512     # set any defaults
 513
 514     foreach my $default (keys %default) {
 515         if (!defined($opt{$default})) {
 516             $opt{$default} = $default{$default};
 517         }
 518     }
 519 }
 520
 521 sub __eval_option {
 522     my ($option, $i) = @_;
 523
 524     # Add space to evaluate the character before $
 525     $option = " $option";
 526     my $retval = "";
 527
 528     while ($option =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 529         my $start = $1;
 530         my $var = $2;
 531         my $end = $3;
 532
 533         # Append beginning of line
 534         $retval = "$retval$start";
 535
 536         # If the iteration option OPT[$i] exists, then use that.
 537         # otherwise see if the default OPT (without [$i]) exists.
 538
 539         my $o = "$var\[$i\]";
 540
 541         if (defined($opt{$o})) {
 542             $o = $opt{$o};
 543             $retval = "$retval$o";
 544         } elsif (defined($opt{$var})) {
 545             $o = $opt{$var};
 546             $retval = "$retval$o";
 547         } else {
 548             $retval = "$retval\$\{$var\}";
 549         }
 550
 551         $option = $end;
 552     }
 553
 554     $retval = "$retval$option";
 555
 556     $retval =~ s/^ //;
 557
 558     return $retval;
 559 }
 560
 561 sub eval_option {
 562     my ($option, $i) = @_;
 563
 564     my $prev = "";
 565
 566     # Since an option can evaluate to another option,
 567     # keep iterating until we do not evaluate any more
 568     # options.
 569     my $r = 0;
 570     while ($prev ne $option) {
 571         # Check for recursive evaluations.
 572         # 100 deep should be more than enough.
 573         if ($r++ > 100) {
 574             die "Over 100 evaluations accurred with $option\n" .
 575                 "Check for recursive variables\n";
 576         }
 577         $prev = $option;
 578         $option = __eval_option($option, $i);
 579     }
 580
 581     return $option;
 582 }
 583
 584 sub _logit {
 585     if (defined($opt{"LOG_FILE"})) {
 586         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
 587         print OUT @_;
 588         close(OUT);
 589     }
 590 }
 591
 592 sub logit {
 593     if (defined($opt{"LOG_FILE"})) {
 594         _logit @_;
 595     } else {
 596         print @_;
 597     }
 598 }
 599
 600 sub doprint {
 601     print @_;
 602     _logit @_;
 603 }
 604
 605 sub run_command;
 606
 607 sub reboot {
 608     # try to reboot normally
 609     if (run_command $reboot) {
 610         if (defined($powercycle_after_reboot)) {
 611             sleep $powercycle_after_reboot;
 612             run_command "$power_cycle";
 613         }
 614     } else {
 615         # nope? power cycle it.
 616         run_command "$power_cycle";
 617     }
 618 }
 619
 620 sub do_not_reboot {
 621     my $i = $iteration;
 622
 623     return $test_type eq "build" ||
 624         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
 625         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
 626 }
 627
 628 sub dodie {
 629     doprint "CRITICAL FAILURE... ", @_, "\n";
 630
 631     my $i = $iteration;
 632
 633     if ($reboot_on_error && !do_not_reboot) {
 634
 635         doprint "REBOOTING\n";
 636         reboot;
 637
 638     } elsif ($poweroff_on_error && defined($power_off)) {
 639         doprint "POWERING OFF\n";
 640         `$power_off`;
 641     }
 642
 643     if (defined($opt{"LOG_FILE"})) {
 644         print " See $opt{LOG_FILE} for more info.\n";
 645     }
 646
 647     die @_, "\n";
 648 }
 649
 650 sub open_console {
 651     my ($fp) = @_;
 652
 653     my $flags;
 654
 655     my $pid = open($fp, "$console|") or
 656         dodie "Can't open console $console";
 657
 658     $flags = fcntl($fp, F_GETFL, 0) or
 659         dodie "Can't get flags for the socket: $!";
 660     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 661         dodie "Can't set flags for the socket: $!";
 662
 663     return $pid;
 664 }
 665
 666 sub close_console {
 667     my ($fp, $pid) = @_;
 668
 669     doprint "kill child process $pid\n";
 670     kill 2, $pid;
 671
 672     print "closing!\n";
 673     close($fp);
 674 }
 675
 676 sub start_monitor {
 677     if ($monitor_cnt++) {
 678         return;
 679     }
 680     $monitor_fp = \*MONFD;
 681     $monitor_pid = open_console $monitor_fp;
 682
 683     return;
 684
 685     open(MONFD, "Stop perl from warning about single use of MONFD");
 686 }
 687
 688 sub end_monitor {
 689     if (--$monitor_cnt) {
 690         return;
 691     }
 692     close_console($monitor_fp, $monitor_pid);
 693 }
 694
 695 sub wait_for_monitor {
 696     my ($time) = @_;
 697     my $line;
 698
 699     doprint "** Wait for monitor to settle down **\n";
 700
 701     # read the monitor and wait for the system to calm down
 702     do {
 703         $line = wait_for_input($monitor_fp, $time);
 704         print "$line" if (defined($line));
 705     } while (defined($line));
 706     print "** Monitor flushed **\n";
 707 }
 708
 709 sub fail {
 710
 711         if ($die_on_failure) {
 712                 dodie @_;
 713         }
 714
 715         doprint "FAILED\n";
 716
 717         my $i = $iteration;
 718
 719         # no need to reboot for just building.
 720         if (!do_not_reboot) {
 721             doprint "REBOOTING\n";
 722             reboot;
 723             start_monitor;
 724             wait_for_monitor $sleep_time;
 725             end_monitor;
 726         }
 727
 728         my $name = "";
 729
 730         if (defined($test_name)) {
 731             $name = " ($test_name)";
 732         }
 733
 734         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 735         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 736         doprint "KTEST RESULT: TEST $i$name Failed: ", @_, "\n";
 737         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 738         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 739
 740         return 1 if (!defined($store_failures));
 741
 742         my @t = localtime;
 743         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
 744                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
 745
 746         my $type = $build_type;
 747         if ($type =~ /useconfig/) {
 748             $type = "useconfig";
 749         }
 750
 751         my $dir = "$machine-$test_type-$type-fail-$date";
 752         my $faildir = "$store_failures/$dir";
 753
 754         if (!-d $faildir) {
 755             mkpath($faildir) or
 756                 die "can't create $faildir";
 757         }
 758         if (-f "$output_config") {
 759             cp "$output_config", "$faildir/config" or
 760                 die "failed to copy .config";
 761         }
 762         if (-f $buildlog) {
 763             cp $buildlog, "$faildir/buildlog" or
 764                 die "failed to move $buildlog";
 765         }
 766         if (-f $dmesg) {
 767             cp $dmesg, "$faildir/dmesg" or
 768                 die "failed to move $dmesg";
 769         }
 770
 771         doprint "*** Saved info to $faildir ***\n";
 772
 773         return 1;
 774 }
 775
 776 sub run_command {
 777     my ($command) = @_;
 778     my $dolog = 0;
 779     my $dord = 0;
 780     my $pid;
 781
 782     $command =~ s/\$SSH_USER/$ssh_user/g;
 783     $command =~ s/\$MACHINE/$machine/g;
 784
 785     doprint("$command ... ");
 786
 787     $pid = open(CMD, "$command 2>&1 |") or
 788         (fail "unable to exec $command" and return 0);
 789
 790     if (defined($opt{"LOG_FILE"})) {
 791         open(LOG, ">>$opt{LOG_FILE}") or
 792             dodie "failed to write to log";
 793         $dolog = 1;
 794     }
 795
 796     if (defined($redirect)) {
 797         open (RD, ">$redirect") or
 798             dodie "failed to write to redirect $redirect";
 799         $dord = 1;
 800     }
 801
 802     while (<CMD>) {
 803         print LOG if ($dolog);
 804         print RD  if ($dord);
 805     }
 806
 807     waitpid($pid, 0);
 808     my $failed = $?;
 809
 810     close(CMD);
 811     close(LOG) if ($dolog);
 812     close(RD)  if ($dord);
 813
 814     if ($failed) {
 815         doprint "FAILED!\n";
 816     } else {
 817         doprint "SUCCESS\n";
 818     }
 819
 820     return !$failed;
 821 }
 822
 823 sub run_ssh {
 824     my ($cmd) = @_;
 825     my $cp_exec = $ssh_exec;
 826
 827     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
 828     return run_command "$cp_exec";
 829 }
 830
 831 sub run_scp {
 832     my ($src, $dst) = @_;
 833     my $cp_scp = $scp_to_target;
 834
 835     $cp_scp =~ s/\$SRC_FILE/$src/g;
 836     $cp_scp =~ s/\$DST_FILE/$dst/g;
 837
 838     return run_command "$cp_scp";
 839 }
 840
 841 sub get_grub_index {
 842
 843     if ($reboot_type ne "grub") {
 844         return;
 845     }
 846     return if (defined($grub_number));
 847
 848     doprint "Find grub menu ... ";
 849     $grub_number = -1;
 850
 851     my $ssh_grub = $ssh_exec;
 852     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
 853
 854     open(IN, "$ssh_grub |")
 855         or die "unable to get menu.lst";
 856
 857     while (<IN>) {
 858         if (/^\s*title\s+$grub_menu\s*$/) {
 859             $grub_number++;
 860             last;
 861         } elsif (/^\s*title\s/) {
 862             $grub_number++;
 863         }
 864     }
 865     close(IN);
 866
 867     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
 868         if ($grub_number < 0);
 869     doprint "$grub_number\n";
 870 }
 871
 872 sub wait_for_input
 873 {
 874     my ($fp, $time) = @_;
 875     my $rin;
 876     my $ready;
 877     my $line;
 878     my $ch;
 879
 880     if (!defined($time)) {
 881         $time = $timeout;
 882     }
 883
 884     $rin = '';
 885     vec($rin, fileno($fp), 1) = 1;
 886     $ready = select($rin, undef, undef, $time);
 887
 888     $line = "";
 889
 890     # try to read one char at a time
 891     while (sysread $fp, $ch, 1) {
 892         $line .= $ch;
 893         last if ($ch eq "\n");
 894     }
 895
 896     if (!length($line)) {
 897         return undef;
 898     }
 899
 900     return $line;
 901 }
 902
 903 sub reboot_to {
 904     if ($reboot_type eq "grub") {
 905         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch && reboot)'";
 906         return;
 907     }
 908
 909     run_command "$reboot_script";
 910 }
 911
 912 sub get_sha1 {
 913     my ($commit) = @_;
 914
 915     doprint "git rev-list --max-count=1 $commit ... ";
 916     my $sha1 = `git rev-list --max-count=1 $commit`;
 917     my $ret = $?;
 918
 919     logit $sha1;
 920
 921     if ($ret) {
 922         doprint "FAILED\n";
 923         dodie "Failed to get git $commit";
 924     }
 925
 926     print "SUCCESS\n";
 927
 928     chomp $sha1;
 929
 930     return $sha1;
 931 }
 932
 933 sub monitor {
 934     my $booted = 0;
 935     my $bug = 0;
 936     my $skip_call_trace = 0;
 937     my $loops;
 938
 939     wait_for_monitor 5;
 940
 941     my $line;
 942     my $full_line = "";
 943
 944     open(DMESG, "> $dmesg") or
 945         die "unable to write to $dmesg";
 946
 947     reboot_to;
 948
 949     my $success_start;
 950     my $failure_start;
 951     my $monitor_start = time;
 952     my $done = 0;
 953     my $version_found = 0;
 954
 955     while (!$done) {
 956
 957         if ($bug && defined($stop_after_failure) &&
 958             $stop_after_failure >= 0) {
 959             my $time = $stop_after_failure - (time - $failure_start);
 960             $line = wait_for_input($monitor_fp, $time);
 961             if (!defined($line)) {
 962                 doprint "bug timed out after $booted_timeout seconds\n";
 963                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
 964                 last;
 965             }
 966         } elsif ($booted) {
 967             $line = wait_for_input($monitor_fp, $booted_timeout);
 968             if (!defined($line)) {
 969                 my $s = $booted_timeout == 1 ? "" : "s";
 970                 doprint "Successful boot found: break after $booted_timeout second$s\n";
 971                 last;
 972             }
 973         } else {
 974             $line = wait_for_input($monitor_fp);
 975             if (!defined($line)) {
 976                 my $s = $timeout == 1 ? "" : "s";
 977                 doprint "Timed out after $timeout second$s\n";
 978                 last;
 979             }
 980         }
 981
 982         doprint $line;
 983         print DMESG $line;
 984
 985         # we are not guaranteed to get a full line
 986         $full_line .= $line;
 987
 988         if ($full_line =~ /$success_line/) {
 989             $booted = 1;
 990             $success_start = time;
 991         }
 992
 993         if ($booted && defined($stop_after_success) &&
 994             $stop_after_success >= 0) {
 995             my $now = time;
 996             if ($now - $success_start >= $stop_after_success) {
 997                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
 998                 last;
 999             }
1000         }
1001
1002         if ($full_line =~ /\[ backtrace testing \]/) {
1003             $skip_call_trace = 1;
1004         }
1005
1006         if ($full_line =~ /call trace:/i) {
1007             if (!$bug && !$skip_call_trace) {
1008                 $bug = 1;
1009                 $failure_start = time;
1010             }
1011         }
1012
1013         if ($bug && defined($stop_after_failure) &&
1014             $stop_after_failure >= 0) {
1015             my $now = time;
1016             if ($now - $failure_start >= $stop_after_failure) {
1017                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
1018                 last;
1019             }
1020         }
1021
1022         if ($full_line =~ /\[ end of backtrace testing \]/) {
1023             $skip_call_trace = 0;
1024         }
1025
1026         if ($full_line =~ /Kernel panic -/) {
1027             $failure_start = time;
1028             $bug = 1;
1029         }
1030
1031         # Detect triple faults by testing the banner
1032         if ($full_line =~ /\bLinux version (\S+).*\n/) {
1033             if ($1 eq $version) {
1034                 $version_found = 1;
1035             } elsif ($version_found && $detect_triplefault) {
1036                 # We already booted into the kernel we are testing,
1037                 # but now we booted into another kernel?
1038                 # Consider this a triple fault.
1039                 doprint "Aleady booted in Linux kernel $version, but now\n";
1040                 doprint "we booted into Linux kernel $1.\n";
1041                 doprint "Assuming that this is a triple fault.\n";
1042                 doprint "To disable this: set DETECT_TRIPLE_FAULT to 0\n";
1043                 last;
1044             }
1045         }
1046
1047         if ($line =~ /\n/) {
1048             $full_line = "";
1049         }
1050
1051         if ($stop_test_after > 0 && !$booted && !$bug) {
1052             if (time - $monitor_start > $stop_test_after) {
1053                 doprint "STOP_TEST_AFTER ($stop_test_after seconds) timed out\n";
1054                 $done = 1;
1055             }
1056         }
1057     }
1058
1059     close(DMESG);
1060
1061     if ($bug) {
1062         return 0 if ($in_bisect);
1063         fail "failed - got a bug report" and return 0;
1064     }
1065
1066     if (!$booted) {
1067         return 0 if ($in_bisect);
1068         fail "failed - never got a boot prompt." and return 0;
1069     }
1070
1071     return 1;
1072 }
1073
1074 sub do_post_install {
1075
1076     return if (!defined($post_install));
1077
1078     my $cp_post_install = $post_install;
1079     $cp_post_install =~ s/\$KERNEL_VERSION/$version/g;
1080     run_command "$cp_post_install" or
1081         dodie "Failed to run post install";
1082 }
1083
1084 sub install {
1085
1086     run_scp "$outputdir/$build_target", "$target_image" or
1087         dodie "failed to copy image";
1088
1089     my $install_mods = 0;
1090
1091     # should we process modules?
1092     $install_mods = 0;
1093     open(IN, "$output_config") or dodie("Can't read config file");
1094     while (<IN>) {
1095         if (/CONFIG_MODULES(=y)?/) {
1096             $install_mods = 1 if (defined($1));
1097             last;
1098         }
1099     }
1100     close(IN);
1101
1102     if (!$install_mods) {
1103         do_post_install;
1104         doprint "No modules needed\n";
1105         return;
1106     }
1107
1108     run_command "$make INSTALL_MOD_PATH=$tmpdir modules_install" or
1109         dodie "Failed to install modules";
1110
1111     my $modlib = "/lib/modules/$version";
1112     my $modtar = "ktest-mods.tar.bz2";
1113
1114     run_ssh "rm -rf $modlib" or
1115         dodie "failed to remove old mods: $modlib";
1116
1117     # would be nice if scp -r did not follow symbolic links
1118     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
1119         dodie "making tarball";
1120
1121     run_scp "$tmpdir/$modtar", "/tmp" or
1122         dodie "failed to copy modules";
1123
1124     unlink "$tmpdir/$modtar";
1125
1126     run_ssh "'(cd / && tar xjf /tmp/$modtar)'" or
1127         dodie "failed to tar modules";
1128
1129     run_ssh "rm -f /tmp/$modtar";
1130
1131     do_post_install;
1132 }
1133
1134 sub get_version {
1135     # get the release name
1136     doprint "$make kernelrelease ... ";
1137     $version = `$make kernelrelease | tail -1`;
1138     chomp($version);
1139     doprint "$version\n";
1140 }
1141
1142 sub start_monitor_and_boot {
1143     get_grub_index;
1144     get_version;
1145     install;
1146
1147     start_monitor;
1148     return monitor;
1149 }
1150
1151 sub check_buildlog {
1152     my ($patch) = @_;
1153
1154     my @files = `git show $patch | diffstat -l`;
1155
1156     open(IN, "git show $patch |") or
1157         dodie "failed to show $patch";
1158     while (<IN>) {
1159         if (m,^--- a/(.*),) {
1160             chomp $1;
1161             $files[$#files] = $1;
1162         }
1163     }
1164     close(IN);
1165
1166     open(IN, $buildlog) or dodie "Can't open $buildlog";
1167     while (<IN>) {
1168         if (/^\s*(.*?):.*(warning|error)/) {
1169             my $err = $1;
1170             foreach my $file (@files) {
1171                 my $fullpath = "$builddir/$file";
1172                 if ($file eq $err || $fullpath eq $err) {
1173                     fail "$file built with warnings" and return 0;
1174                 }
1175             }
1176         }
1177     }
1178     close(IN);
1179
1180     return 1;
1181 }
1182
1183 sub apply_min_config {
1184     my $outconfig = "$output_config.new";
1185
1186     # Read the config file and remove anything that
1187     # is in the force_config hash (from minconfig and others)
1188     # then add the force config back.
1189
1190     doprint "Applying minimum configurations into $output_config.new\n";
1191
1192     open (OUT, ">$outconfig") or
1193         dodie "Can't create $outconfig";
1194
1195     if (-f $output_config) {
1196         open (IN, $output_config) or
1197             dodie "Failed to open $output_config";
1198         while (<IN>) {
1199             if (/^(# )?(CONFIG_[^\s=]*)/) {
1200                 next if (defined($force_config{$2}));
1201             }
1202             print OUT;
1203         }
1204         close IN;
1205     }
1206     foreach my $config (keys %force_config) {
1207         print OUT "$force_config{$config}\n";
1208     }
1209     close OUT;
1210
1211     run_command "mv $outconfig $output_config";
1212 }
1213
1214 sub make_oldconfig {
1215
1216     my @force_list = keys %force_config;
1217
1218     if ($#force_list >= 0) {
1219         apply_min_config;
1220     }
1221
1222     if (!run_command "$make oldnoconfig") {
1223         # Perhaps oldnoconfig doesn't exist in this version of the kernel
1224         # try a yes '' | oldconfig
1225         doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
1226         run_command "yes '' | $make oldconfig" or
1227             dodie "failed make config oldconfig";
1228     }
1229 }
1230
1231 # read a config file and use this to force new configs.
1232 sub load_force_config {
1233     my ($config) = @_;
1234
1235     open(IN, $config) or
1236         dodie "failed to read $config";
1237     while (<IN>) {
1238         chomp;
1239         if (/^(CONFIG[^\s=]*)(\s*=.*)/) {
1240             $force_config{$1} = $_;
1241         } elsif (/^# (CONFIG_\S*) is not set/) {
1242             $force_config{$1} = $_;
1243         }
1244     }
1245     close IN;
1246 }
1247
1248 sub build {
1249     my ($type) = @_;
1250
1251     unlink $buildlog;
1252
1253     if (defined($pre_build)) {
1254         my $ret = run_command $pre_build;
1255         if (!$ret && defined($pre_build_die) &&
1256             $pre_build_die) {
1257             dodie "failed to pre_build\n";
1258         }
1259     }
1260
1261     if ($type =~ /^useconfig:(.*)/) {
1262         run_command "cp $1 $output_config" or
1263             dodie "could not copy $1 to .config";
1264
1265         $type = "oldconfig";
1266     }
1267
1268     # old config can ask questions
1269     if ($type eq "oldconfig") {
1270         $type = "oldnoconfig";
1271
1272         # allow for empty configs
1273         run_command "touch $output_config";
1274
1275         run_command "mv $output_config $outputdir/config_temp" or
1276             dodie "moving .config";
1277
1278         if (!$noclean && !run_command "$make mrproper") {
1279             dodie "make mrproper";
1280         }
1281
1282         run_command "mv $outputdir/config_temp $output_config" or
1283             dodie "moving config_temp";
1284
1285     } elsif (!$noclean) {
1286         unlink "$output_config";
1287         run_command "$make mrproper" or
1288             dodie "make mrproper";
1289     }
1290
1291     # add something to distinguish this build
1292     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
1293     print OUT "$localversion\n";
1294     close(OUT);
1295
1296     if (defined($minconfig)) {
1297         load_force_config($minconfig);
1298     }
1299
1300     if ($type ne "oldnoconfig") {
1301         run_command "$make $type" or
1302             dodie "failed make config";
1303     }
1304     # Run old config regardless, to enforce min configurations
1305     make_oldconfig;
1306
1307     $redirect = "$buildlog";
1308     my $build_ret = run_command "$make $build_options";
1309     undef $redirect;
1310
1311     if (defined($post_build)) {
1312         my $ret = run_command $post_build;
1313         if (!$ret && defined($post_build_die) &&
1314             $post_build_die) {
1315             dodie "failed to post_build\n";
1316         }
1317     }
1318
1319     if (!$build_ret) {
1320         # bisect may need this to pass
1321         return 0 if ($in_bisect);
1322         fail "failed build" and return 0;
1323     }
1324
1325     return 1;
1326 }
1327
1328 sub halt {
1329     if (!run_ssh "halt" or defined($power_off)) {
1330         if (defined($poweroff_after_halt)) {
1331             sleep $poweroff_after_halt;
1332             run_command "$power_off";
1333         }
1334     } else {
1335         # nope? the zap it!
1336         run_command "$power_off";
1337     }
1338 }
1339
1340 sub success {
1341     my ($i) = @_;
1342
1343     $successes++;
1344
1345     my $name = "";
1346
1347     if (defined($test_name)) {
1348         $name = " ($test_name)";
1349     }
1350
1351     doprint "\n\n*******************************************\n";
1352     doprint     "*******************************************\n";
1353     doprint     "KTEST RESULT: TEST $i$name SUCCESS!!!!         **\n";
1354     doprint     "*******************************************\n";
1355     doprint     "*******************************************\n";
1356
1357     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
1358         doprint "Reboot and wait $sleep_time seconds\n";
1359         reboot;
1360         start_monitor;
1361         wait_for_monitor $sleep_time;
1362         end_monitor;
1363     }
1364 }
1365
1366 sub answer_bisect {
1367     for (;;) {
1368         doprint "Pass or fail? [p/f]";
1369         my $ans = <STDIN>;
1370         chomp $ans;
1371         if ($ans eq "p" || $ans eq "P") {
1372             return 1;
1373         } elsif ($ans eq "f" || $ans eq "F") {
1374             return 0;
1375         } else {
1376             print "Please answer 'P' or 'F'\n";
1377         }
1378     }
1379 }
1380
1381 sub child_run_test {
1382     my $failed = 0;
1383
1384     # child should have no power
1385     $reboot_on_error = 0;
1386     $poweroff_on_error = 0;
1387     $die_on_failure = 1;
1388
1389     run_command $run_test or $failed = 1;
1390     exit $failed;
1391 }
1392
1393 my $child_done;
1394
1395 sub child_finished {
1396     $child_done = 1;
1397 }
1398
1399 sub do_run_test {
1400     my $child_pid;
1401     my $child_exit;
1402     my $line;
1403     my $full_line;
1404     my $bug = 0;
1405
1406     wait_for_monitor 1;
1407
1408     doprint "run test $run_test\n";
1409
1410     $child_done = 0;
1411
1412     $SIG{CHLD} = qw(child_finished);
1413
1414     $child_pid = fork;
1415
1416     child_run_test if (!$child_pid);
1417
1418     $full_line = "";
1419
1420     do {
1421         $line = wait_for_input($monitor_fp, 1);
1422         if (defined($line)) {
1423
1424             # we are not guaranteed to get a full line
1425             $full_line .= $line;
1426             doprint $line;
1427
1428             if ($full_line =~ /call trace:/i) {
1429                 $bug = 1;
1430             }
1431
1432             if ($full_line =~ /Kernel panic -/) {
1433                 $bug = 1;
1434             }
1435
1436             if ($line =~ /\n/) {
1437                 $full_line = "";
1438             }
1439         }
1440     } while (!$child_done && !$bug);
1441
1442     if ($bug) {
1443         my $failure_start = time;
1444         my $now;
1445         do {
1446             $line = wait_for_input($monitor_fp, 1);
1447             if (defined($line)) {
1448                 doprint $line;
1449             }
1450             $now = time;
1451             if ($now - $failure_start >= $stop_after_failure) {
1452                 last;
1453             }
1454         } while (defined($line));
1455
1456         doprint "Detected kernel crash!\n";
1457         # kill the child with extreme prejudice
1458         kill 9, $child_pid;
1459     }
1460
1461     waitpid $child_pid, 0;
1462     $child_exit = $?;
1463
1464     if ($bug || $child_exit) {
1465         return 0 if $in_bisect;
1466         fail "test failed" and return 0;
1467     }
1468     return 1;
1469 }
1470
1471 sub run_git_bisect {
1472     my ($command) = @_;
1473
1474     doprint "$command ... ";
1475
1476     my $output = `$command 2>&1`;
1477     my $ret = $?;
1478
1479     logit $output;
1480
1481     if ($ret) {
1482         doprint "FAILED\n";
1483         dodie "Failed to git bisect";
1484     }
1485
1486     doprint "SUCCESS\n";
1487     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
1488         doprint "$1 [$2]\n";
1489     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
1490         $bisect_bad = $1;
1491         doprint "Found bad commit... $1\n";
1492         return 0;
1493     } else {
1494         # we already logged it, just print it now.
1495         print $output;
1496     }
1497
1498     return 1;
1499 }
1500
1501 sub bisect_reboot {
1502     doprint "Reboot and sleep $bisect_sleep_time seconds\n";
1503     reboot;
1504     start_monitor;
1505     wait_for_monitor $bisect_sleep_time;
1506     end_monitor;
1507 }
1508
1509 # returns 1 on success, 0 on failure, -1 on skip
1510 sub run_bisect_test {
1511     my ($type, $buildtype) = @_;
1512
1513     my $failed = 0;
1514     my $result;
1515     my $output;
1516     my $ret;
1517
1518     $in_bisect = 1;
1519
1520     build $buildtype or $failed = 1;
1521
1522     if ($type ne "build") {
1523         if ($failed && $bisect_skip) {
1524             $in_bisect = 0;
1525             return -1;
1526         }
1527         dodie "Failed on build" if $failed;
1528
1529         # Now boot the box
1530         start_monitor_and_boot or $failed = 1;
1531
1532         if ($type ne "boot") {
1533             if ($failed && $bisect_skip) {
1534                 end_monitor;
1535                 bisect_reboot;
1536                 $in_bisect = 0;
1537                 return -1;
1538             }
1539             dodie "Failed on boot" if $failed;
1540
1541             do_run_test or $failed = 1;
1542         }
1543         end_monitor;
1544     }
1545
1546     if ($failed) {
1547         $result = 0;
1548     } else {
1549         $result = 1;
1550     }
1551
1552     # reboot the box to a kernel we can ssh to
1553     if ($type ne "build") {
1554         bisect_reboot;
1555     }
1556     $in_bisect = 0;
1557
1558     return $result;
1559 }
1560
1561 sub run_bisect {
1562     my ($type) = @_;
1563     my $buildtype = "oldconfig";
1564
1565     # We should have a minconfig to use?
1566     if (defined($minconfig)) {
1567         $buildtype = "useconfig:$minconfig";
1568     }
1569
1570     my $ret = run_bisect_test $type, $buildtype;
1571
1572     if ($bisect_manual) {
1573         $ret = answer_bisect;
1574     }
1575
1576     # Are we looking for where it worked, not failed?
1577     if ($reverse_bisect) {
1578         $ret = !$ret;
1579     }
1580
1581     if ($ret > 0) {
1582         return "good";
1583     } elsif ($ret == 0) {
1584         return  "bad";
1585     } elsif ($bisect_skip) {
1586         doprint "HIT A BAD COMMIT ... SKIPPING\n";
1587         return "skip";
1588     }
1589 }
1590
1591 sub bisect {
1592     my ($i) = @_;
1593
1594     my $result;
1595
1596     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
1597     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
1598     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
1599
1600     my $good = $opt{"BISECT_GOOD[$i]"};
1601     my $bad = $opt{"BISECT_BAD[$i]"};
1602     my $type = $opt{"BISECT_TYPE[$i]"};
1603     my $start = $opt{"BISECT_START[$i]"};
1604     my $replay = $opt{"BISECT_REPLAY[$i]"};
1605     my $start_files = $opt{"BISECT_FILES[$i]"};
1606
1607     if (defined($start_files)) {
1608         $start_files = " -- " . $start_files;
1609     } else {
1610         $start_files = "";
1611     }
1612
1613     # convert to true sha1's
1614     $good = get_sha1($good);
1615     $bad = get_sha1($bad);
1616
1617     if (defined($opt{"BISECT_REVERSE[$i]"}) &&
1618         $opt{"BISECT_REVERSE[$i]"} == 1) {
1619         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
1620         $reverse_bisect = 1;
1621     } else {
1622         $reverse_bisect = 0;
1623     }
1624
1625     # Can't have a test without having a test to run
1626     if ($type eq "test" && !defined($run_test)) {
1627         $type = "boot";
1628     }
1629
1630     my $check = $opt{"BISECT_CHECK[$i]"};
1631     if (defined($check) && $check ne "0") {
1632
1633         # get current HEAD
1634         my $head = get_sha1("HEAD");
1635
1636         if ($check ne "good") {
1637             doprint "TESTING BISECT BAD [$bad]\n";
1638             run_command "git checkout $bad" or
1639                 die "Failed to checkout $bad";
1640
1641             $result = run_bisect $type;
1642
1643             if ($result ne "bad") {
1644                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
1645             }
1646         }
1647
1648         if ($check ne "bad") {
1649             doprint "TESTING BISECT GOOD [$good]\n";
1650             run_command "git checkout $good" or
1651                 die "Failed to checkout $good";
1652
1653             $result = run_bisect $type;
1654
1655             if ($result ne "good") {
1656                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
1657             }
1658         }
1659
1660         # checkout where we started
1661         run_command "git checkout $head" or
1662             die "Failed to checkout $head";
1663     }
1664
1665     run_command "git bisect start$start_files" or
1666         dodie "could not start bisect";
1667
1668     run_command "git bisect good $good" or
1669         dodie "could not set bisect good to $good";
1670
1671     run_git_bisect "git bisect bad $bad" or
1672         dodie "could not set bisect bad to $bad";
1673
1674     if (defined($replay)) {
1675         run_command "git bisect replay $replay" or
1676             dodie "failed to run replay";
1677     }
1678
1679     if (defined($start)) {
1680         run_command "git checkout $start" or
1681             dodie "failed to checkout $start";
1682     }
1683
1684     my $test;
1685     do {
1686         $result = run_bisect $type;
1687         $test = run_git_bisect "git bisect $result";
1688     } while ($test);
1689
1690     run_command "git bisect log" or
1691         dodie "could not capture git bisect log";
1692
1693     run_command "git bisect reset" or
1694         dodie "could not reset git bisect";
1695
1696     doprint "Bad commit was [$bisect_bad]\n";
1697
1698     success $i;
1699 }
1700
1701 my %config_ignore;
1702 my %config_set;
1703
1704 my %config_list;
1705 my %null_config;
1706
1707 my %dependency;
1708
1709 sub assign_configs {
1710     my ($hash, $config) = @_;
1711
1712     open (IN, $config)
1713         or dodie "Failed to read $config";
1714
1715     while (<IN>) {
1716         if (/^((CONFIG\S*)=.*)/) {
1717             ${$hash}{$2} = $1;
1718         }
1719     }
1720
1721     close(IN);
1722 }
1723
1724 sub process_config_ignore {
1725     my ($config) = @_;
1726
1727     assign_configs \%config_ignore, $config;
1728 }
1729
1730 sub read_current_config {
1731     my ($config_ref) = @_;
1732
1733     %{$config_ref} = ();
1734     undef %{$config_ref};
1735
1736     my @key = keys %{$config_ref};
1737     if ($#key >= 0) {
1738         print "did not delete!\n";
1739         exit;
1740     }
1741     open (IN, "$output_config");
1742
1743     while (<IN>) {
1744         if (/^(CONFIG\S+)=(.*)/) {
1745             ${$config_ref}{$1} = $2;
1746         }
1747     }
1748     close(IN);
1749 }
1750
1751 sub get_dependencies {
1752     my ($config) = @_;
1753
1754     my $arr = $dependency{$config};
1755     if (!defined($arr)) {
1756         return ();
1757     }
1758
1759     my @deps = @{$arr};
1760
1761     foreach my $dep (@{$arr}) {
1762         print "ADD DEP $dep\n";
1763         @deps = (@deps, get_dependencies $dep);
1764     }
1765
1766     return @deps;
1767 }
1768
1769 sub create_config {
1770     my @configs = @_;
1771
1772     open(OUT, ">$output_config") or dodie "Can not write to $output_config";
1773
1774     foreach my $config (@configs) {
1775         print OUT "$config_set{$config}\n";
1776         my @deps = get_dependencies $config;
1777         foreach my $dep (@deps) {
1778             print OUT "$config_set{$dep}\n";
1779         }
1780     }
1781
1782     foreach my $config (keys %config_ignore) {
1783         print OUT "$config_ignore{$config}\n";
1784     }
1785     close(OUT);
1786
1787 #    exit;
1788     make_oldconfig;
1789 }
1790
1791 sub compare_configs {
1792     my (%a, %b) = @_;
1793
1794     foreach my $item (keys %a) {
1795         if (!defined($b{$item})) {
1796             print "diff $item\n";
1797             return 1;
1798         }
1799         delete $b{$item};
1800     }
1801
1802     my @keys = keys %b;
1803     if ($#keys) {
1804         print "diff2 $keys[0]\n";
1805     }
1806     return -1 if ($#keys >= 0);
1807
1808     return 0;
1809 }
1810
1811 sub run_config_bisect_test {
1812     my ($type) = @_;
1813
1814     return run_bisect_test $type, "oldconfig";
1815 }
1816
1817 sub process_passed {
1818     my (%configs) = @_;
1819
1820     doprint "These configs had no failure: (Enabling them for further compiles)\n";
1821     # Passed! All these configs are part of a good compile.
1822     # Add them to the min options.
1823     foreach my $config (keys %configs) {
1824         if (defined($config_list{$config})) {
1825             doprint " removing $config\n";
1826             $config_ignore{$config} = $config_list{$config};
1827             delete $config_list{$config};
1828         }
1829     }
1830     doprint "config copied to $outputdir/config_good\n";
1831     run_command "cp -f $output_config $outputdir/config_good";
1832 }
1833
1834 sub process_failed {
1835     my ($config) = @_;
1836
1837     doprint "\n\n***************************************\n";
1838     doprint "Found bad config: $config\n";
1839     doprint "***************************************\n\n";
1840 }
1841
1842 sub run_config_bisect {
1843
1844     my @start_list = keys %config_list;
1845
1846     if ($#start_list < 0) {
1847         doprint "No more configs to test!!!\n";
1848         return -1;
1849     }
1850
1851     doprint "***** RUN TEST ***\n";
1852     my $type = $opt{"CONFIG_BISECT_TYPE[$iteration]"};
1853     my $ret;
1854     my %current_config;
1855
1856     my $count = $#start_list + 1;
1857     doprint "  $count configs to test\n";
1858
1859     my $half = int($#start_list / 2);
1860
1861     do {
1862         my @tophalf = @start_list[0 .. $half];
1863
1864         create_config @tophalf;
1865         read_current_config \%current_config;
1866
1867         $count = $#tophalf + 1;
1868         doprint "Testing $count configs\n";
1869         my $found = 0;
1870         # make sure we test something
1871         foreach my $config (@tophalf) {
1872             if (defined($current_config{$config})) {
1873                 logit " $config\n";
1874                 $found = 1;
1875             }
1876         }
1877         if (!$found) {
1878             # try the other half
1879             doprint "Top half produced no set configs, trying bottom half\n";
1880             @tophalf = @start_list[$half + 1 .. $#start_list];
1881             create_config @tophalf;
1882             read_current_config \%current_config;
1883             foreach my $config (@tophalf) {
1884                 if (defined($current_config{$config})) {
1885                     logit " $config\n";
1886                     $found = 1;
1887                 }
1888             }
1889             if (!$found) {
1890                 doprint "Failed: Can't make new config with current configs\n";
1891                 foreach my $config (@start_list) {
1892                     doprint "  CONFIG: $config\n";
1893                 }
1894                 return -1;
1895             }
1896             $count = $#tophalf + 1;
1897             doprint "Testing $count configs\n";
1898         }
1899
1900         $ret = run_config_bisect_test $type;
1901         if ($bisect_manual) {
1902             $ret = answer_bisect;
1903         }
1904         if ($ret) {
1905             process_passed %current_config;
1906             return 0;
1907         }
1908
1909         doprint "This config had a failure.\n";
1910         doprint "Removing these configs that were not set in this config:\n";
1911         doprint "config copied to $outputdir/config_bad\n";
1912         run_command "cp -f $output_config $outputdir/config_bad";
1913
1914         # A config exists in this group that was bad.
1915         foreach my $config (keys %config_list) {
1916             if (!defined($current_config{$config})) {
1917                 doprint " removing $config\n";
1918                 delete $config_list{$config};
1919             }
1920         }
1921
1922         @start_list = @tophalf;
1923
1924         if ($#start_list == 0) {
1925             process_failed $start_list[0];
1926             return 1;
1927         }
1928
1929         # remove half the configs we are looking at and see if
1930         # they are good.
1931         $half = int($#start_list / 2);
1932     } while ($#start_list > 0);
1933
1934     # we found a single config, try it again unless we are running manually
1935
1936     if ($bisect_manual) {
1937         process_failed $start_list[0];
1938         return 1;
1939     }
1940
1941     my @tophalf = @start_list[0 .. 0];
1942
1943     $ret = run_config_bisect_test $type;
1944     if ($ret) {
1945         process_passed %current_config;
1946         return 0;
1947     }
1948
1949     process_failed $start_list[0];
1950     return 1;
1951 }
1952
1953 sub config_bisect {
1954     my ($i) = @_;
1955
1956     my $start_config = $opt{"CONFIG_BISECT[$i]"};
1957
1958     my $tmpconfig = "$tmpdir/use_config";
1959
1960     if (defined($config_bisect_good)) {
1961         process_config_ignore $config_bisect_good;
1962     }
1963
1964     # Make the file with the bad config and the min config
1965     if (defined($minconfig)) {
1966         # read the min config for things to ignore
1967         run_command "cp $minconfig $tmpconfig" or
1968             dodie "failed to copy $minconfig to $tmpconfig";
1969     } else {
1970         unlink $tmpconfig;
1971     }
1972
1973     # Add other configs
1974     if (defined($addconfig)) {
1975         run_command "cat $addconfig >> $tmpconfig" or
1976             dodie "failed to append $addconfig";
1977     }
1978
1979     if (-f $tmpconfig) {
1980         load_force_config($tmpconfig);
1981         process_config_ignore $tmpconfig;
1982     }
1983
1984     # now process the start config
1985     run_command "cp $start_config $output_config" or
1986         dodie "failed to copy $start_config to $output_config";
1987
1988     # read directly what we want to check
1989     my %config_check;
1990     open (IN, $output_config)
1991         or dodie "faied to open $output_config";
1992
1993     while (<IN>) {
1994         if (/^((CONFIG\S*)=.*)/) {
1995             $config_check{$2} = $1;
1996         }
1997     }
1998     close(IN);
1999
2000     # Now run oldconfig with the minconfig (and addconfigs)
2001     make_oldconfig;
2002
2003     # check to see what we lost (or gained)
2004     open (IN, $output_config)
2005         or dodie "Failed to read $start_config";
2006
2007     my %removed_configs;
2008     my %added_configs;
2009
2010     while (<IN>) {
2011         if (/^((CONFIG\S*)=.*)/) {
2012             # save off all options
2013             $config_set{$2} = $1;
2014             if (defined($config_check{$2})) {
2015                 if (defined($config_ignore{$2})) {
2016                     $removed_configs{$2} = $1;
2017                 } else {
2018                     $config_list{$2} = $1;
2019                 }
2020             } elsif (!defined($config_ignore{$2})) {
2021                 $added_configs{$2} = $1;
2022                 $config_list{$2} = $1;
2023             }
2024         }
2025     }
2026     close(IN);
2027
2028     my @confs = keys %removed_configs;
2029     if ($#confs >= 0) {
2030         doprint "Configs overridden by default configs and removed from check:\n";
2031         foreach my $config (@confs) {
2032             doprint " $config\n";
2033         }
2034     }
2035     @confs = keys %added_configs;
2036     if ($#confs >= 0) {
2037         doprint "Configs appearing in make oldconfig and added:\n";
2038         foreach my $config (@confs) {
2039             doprint " $config\n";
2040         }
2041     }
2042
2043     my %config_test;
2044     my $once = 0;
2045
2046     # Sometimes kconfig does weird things. We must make sure
2047     # that the config we autocreate has everything we need
2048     # to test, otherwise we may miss testing configs, or
2049     # may not be able to create a new config.
2050     # Here we create a config with everything set.
2051     create_config (keys %config_list);
2052     read_current_config \%config_test;
2053     foreach my $config (keys %config_list) {
2054         if (!defined($config_test{$config})) {
2055             if (!$once) {
2056                 $once = 1;
2057                 doprint "Configs not produced by kconfig (will not be checked):\n";
2058             }
2059             doprint "  $config\n";
2060             delete $config_list{$config};
2061         }
2062     }
2063     my $ret;
2064     do {
2065         $ret = run_config_bisect;
2066     } while (!$ret);
2067
2068     return $ret if ($ret < 0);
2069
2070     success $i;
2071 }
2072
2073 sub patchcheck_reboot {
2074     doprint "Reboot and sleep $patchcheck_sleep_time seconds\n";
2075     reboot;
2076     start_monitor;
2077     wait_for_monitor $patchcheck_sleep_time;
2078     end_monitor;
2079 }
2080
2081 sub patchcheck {
2082     my ($i) = @_;
2083
2084     die "PATCHCHECK_START[$i] not defined\n"
2085         if (!defined($opt{"PATCHCHECK_START[$i]"}));
2086     die "PATCHCHECK_TYPE[$i] not defined\n"
2087         if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
2088
2089     my $start = $opt{"PATCHCHECK_START[$i]"};
2090
2091     my $end = "HEAD";
2092     if (defined($opt{"PATCHCHECK_END[$i]"})) {
2093         $end = $opt{"PATCHCHECK_END[$i]"};
2094     }
2095
2096     # Get the true sha1's since we can use things like HEAD~3
2097     $start = get_sha1($start);
2098     $end = get_sha1($end);
2099
2100     my $type = $opt{"PATCHCHECK_TYPE[$i]"};
2101
2102     # Can't have a test without having a test to run
2103     if ($type eq "test" && !defined($run_test)) {
2104         $type = "boot";
2105     }
2106
2107     open (IN, "git log --pretty=oneline $end|") or
2108         dodie "could not get git list";
2109
2110     my @list;
2111
2112     while (<IN>) {
2113         chomp;
2114         $list[$#list+1] = $_;
2115         last if (/^$start/);
2116     }
2117     close(IN);
2118
2119     if ($list[$#list] !~ /^$start/) {
2120         fail "SHA1 $start not found";
2121     }
2122
2123     # go backwards in the list
2124     @list = reverse @list;
2125
2126     my $save_clean = $noclean;
2127     my %ignored_warnings;
2128
2129     if (defined($ignore_warnings)) {
2130         foreach my $sha1 (split /\s+/, $ignore_warnings) {
2131             $ignored_warnings{$sha1} = 1;
2132         }
2133     }
2134
2135     $in_patchcheck = 1;
2136     foreach my $item (@list) {
2137         my $sha1 = $item;
2138         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
2139
2140         doprint "\nProcessing commit $item\n\n";
2141
2142         run_command "git checkout $sha1" or
2143             die "Failed to checkout $sha1";
2144
2145         # only clean on the first and last patch
2146         if ($item eq $list[0] ||
2147             $item eq $list[$#list]) {
2148             $noclean = $save_clean;
2149         } else {
2150             $noclean = 1;
2151         }
2152
2153         if (defined($minconfig)) {
2154             build "useconfig:$minconfig" or return 0;
2155         } else {
2156             # ?? no config to use?
2157             build "oldconfig" or return 0;
2158         }
2159
2160
2161         if (!defined($ignored_warnings{$sha1})) {
2162             check_buildlog $sha1 or return 0;
2163         }
2164
2165         next if ($type eq "build");
2166
2167         my $failed = 0;
2168
2169         start_monitor_and_boot or $failed = 1;
2170
2171         if (!$failed && $type ne "boot"){
2172             do_run_test or $failed = 1;
2173         }
2174         end_monitor;
2175         return 0 if ($failed);
2176
2177         patchcheck_reboot;
2178
2179     }
2180     $in_patchcheck = 0;
2181     success $i;
2182
2183     return 1;
2184 }
2185
2186 my %depends;
2187 my $iflevel = 0;
2188 my @ifdeps;
2189
2190 # prevent recursion
2191 my %read_kconfigs;
2192
2193 # taken from streamline_config.pl
2194 sub read_kconfig {
2195     my ($kconfig) = @_;
2196
2197     my $state = "NONE";
2198     my $config;
2199     my @kconfigs;
2200
2201     my $cont = 0;
2202     my $line;
2203
2204
2205     if (! -f $kconfig) {
2206         doprint "file $kconfig does not exist, skipping\n";
2207         return;
2208     }
2209
2210     open(KIN, "$kconfig")
2211         or die "Can't open $kconfig";
2212     while (<KIN>) {
2213         chomp;
2214
2215         # Make sure that lines ending with \ continue
2216         if ($cont) {
2217             $_ = $line . " " . $_;
2218         }
2219
2220         if (s/\\$//) {
2221             $cont = 1;
2222             $line = $_;
2223             next;
2224         }
2225
2226         $cont = 0;
2227
2228         # collect any Kconfig sources
2229         if (/^source\s*"(.*)"/) {
2230             $kconfigs[$#kconfigs+1] = $1;
2231         }
2232
2233         # configs found
2234         if (/^\s*(menu)?config\s+(\S+)\s*$/) {
2235             $state = "NEW";
2236             $config = $2;
2237
2238             for (my $i = 0; $i < $iflevel; $i++) {
2239                 if ($i) {
2240                     $depends{$config} .= " " . $ifdeps[$i];
2241                 } else {
2242                     $depends{$config} = $ifdeps[$i];
2243                 }
2244                 $state = "DEP";
2245             }
2246
2247         # collect the depends for the config
2248         } elsif ($state eq "NEW" && /^\s*depends\s+on\s+(.*)$/) {
2249
2250             if (defined($depends{$1})) {
2251                 $depends{$config} .= " " . $1;
2252             } else {
2253                 $depends{$config} = $1;
2254             }
2255
2256         # Get the configs that select this config
2257         } elsif ($state ne "NONE" && /^\s*select\s+(\S+)/) {
2258             if (defined($depends{$1})) {
2259                 $depends{$1} .= " " . $config;
2260             } else {
2261                 $depends{$1} = $config;
2262             }
2263
2264         # Check for if statements
2265         } elsif (/^if\s+(.*\S)\s*$/) {
2266             my $deps = $1;
2267             # remove beginning and ending non text
2268             $deps =~ s/^[^a-zA-Z0-9_]*//;
2269             $deps =~ s/[^a-zA-Z0-9_]*$//;
2270
2271             my @deps = split /[^a-zA-Z0-9_]+/, $deps;
2272
2273             $ifdeps[$iflevel++] = join ':', @deps;
2274
2275         } elsif (/^endif/) {
2276
2277             $iflevel-- if ($iflevel);
2278
2279         # stop on "help"
2280         } elsif (/^\s*help\s*$/) {
2281             $state = "NONE";
2282         }
2283     }
2284     close(KIN);
2285
2286     # read in any configs that were found.
2287     foreach $kconfig (@kconfigs) {
2288         if (!defined($read_kconfigs{$kconfig})) {
2289             $read_kconfigs{$kconfig} = 1;
2290             read_kconfig("$builddir/$kconfig");
2291         }
2292     }
2293 }
2294
2295 sub read_depends {
2296     # find out which arch this is by the kconfig file
2297     open (IN, $output_config)
2298         or dodie "Failed to read $output_config";
2299     my $arch;
2300     while (<IN>) {
2301         if (m,Linux/(\S+)\s+\S+\s+Kernel Configuration,) {
2302             $arch = $1;
2303             last;
2304         }
2305     }
2306     close IN;
2307
2308     if (!defined($arch)) {
2309         doprint "Could not find arch from config file\n";
2310         doprint "no dependencies used\n";
2311         return;
2312     }
2313
2314     # arch is really the subarch, we need to know
2315     # what directory to look at.
2316     if ($arch eq "i386" || $arch eq "x86_64") {
2317         $arch = "x86";
2318     } elsif ($arch =~ /^tile/) {
2319         $arch = "tile";
2320     }
2321
2322     my $kconfig = "$builddir/arch/$arch/Kconfig";
2323
2324     if (! -f $kconfig && $arch =~ /\d$/) {
2325         my $orig = $arch;
2326         # some subarchs have numbers, truncate them
2327         $arch =~ s/\d*$//;
2328         $kconfig = "$builddir/arch/$arch/Kconfig";
2329         if (! -f $kconfig) {
2330             doprint "No idea what arch dir $orig is for\n";
2331             doprint "no dependencies used\n";
2332             return;
2333         }
2334     }
2335
2336     read_kconfig($kconfig);
2337 }
2338
2339 sub read_config_list {
2340     my ($config) = @_;
2341
2342     open (IN, $config)
2343         or dodie "Failed to read $config";
2344
2345     while (<IN>) {
2346         if (/^((CONFIG\S*)=.*)/) {
2347             if (!defined($config_ignore{$2})) {
2348                 $config_list{$2} = $1;
2349             }
2350         }
2351     }
2352
2353     close(IN);
2354 }
2355
2356 sub read_output_config {
2357     my ($config) = @_;
2358
2359     assign_configs \%config_ignore, $config;
2360 }
2361
2362 sub make_new_config {
2363     my @configs = @_;
2364
2365     open (OUT, ">$output_config")
2366         or dodie "Failed to write $output_config";
2367
2368     foreach my $config (@configs) {
2369         print OUT "$config\n";
2370     }
2371     close OUT;
2372 }
2373
2374 sub get_depends {
2375     my ($dep) = @_;
2376
2377     my $kconfig = $dep;
2378     $kconfig =~ s/CONFIG_//;
2379
2380     $dep = $depends{"$kconfig"};
2381
2382     # the dep string we have saves the dependencies as they
2383     # were found, including expressions like ! && ||. We
2384     # want to split this out into just an array of configs.
2385
2386     my $valid = "A-Za-z_0-9";
2387
2388     my @configs;
2389
2390     while ($dep =~ /[$valid]/) {
2391
2392         if ($dep =~ /^[^$valid]*([$valid]+)/) {
2393             my $conf = "CONFIG_" . $1;
2394
2395             $configs[$#configs + 1] = $conf;
2396
2397             $dep =~ s/^[^$valid]*[$valid]+//;
2398         } else {
2399             die "this should never happen";
2400         }
2401     }
2402
2403     return @configs;
2404 }
2405
2406 my %min_configs;
2407 my %keep_configs;
2408 my %processed_configs;
2409 my %nochange_config;
2410
2411 sub test_this_config {
2412     my ($config) = @_;
2413
2414     my $found;
2415
2416     # if we already processed this config, skip it
2417     if (defined($processed_configs{$config})) {
2418         return undef;
2419     }
2420     $processed_configs{$config} = 1;
2421
2422     # if this config failed during this round, skip it
2423     if (defined($nochange_config{$config})) {
2424         return undef;
2425     }
2426
2427     my $kconfig = $config;
2428     $kconfig =~ s/CONFIG_//;
2429
2430     # Test dependencies first
2431     if (defined($depends{"$kconfig"})) {
2432         my @parents = get_depends $config;
2433         foreach my $parent (@parents) {
2434             # if the parent is in the min config, check it first
2435             next if (!defined($min_configs{$parent}));
2436             $found = test_this_config($parent);
2437             if (defined($found)) {
2438                 return $found;
2439             }
2440         }
2441     }
2442
2443     # Remove this config from the list of configs
2444     # do a make oldnoconfig and then read the resulting
2445     # .config to make sure it is missing the config that
2446     # we had before
2447     my %configs = %min_configs;
2448     delete $configs{$config};
2449     make_new_config ((values %configs), (values %keep_configs));
2450     make_oldconfig;
2451     undef %configs;
2452     assign_configs \%configs, $output_config;
2453
2454     return $config if (!defined($configs{$config}));
2455
2456     doprint "disabling config $config did not change .config\n";
2457
2458     $nochange_config{$config} = 1;
2459
2460     return undef;
2461 }
2462
2463 sub make_min_config {
2464     my ($i) = @_;
2465
2466     if (!defined($output_minconfig)) {
2467         fail "OUTPUT_MIN_CONFIG not defined" and return;
2468     }
2469
2470     # If output_minconfig exists, and the start_minconfig
2471     # came from min_config, than ask if we should use
2472     # that instead.
2473     if (-f $output_minconfig && !$start_minconfig_defined) {
2474         print "$output_minconfig exists\n";
2475         if (read_yn " Use it as minconfig?") {
2476             $start_minconfig = $output_minconfig;
2477         }
2478     }
2479
2480     if (!defined($start_minconfig)) {
2481         fail "START_MIN_CONFIG or MIN_CONFIG not defined" and return;
2482     }
2483
2484     my $temp_config = "$tmpdir/temp_config";
2485
2486     # First things first. We build an allnoconfig to find
2487     # out what the defaults are that we can't touch.
2488     # Some are selections, but we really can't handle selections.
2489
2490     my $save_minconfig = $minconfig;
2491     undef $minconfig;
2492
2493     run_command "$make allnoconfig" or return 0;
2494
2495     read_depends;
2496
2497     process_config_ignore $output_config;
2498
2499     undef %keep_configs;
2500     undef %min_configs;
2501
2502     if (defined($ignore_config)) {
2503         # make sure the file exists
2504         `touch $ignore_config`;
2505         assign_configs \%keep_configs, $ignore_config;
2506     }
2507
2508     doprint "Load initial configs from $start_minconfig\n";
2509
2510     # Look at the current min configs, and save off all the
2511     # ones that were set via the allnoconfig
2512     assign_configs \%min_configs, $start_minconfig;
2513
2514     my @config_keys = keys %min_configs;
2515
2516     # Remove anything that was set by the make allnoconfig
2517     # we shouldn't need them as they get set for us anyway.
2518     foreach my $config (@config_keys) {
2519         # Remove anything in the ignore_config
2520         if (defined($keep_configs{$config})) {
2521             my $file = $ignore_config;
2522             $file =~ s,.*/(.*?)$,$1,;
2523             doprint "$config set by $file ... ignored\n";
2524             delete $min_configs{$config};
2525             next;
2526         }
2527         # But make sure the settings are the same. If a min config
2528         # sets a selection, we do not want to get rid of it if
2529         # it is not the same as what we have. Just move it into
2530         # the keep configs.
2531         if (defined($config_ignore{$config})) {
2532             if ($config_ignore{$config} ne $min_configs{$config}) {
2533                 doprint "$config is in allnoconfig as '$config_ignore{$config}'";
2534                 doprint " but it is '$min_configs{$config}' in minconfig .. keeping\n";
2535                 $keep_configs{$config} = $min_configs{$config};
2536             } else {
2537                 doprint "$config set by allnoconfig ... ignored\n";
2538             }
2539             delete $min_configs{$config};
2540         }
2541     }
2542
2543     my $done = 0;
2544     my $take_two = 0;
2545
2546     while (!$done) {
2547
2548         my $config;
2549         my $found;
2550
2551         # Now disable each config one by one and do a make oldconfig
2552         # till we find a config that changes our list.
2553
2554         # Put configs that did not modify the config at the end.
2555         my @test_configs = keys %min_configs;
2556         my $reset = 1;
2557         for (my $i = 0; $i < $#test_configs; $i++) {
2558             if (!defined($nochange_config{$test_configs[0]})) {
2559                 $reset = 0;
2560                 last;
2561             }
2562             # This config didn't change the .config last time.
2563             # Place it at the end
2564             my $config = shift @test_configs;
2565             push @test_configs, $config;
2566         }
2567
2568         # if every test config has failed to modify the .config file
2569         # in the past, then reset and start over.
2570         if ($reset) {
2571             undef %nochange_config;
2572         }
2573
2574         undef %processed_configs;
2575
2576         foreach my $config (@test_configs) {
2577
2578             $found = test_this_config $config;
2579
2580             last if (defined($found));
2581
2582             # oh well, try another config
2583         }
2584
2585         if (!defined($found)) {
2586             # we could have failed due to the nochange_config hash
2587             # reset and try again
2588             if (!$take_two) {
2589                 undef %nochange_config;
2590                 $take_two = 1;
2591                 next;
2592             }
2593             doprint "No more configs found that we can disable\n";
2594             $done = 1;
2595             last;
2596         }
2597         $take_two = 0;
2598
2599         $config = $found;
2600
2601         doprint "Test with $config disabled\n";
2602
2603         # set in_bisect to keep build and monitor from dieing
2604         $in_bisect = 1;
2605
2606         my $failed = 0;
2607         build "oldconfig";
2608         start_monitor_and_boot or $failed = 1;
2609         end_monitor;
2610
2611         $in_bisect = 0;
2612
2613         if ($failed) {
2614             doprint "$min_configs{$config} is needed to boot the box... keeping\n";
2615             # this config is needed, add it to the ignore list.
2616             $keep_configs{$config} = $min_configs{$config};
2617             delete $min_configs{$config};
2618
2619             # update new ignore configs
2620             if (defined($ignore_config)) {
2621                 open (OUT, ">$temp_config")
2622                     or die "Can't write to $temp_config";
2623                 foreach my $config (keys %keep_configs) {
2624                     print OUT "$keep_configs{$config}\n";
2625                 }
2626                 close OUT;
2627                 run_command "mv $temp_config $ignore_config" or
2628                     dodie "failed to copy update to $ignore_config";
2629             }
2630
2631         } else {
2632             # We booted without this config, remove it from the minconfigs.
2633             doprint "$config is not needed, disabling\n";
2634
2635             delete $min_configs{$config};
2636
2637             # Also disable anything that is not enabled in this config
2638             my %configs;
2639             assign_configs \%configs, $output_config;
2640             my @config_keys = keys %min_configs;
2641             foreach my $config (@config_keys) {
2642                 if (!defined($configs{$config})) {
2643                     doprint "$config is not set, disabling\n";
2644                     delete $min_configs{$config};
2645                 }
2646             }
2647
2648             # Save off all the current mandidory configs
2649             open (OUT, ">$temp_config")
2650                 or die "Can't write to $temp_config";
2651             foreach my $config (keys %keep_configs) {
2652                 print OUT "$keep_configs{$config}\n";
2653             }
2654             foreach my $config (keys %min_configs) {
2655                 print OUT "$min_configs{$config}\n";
2656             }
2657             close OUT;
2658
2659             run_command "mv $temp_config $output_minconfig" or
2660                 dodie "failed to copy update to $output_minconfig";
2661         }
2662
2663         doprint "Reboot and wait $sleep_time seconds\n";
2664         reboot;
2665         start_monitor;
2666         wait_for_monitor $sleep_time;
2667         end_monitor;
2668     }
2669
2670     success $i;
2671     return 1;
2672 }
2673
2674 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
2675
2676 if ($#ARGV == 0) {
2677     $ktest_config = $ARGV[0];
2678     if (! -f $ktest_config) {
2679         print "$ktest_config does not exist.\n";
2680         if (!read_yn "Create it?") {
2681             exit 0;
2682         }
2683     }
2684 } else {
2685     $ktest_config = "ktest.conf";
2686 }
2687
2688 if (! -f $ktest_config) {
2689     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
2690     print OUT << "EOF"
2691 # Generated by ktest.pl
2692 #
2693 # Define each test with TEST_START
2694 # The config options below it will override the defaults
2695 TEST_START
2696
2697 DEFAULTS
2698 EOF
2699 ;
2700     close(OUT);
2701 }
2702 read_config $ktest_config;
2703
2704 if (defined($opt{"LOG_FILE"})) {
2705     $opt{"LOG_FILE"} = eval_option($opt{"LOG_FILE"}, -1);
2706 }
2707
2708 # Append any configs entered in manually to the config file.
2709 my @new_configs = keys %entered_configs;
2710 if ($#new_configs >= 0) {
2711     print "\nAppending entered in configs to $ktest_config\n";
2712     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
2713     foreach my $config (@new_configs) {
2714         print OUT "$config = $entered_configs{$config}\n";
2715         $opt{$config} = $entered_configs{$config};
2716     }
2717 }
2718
2719 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
2720     unlink $opt{"LOG_FILE"};
2721 }
2722
2723 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
2724
2725 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
2726
2727     if (!$i) {
2728         doprint "DEFAULT OPTIONS:\n";
2729     } else {
2730         doprint "\nTEST $i OPTIONS";
2731         if (defined($repeat_tests{$i})) {
2732             $repeat = $repeat_tests{$i};
2733             doprint " ITERATE $repeat";
2734         }
2735         doprint "\n";
2736     }
2737
2738     foreach my $option (sort keys %opt) {
2739
2740         if ($option =~ /\[(\d+)\]$/) {
2741             next if ($i != $1);
2742         } else {
2743             next if ($i);
2744         }
2745
2746         doprint "$option = $opt{$option}\n";
2747     }
2748 }
2749
2750 sub __set_test_option {
2751     my ($name, $i) = @_;
2752
2753     my $option = "$name\[$i\]";
2754
2755     if (defined($opt{$option})) {
2756         return $opt{$option};
2757     }
2758
2759     foreach my $test (keys %repeat_tests) {
2760         if ($i >= $test &&
2761             $i < $test + $repeat_tests{$test}) {
2762             $option = "$name\[$test\]";
2763             if (defined($opt{$option})) {
2764                 return $opt{$option};
2765             }
2766         }
2767     }
2768
2769     if (defined($opt{$name})) {
2770         return $opt{$name};
2771     }
2772
2773     return undef;
2774 }
2775
2776 sub set_test_option {
2777     my ($name, $i) = @_;
2778
2779     my $option = __set_test_option($name, $i);
2780     return $option if (!defined($option));
2781
2782     return eval_option($option, $i);
2783 }
2784
2785 # First we need to do is the builds
2786 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
2787
2788     $iteration = $i;
2789
2790     my $makecmd = set_test_option("MAKE_CMD", $i);
2791
2792     $machine = set_test_option("MACHINE", $i);
2793     $ssh_user = set_test_option("SSH_USER", $i);
2794     $tmpdir = set_test_option("TMP_DIR", $i);
2795     $outputdir = set_test_option("OUTPUT_DIR", $i);
2796     $builddir = set_test_option("BUILD_DIR", $i);
2797     $test_type = set_test_option("TEST_TYPE", $i);
2798     $build_type = set_test_option("BUILD_TYPE", $i);
2799     $build_options = set_test_option("BUILD_OPTIONS", $i);
2800     $pre_build = set_test_option("PRE_BUILD", $i);
2801     $post_build = set_test_option("POST_BUILD", $i);
2802     $pre_build_die = set_test_option("PRE_BUILD_DIE", $i);
2803     $post_build_die = set_test_option("POST_BUILD_DIE", $i);
2804     $power_cycle = set_test_option("POWER_CYCLE", $i);
2805     $reboot = set_test_option("REBOOT", $i);
2806     $noclean = set_test_option("BUILD_NOCLEAN", $i);
2807     $minconfig = set_test_option("MIN_CONFIG", $i);
2808     $output_minconfig = set_test_option("OUTPUT_MIN_CONFIG", $i);
2809     $start_minconfig = set_test_option("START_MIN_CONFIG", $i);
2810     $ignore_config = set_test_option("IGNORE_CONFIG", $i);
2811     $run_test = set_test_option("TEST", $i);
2812     $addconfig = set_test_option("ADD_CONFIG", $i);
2813     $reboot_type = set_test_option("REBOOT_TYPE", $i);
2814     $grub_menu = set_test_option("GRUB_MENU", $i);
2815     $post_install = set_test_option("POST_INSTALL", $i);
2816     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
2817     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
2818     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
2819     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
2820     $power_off = set_test_option("POWER_OFF", $i);
2821     $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
2822     $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
2823     $sleep_time = set_test_option("SLEEP_TIME", $i);
2824     $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
2825     $patchcheck_sleep_time = set_test_option("PATCHCHECK_SLEEP_TIME", $i);
2826     $ignore_warnings = set_test_option("IGNORE_WARNINGS", $i);
2827     $bisect_manual = set_test_option("BISECT_MANUAL", $i);
2828     $bisect_skip = set_test_option("BISECT_SKIP", $i);
2829     $config_bisect_good = set_test_option("CONFIG_BISECT_GOOD", $i);
2830     $store_failures = set_test_option("STORE_FAILURES", $i);
2831     $test_name = set_test_option("TEST_NAME", $i);
2832     $timeout = set_test_option("TIMEOUT", $i);
2833     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
2834     $console = set_test_option("CONSOLE", $i);
2835     $detect_triplefault = set_test_option("DETECT_TRIPLE_FAULT", $i);
2836     $success_line = set_test_option("SUCCESS_LINE", $i);
2837     $stop_after_success = set_test_option("STOP_AFTER_SUCCESS", $i);
2838     $stop_after_failure = set_test_option("STOP_AFTER_FAILURE", $i);
2839     $stop_test_after = set_test_option("STOP_TEST_AFTER", $i);
2840     $build_target = set_test_option("BUILD_TARGET", $i);
2841     $ssh_exec = set_test_option("SSH_EXEC", $i);
2842     $scp_to_target = set_test_option("SCP_TO_TARGET", $i);
2843     $target_image = set_test_option("TARGET_IMAGE", $i);
2844     $localversion = set_test_option("LOCALVERSION", $i);
2845
2846     $start_minconfig_defined = 1;
2847
2848     if (!defined($start_minconfig)) {
2849         $start_minconfig_defined = 0;
2850         $start_minconfig = $minconfig;
2851     }
2852
2853     chdir $builddir || die "can't change directory to $builddir";
2854
2855     if (!-d $tmpdir) {
2856         mkpath($tmpdir) or
2857             die "can't create $tmpdir";
2858     }
2859
2860     $ENV{"SSH_USER"} = $ssh_user;
2861     $ENV{"MACHINE"} = $machine;
2862
2863     $target = "$ssh_user\@$machine";
2864
2865     $buildlog = "$tmpdir/buildlog-$machine";
2866     $dmesg = "$tmpdir/dmesg-$machine";
2867     $make = "$makecmd O=$outputdir";
2868     $output_config = "$outputdir/.config";
2869
2870     if ($reboot_type eq "grub") {
2871         dodie "GRUB_MENU not defined" if (!defined($grub_menu));
2872     } elsif (!defined($reboot_script)) {
2873         dodie "REBOOT_SCRIPT not defined"
2874     }
2875
2876     my $run_type = $build_type;
2877     if ($test_type eq "patchcheck") {
2878         $run_type = $opt{"PATCHCHECK_TYPE[$i]"};
2879     } elsif ($test_type eq "bisect") {
2880         $run_type = $opt{"BISECT_TYPE[$i]"};
2881     } elsif ($test_type eq "config_bisect") {
2882         $run_type = $opt{"CONFIG_BISECT_TYPE[$i]"};
2883     }
2884
2885     if ($test_type eq "make_min_config") {
2886         $run_type = "";
2887     }
2888
2889     # mistake in config file?
2890     if (!defined($run_type)) {
2891         $run_type = "ERROR";
2892     }
2893
2894     doprint "\n\n";
2895     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type\n\n";
2896
2897     unlink $dmesg;
2898     unlink $buildlog;
2899
2900     if (!defined($minconfig)) {
2901         $minconfig = $addconfig;
2902
2903     } elsif (defined($addconfig)) {
2904         run_command "cat $addconfig $minconfig > $tmpdir/add_config" or
2905             dodie "Failed to create temp config";
2906         $minconfig = "$tmpdir/add_config";
2907     }
2908
2909     my $checkout = $opt{"CHECKOUT[$i]"};
2910     if (defined($checkout)) {
2911         run_command "git checkout $checkout" or
2912             die "failed to checkout $checkout";
2913     }
2914
2915     if ($test_type eq "bisect") {
2916         bisect $i;
2917         next;
2918     } elsif ($test_type eq "config_bisect") {
2919         config_bisect $i;
2920         next;
2921     } elsif ($test_type eq "patchcheck") {
2922         patchcheck $i;
2923         next;
2924     } elsif ($test_type eq "make_min_config") {
2925         make_min_config $i;
2926         next;
2927     }
2928
2929     if ($build_type ne "nobuild") {
2930         build $build_type or next;
2931     }
2932
2933     if ($test_type ne "build") {
2934         my $failed = 0;
2935         start_monitor_and_boot or $failed = 1;
2936
2937         if (!$failed && $test_type ne "boot" && defined($run_test)) {
2938             do_run_test or $failed = 1;
2939         }
2940         end_monitor;
2941         next if ($failed);
2942     }
2943
2944     success $i;
2945 }
2946
2947 if ($opt{"POWEROFF_ON_SUCCESS"}) {
2948     halt;
2949 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
2950     reboot;
2951 }
2952
2953 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
2954
2955 exit 0;