tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copyright 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21 my %default;
  22
  23 #default opts
  24 $default{"NUM_TESTS"}           = 1;
  25 $default{"REBOOT_TYPE"}         = "grub";
  26 $default{"TEST_TYPE"}           = "test";
  27 $default{"BUILD_TYPE"}          = "randconfig";
  28 $default{"MAKE_CMD"}            = "make";
  29 $default{"TIMEOUT"}             = 120;
  30 $default{"TMP_DIR"}             = "/tmp/ktest";
  31 $default{"SLEEP_TIME"}          = 60;   # sleep time between tests
  32 $default{"BUILD_NOCLEAN"}       = 0;
  33 $default{"REBOOT_ON_ERROR"}     = 0;
  34 $default{"POWEROFF_ON_ERROR"}   = 0;
  35 $default{"REBOOT_ON_SUCCESS"}   = 1;
  36 $default{"POWEROFF_ON_SUCCESS"} = 0;
  37 $default{"BUILD_OPTIONS"}       = "";
  38 $default{"BISECT_SLEEP_TIME"}   = 60;   # sleep time between bisects
  39 $default{"PATCHCHECK_SLEEP_TIME"} = 60; # sleep time between patch checks
  40 $default{"CLEAR_LOG"}           = 0;
  41 $default{"BISECT_MANUAL"}       = 0;
  42 $default{"BISECT_SKIP"}         = 1;
  43 $default{"SUCCESS_LINE"}        = "login:";
  44 $default{"BOOTED_TIMEOUT"}      = 1;
  45 $default{"DIE_ON_FAILURE"}      = 1;
  46 $default{"SSH_EXEC"}            = "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND";
  47 $default{"SCP_TO_TARGET"}       = "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE";
  48 $default{"REBOOT"}              = "ssh \$SSH_USER\@\$MACHINE reboot";
  49 $default{"STOP_AFTER_SUCCESS"}  = 10;
  50 $default{"STOP_AFTER_FAILURE"}  = 60;
  51 $default{"STOP_TEST_AFTER"}     = 600;
  52 $default{"LOCALVERSION"}        = "-test";
  53
  54 my $ktest_config;
  55 my $version;
  56 my $machine;
  57 my $ssh_user;
  58 my $tmpdir;
  59 my $builddir;
  60 my $outputdir;
  61 my $output_config;
  62 my $test_type;
  63 my $build_type;
  64 my $build_options;
  65 my $reboot_type;
  66 my $reboot_script;
  67 my $power_cycle;
  68 my $reboot;
  69 my $reboot_on_error;
  70 my $poweroff_on_error;
  71 my $die_on_failure;
  72 my $powercycle_after_reboot;
  73 my $poweroff_after_halt;
  74 my $ssh_exec;
  75 my $scp_to_target;
  76 my $power_off;
  77 my $grub_menu;
  78 my $grub_number;
  79 my $target;
  80 my $make;
  81 my $post_install;
  82 my $noclean;
  83 my $minconfig;
  84 my $addconfig;
  85 my $in_bisect = 0;
  86 my $bisect_bad = "";
  87 my $reverse_bisect;
  88 my $bisect_manual;
  89 my $bisect_skip;
  90 my $in_patchcheck = 0;
  91 my $run_test;
  92 my $redirect;
  93 my $buildlog;
  94 my $dmesg;
  95 my $monitor_fp;
  96 my $monitor_pid;
  97 my $monitor_cnt = 0;
  98 my $sleep_time;
  99 my $bisect_sleep_time;
 100 my $patchcheck_sleep_time;
 101 my $store_failures;
 102 my $timeout;
 103 my $booted_timeout;
 104 my $console;
 105 my $success_line;
 106 my $stop_after_success;
 107 my $stop_after_failure;
 108 my $stop_test_after;
 109 my $build_target;
 110 my $target_image;
 111 my $localversion;
 112 my $iteration = 0;
 113 my $successes = 0;
 114
 115 my %entered_configs;
 116 my %config_help;
 117 my %variable;
 118
 119 $config_help{"MACHINE"} = << "EOF"
 120  The machine hostname that you will test.
 121 EOF
 122     ;
 123 $config_help{"SSH_USER"} = << "EOF"
 124  The box is expected to have ssh on normal bootup, provide the user
 125   (most likely root, since you need privileged operations)
 126 EOF
 127     ;
 128 $config_help{"BUILD_DIR"} = << "EOF"
 129  The directory that contains the Linux source code (full path).
 130 EOF
 131     ;
 132 $config_help{"OUTPUT_DIR"} = << "EOF"
 133  The directory that the objects will be built (full path).
 134  (can not be same as BUILD_DIR)
 135 EOF
 136     ;
 137 $config_help{"BUILD_TARGET"} = << "EOF"
 138  The location of the compiled file to copy to the target.
 139  (relative to OUTPUT_DIR)
 140 EOF
 141     ;
 142 $config_help{"TARGET_IMAGE"} = << "EOF"
 143  The place to put your image on the test machine.
 144 EOF
 145     ;
 146 $config_help{"POWER_CYCLE"} = << "EOF"
 147  A script or command to reboot the box.
 148
 149  Here is a digital loggers power switch example
 150  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 151
 152  Here is an example to reboot a virtual box on the current host
 153  with the name "Guest".
 154  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 155 EOF
 156     ;
 157 $config_help{"CONSOLE"} = << "EOF"
 158  The script or command that reads the console
 159
 160   If you use ttywatch server, something like the following would work.
 161 CONSOLE = nc -d localhost 3001
 162
 163  For a virtual machine with guest name "Guest".
 164 CONSOLE =  virsh console Guest
 165 EOF
 166     ;
 167 $config_help{"LOCALVERSION"} = << "EOF"
 168  Required version ending to differentiate the test
 169  from other linux builds on the system.
 170 EOF
 171     ;
 172 $config_help{"REBOOT_TYPE"} = << "EOF"
 173  Way to reboot the box to the test kernel.
 174  Only valid options so far are "grub" and "script".
 175
 176  If you specify grub, it will assume grub version 1
 177  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 178  and select that target to reboot to the kernel. If this is not
 179  your setup, then specify "script" and have a command or script
 180  specified in REBOOT_SCRIPT to boot to the target.
 181
 182  The entry in /boot/grub/menu.lst must be entered in manually.
 183  The test will not modify that file.
 184 EOF
 185     ;
 186 $config_help{"GRUB_MENU"} = << "EOF"
 187  The grub title name for the test kernel to boot
 188  (Only mandatory if REBOOT_TYPE = grub)
 189
 190  Note, ktest.pl will not update the grub menu.lst, you need to
 191  manually add an option for the test. ktest.pl will search
 192  the grub menu.lst for this option to find what kernel to
 193  reboot into.
 194
 195  For example, if in the /boot/grub/menu.lst the test kernel title has:
 196  title Test Kernel
 197  kernel vmlinuz-test
 198  GRUB_MENU = Test Kernel
 199 EOF
 200     ;
 201 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 202  A script to reboot the target into the test kernel
 203  (Only mandatory if REBOOT_TYPE = script)
 204 EOF
 205     ;
 206
 207
 208 sub get_ktest_config {
 209     my ($config) = @_;
 210
 211     return if (defined($opt{$config}));
 212
 213     if (defined($config_help{$config})) {
 214         print "\n";
 215         print $config_help{$config};
 216     }
 217
 218     for (;;) {
 219         print "$config = ";
 220         if (defined($default{$config})) {
 221             print "\[$default{$config}\] ";
 222         }
 223         $entered_configs{$config} = <STDIN>;
 224         $entered_configs{$config} =~ s/^\s*(.*\S)\s*$/$1/;
 225         if ($entered_configs{$config} =~ /^\s*$/) {
 226             if ($default{$config}) {
 227                 $entered_configs{$config} = $default{$config};
 228             } else {
 229                 print "Your answer can not be blank\n";
 230                 next;
 231             }
 232         }
 233         last;
 234     }
 235 }
 236
 237 sub get_ktest_configs {
 238     get_ktest_config("MACHINE");
 239     get_ktest_config("SSH_USER");
 240     get_ktest_config("BUILD_DIR");
 241     get_ktest_config("OUTPUT_DIR");
 242     get_ktest_config("BUILD_TARGET");
 243     get_ktest_config("TARGET_IMAGE");
 244     get_ktest_config("POWER_CYCLE");
 245     get_ktest_config("CONSOLE");
 246     get_ktest_config("LOCALVERSION");
 247
 248     my $rtype = $opt{"REBOOT_TYPE"};
 249
 250     if (!defined($rtype)) {
 251         if (!defined($opt{"GRUB_MENU"})) {
 252             get_ktest_config("REBOOT_TYPE");
 253             $rtype = $entered_configs{"REBOOT_TYPE"};
 254         } else {
 255             $rtype = "grub";
 256         }
 257     }
 258
 259     if ($rtype eq "grub") {
 260         get_ktest_config("GRUB_MENU");
 261     } else {
 262         get_ktest_config("REBOOT_SCRIPT");
 263     }
 264 }
 265
 266 sub process_variables {
 267     my ($value) = @_;
 268     my $retval = "";
 269
 270     # We want to check for '\', and it is just easier
 271     # to check the previous characet of '$' and not need
 272     # to worry if '$' is the first character. By adding
 273     # a space to $value, we can just check [^\\]\$ and
 274     # it will still work.
 275     $value = " $value";
 276
 277     while ($value =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 278         my $begin = $1;
 279         my $var = $2;
 280         my $end = $3;
 281         # append beginning of value to retval
 282         $retval = "$retval$begin";
 283         if (defined($variable{$var})) {
 284             $retval = "$retval$variable{$var}";
 285         } else {
 286             # put back the origin piece.
 287             $retval = "$retval\$\{$var\}";
 288         }
 289         $value = $end;
 290     }
 291     $retval = "$retval$value";
 292
 293     # remove the space added in the beginning
 294     $retval =~ s/ //;
 295
 296     return "$retval"
 297 }
 298
 299 sub set_value {
 300     my ($lvalue, $rvalue) = @_;
 301
 302     if (defined($opt{$lvalue})) {
 303         die "Error: Option $lvalue defined more than once!\n";
 304     }
 305     if ($rvalue =~ /^\s*$/) {
 306         delete $opt{$lvalue};
 307     } else {
 308         $rvalue = process_variables($rvalue);
 309         $opt{$lvalue} = $rvalue;
 310     }
 311 }
 312
 313 sub set_variable {
 314     my ($lvalue, $rvalue) = @_;
 315
 316     if ($rvalue =~ /^\s*$/) {
 317         delete $variable{$lvalue};
 318     } else {
 319         $rvalue = process_variables($rvalue);
 320         $variable{$lvalue} = $rvalue;
 321     }
 322 }
 323
 324 sub read_config {
 325     my ($config) = @_;
 326
 327     open(IN, $config) || die "can't read file $config";
 328
 329     my $name = $config;
 330     $name =~ s,.*/(.*),$1,;
 331
 332     my $test_num = 0;
 333     my $default = 1;
 334     my $repeat = 1;
 335     my $num_tests_set = 0;
 336     my $skip = 0;
 337     my $rest;
 338
 339     while (<IN>) {
 340
 341         # ignore blank lines and comments
 342         next if (/^\s*$/ || /\s*\#/);
 343
 344         if (/^\s*TEST_START(.*)/) {
 345
 346             $rest = $1;
 347
 348             if ($num_tests_set) {
 349                 die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 350             }
 351
 352             my $old_test_num = $test_num;
 353             my $old_repeat = $repeat;
 354
 355             $test_num += $repeat;
 356             $default = 0;
 357             $repeat = 1;
 358
 359             if ($rest =~ /\s+SKIP(.*)/) {
 360                 $rest = $1;
 361                 $skip = 1;
 362             } else {
 363                 $skip = 0;
 364             }
 365
 366             if ($rest =~ /\s+ITERATE\s+(\d+)(.*)$/) {
 367                 $repeat = $1;
 368                 $rest = $2;
 369                 $repeat_tests{"$test_num"} = $repeat;
 370             }
 371
 372             if ($rest =~ /\s+SKIP(.*)/) {
 373                 $rest = $1;
 374                 $skip = 1;
 375             }
 376
 377             if ($rest !~ /^\s*$/) {
 378                 die "$name: $.: Gargbage found after TEST_START\n$_";
 379             }
 380
 381             if ($skip) {
 382                 $test_num = $old_test_num;
 383                 $repeat = $old_repeat;
 384             }
 385
 386         } elsif (/^\s*DEFAULTS(.*)$/) {
 387             $default = 1;
 388
 389             $rest = $1;
 390
 391             if ($rest =~ /\s+SKIP(.*)/) {
 392                 $rest = $1;
 393                 $skip = 1;
 394             } else {
 395                 $skip = 0;
 396             }
 397
 398             if ($rest !~ /^\s*$/) {
 399                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 400             }
 401
 402         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 403
 404             next if ($skip);
 405
 406             my $lvalue = $1;
 407             my $rvalue = $2;
 408
 409             if (!$default &&
 410                 ($lvalue eq "NUM_TESTS" ||
 411                  $lvalue eq "LOG_FILE" ||
 412                  $lvalue eq "CLEAR_LOG")) {
 413                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 414             }
 415
 416             if ($lvalue eq "NUM_TESTS") {
 417                 if ($test_num) {
 418                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 419                 }
 420                 if (!$default) {
 421                     die "$name: $.: NUM_TESTS must be set in default section\n";
 422                 }
 423                 $num_tests_set = 1;
 424             }
 425
 426             if ($default || $lvalue =~ /\[\d+\]$/) {
 427                 set_value($lvalue, $rvalue);
 428             } else {
 429                 my $val = "$lvalue\[$test_num\]";
 430                 set_value($val, $rvalue);
 431
 432                 if ($repeat > 1) {
 433                     $repeats{$val} = $repeat;
 434                 }
 435             }
 436         } elsif (/^\s*([A-Z_\[\]\d]+)\s*:=\s*(.*?)\s*$/) {
 437             next if ($skip);
 438
 439             my $lvalue = $1;
 440             my $rvalue = $2;
 441
 442             # process config variables.
 443             # Config variables are only active while reading the
 444             # config and can be defined anywhere. They also ignore
 445             # TEST_START and DEFAULTS, but are skipped if they are in
 446             # on of these sections that have SKIP defined.
 447             # The save variable can be
 448             # defined multiple times and the new one simply overrides
 449             # the prevous one.
 450             set_variable($lvalue, $rvalue);
 451
 452         } else {
 453             die "$name: $.: Garbage found in config\n$_";
 454         }
 455     }
 456
 457     close(IN);
 458
 459     if ($test_num) {
 460         $test_num += $repeat - 1;
 461         $opt{"NUM_TESTS"} = $test_num;
 462     }
 463
 464     # make sure we have all mandatory configs
 465     get_ktest_configs;
 466
 467     # set any defaults
 468
 469     foreach my $default (keys %default) {
 470         if (!defined($opt{$default})) {
 471             $opt{$default} = $default{$default};
 472         }
 473     }
 474 }
 475
 476 sub _logit {
 477     if (defined($opt{"LOG_FILE"})) {
 478         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
 479         print OUT @_;
 480         close(OUT);
 481     }
 482 }
 483
 484 sub logit {
 485     if (defined($opt{"LOG_FILE"})) {
 486         _logit @_;
 487     } else {
 488         print @_;
 489     }
 490 }
 491
 492 sub doprint {
 493     print @_;
 494     _logit @_;
 495 }
 496
 497 sub run_command;
 498
 499 sub reboot {
 500     # try to reboot normally
 501     if (run_command $reboot) {
 502         if (defined($powercycle_after_reboot)) {
 503             sleep $powercycle_after_reboot;
 504             run_command "$power_cycle";
 505         }
 506     } else {
 507         # nope? power cycle it.
 508         run_command "$power_cycle";
 509     }
 510 }
 511
 512 sub do_not_reboot {
 513     my $i = $iteration;
 514
 515     return $test_type eq "build" ||
 516         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
 517         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
 518 }
 519
 520 sub dodie {
 521     doprint "CRITICAL FAILURE... ", @_, "\n";
 522
 523     my $i = $iteration;
 524
 525     if ($reboot_on_error && !do_not_reboot) {
 526
 527         doprint "REBOOTING\n";
 528         reboot;
 529
 530     } elsif ($poweroff_on_error && defined($power_off)) {
 531         doprint "POWERING OFF\n";
 532         `$power_off`;
 533     }
 534
 535     if (defined($opt{"LOG_FILE"})) {
 536         print " See $opt{LOG_FILE} for more info.\n";
 537     }
 538
 539     die @_, "\n";
 540 }
 541
 542 sub open_console {
 543     my ($fp) = @_;
 544
 545     my $flags;
 546
 547     my $pid = open($fp, "$console|") or
 548         dodie "Can't open console $console";
 549
 550     $flags = fcntl($fp, F_GETFL, 0) or
 551         dodie "Can't get flags for the socket: $!";
 552     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 553         dodie "Can't set flags for the socket: $!";
 554
 555     return $pid;
 556 }
 557
 558 sub close_console {
 559     my ($fp, $pid) = @_;
 560
 561     doprint "kill child process $pid\n";
 562     kill 2, $pid;
 563
 564     print "closing!\n";
 565     close($fp);
 566 }
 567
 568 sub start_monitor {
 569     if ($monitor_cnt++) {
 570         return;
 571     }
 572     $monitor_fp = \*MONFD;
 573     $monitor_pid = open_console $monitor_fp;
 574
 575     return;
 576
 577     open(MONFD, "Stop perl from warning about single use of MONFD");
 578 }
 579
 580 sub end_monitor {
 581     if (--$monitor_cnt) {
 582         return;
 583     }
 584     close_console($monitor_fp, $monitor_pid);
 585 }
 586
 587 sub wait_for_monitor {
 588     my ($time) = @_;
 589     my $line;
 590
 591     doprint "** Wait for monitor to settle down **\n";
 592
 593     # read the monitor and wait for the system to calm down
 594     do {
 595         $line = wait_for_input($monitor_fp, $time);
 596         print "$line" if (defined($line));
 597     } while (defined($line));
 598     print "** Monitor flushed **\n";
 599 }
 600
 601 sub fail {
 602
 603         if ($die_on_failure) {
 604                 dodie @_;
 605         }
 606
 607         doprint "FAILED\n";
 608
 609         my $i = $iteration;
 610
 611         # no need to reboot for just building.
 612         if (!do_not_reboot) {
 613             doprint "REBOOTING\n";
 614             reboot;
 615             start_monitor;
 616             wait_for_monitor $sleep_time;
 617             end_monitor;
 618         }
 619
 620         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 621         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 622         doprint "KTEST RESULT: TEST $i Failed: ", @_, "\n";
 623         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 624         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 625
 626         return 1 if (!defined($store_failures));
 627
 628         my @t = localtime;
 629         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
 630                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
 631
 632         my $type = $build_type;
 633         if ($type =~ /useconfig/) {
 634             $type = "useconfig";
 635         }
 636
 637         my $dir = "$machine-$test_type-$type-fail-$date";
 638         my $faildir = "$store_failures/$dir";
 639
 640         if (!-d $faildir) {
 641             mkpath($faildir) or
 642                 die "can't create $faildir";
 643         }
 644         if (-f "$output_config") {
 645             cp "$output_config", "$faildir/config" or
 646                 die "failed to copy .config";
 647         }
 648         if (-f $buildlog) {
 649             cp $buildlog, "$faildir/buildlog" or
 650                 die "failed to move $buildlog";
 651         }
 652         if (-f $dmesg) {
 653             cp $dmesg, "$faildir/dmesg" or
 654                 die "failed to move $dmesg";
 655         }
 656
 657         doprint "*** Saved info to $faildir ***\n";
 658
 659         return 1;
 660 }
 661
 662 sub run_command {
 663     my ($command) = @_;
 664     my $dolog = 0;
 665     my $dord = 0;
 666     my $pid;
 667
 668     $command =~ s/\$SSH_USER/$ssh_user/g;
 669     $command =~ s/\$MACHINE/$machine/g;
 670
 671     doprint("$command ... ");
 672
 673     $pid = open(CMD, "$command 2>&1 |") or
 674         (fail "unable to exec $command" and return 0);
 675
 676     if (defined($opt{"LOG_FILE"})) {
 677         open(LOG, ">>$opt{LOG_FILE}") or
 678             dodie "failed to write to log";
 679         $dolog = 1;
 680     }
 681
 682     if (defined($redirect)) {
 683         open (RD, ">$redirect") or
 684             dodie "failed to write to redirect $redirect";
 685         $dord = 1;
 686     }
 687
 688     while (<CMD>) {
 689         print LOG if ($dolog);
 690         print RD  if ($dord);
 691     }
 692
 693     waitpid($pid, 0);
 694     my $failed = $?;
 695
 696     close(CMD);
 697     close(LOG) if ($dolog);
 698     close(RD)  if ($dord);
 699
 700     if ($failed) {
 701         doprint "FAILED!\n";
 702     } else {
 703         doprint "SUCCESS\n";
 704     }
 705
 706     return !$failed;
 707 }
 708
 709 sub run_ssh {
 710     my ($cmd) = @_;
 711     my $cp_exec = $ssh_exec;
 712
 713     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
 714     return run_command "$cp_exec";
 715 }
 716
 717 sub run_scp {
 718     my ($src, $dst) = @_;
 719     my $cp_scp = $scp_to_target;
 720
 721     $cp_scp =~ s/\$SRC_FILE/$src/g;
 722     $cp_scp =~ s/\$DST_FILE/$dst/g;
 723
 724     return run_command "$cp_scp";
 725 }
 726
 727 sub get_grub_index {
 728
 729     if ($reboot_type ne "grub") {
 730         return;
 731     }
 732     return if (defined($grub_number));
 733
 734     doprint "Find grub menu ... ";
 735     $grub_number = -1;
 736
 737     my $ssh_grub = $ssh_exec;
 738     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
 739
 740     open(IN, "$ssh_grub |")
 741         or die "unable to get menu.lst";
 742
 743     while (<IN>) {
 744         if (/^\s*title\s+$grub_menu\s*$/) {
 745             $grub_number++;
 746             last;
 747         } elsif (/^\s*title\s/) {
 748             $grub_number++;
 749         }
 750     }
 751     close(IN);
 752
 753     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
 754         if ($grub_number < 0);
 755     doprint "$grub_number\n";
 756 }
 757
 758 sub wait_for_input
 759 {
 760     my ($fp, $time) = @_;
 761     my $rin;
 762     my $ready;
 763     my $line;
 764     my $ch;
 765
 766     if (!defined($time)) {
 767         $time = $timeout;
 768     }
 769
 770     $rin = '';
 771     vec($rin, fileno($fp), 1) = 1;
 772     $ready = select($rin, undef, undef, $time);
 773
 774     $line = "";
 775
 776     # try to read one char at a time
 777     while (sysread $fp, $ch, 1) {
 778         $line .= $ch;
 779         last if ($ch eq "\n");
 780     }
 781
 782     if (!length($line)) {
 783         return undef;
 784     }
 785
 786     return $line;
 787 }
 788
 789 sub reboot_to {
 790     if ($reboot_type eq "grub") {
 791         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch && reboot)'";
 792         return;
 793     }
 794
 795     run_command "$reboot_script";
 796 }
 797
 798 sub get_sha1 {
 799     my ($commit) = @_;
 800
 801     doprint "git rev-list --max-count=1 $commit ... ";
 802     my $sha1 = `git rev-list --max-count=1 $commit`;
 803     my $ret = $?;
 804
 805     logit $sha1;
 806
 807     if ($ret) {
 808         doprint "FAILED\n";
 809         dodie "Failed to get git $commit";
 810     }
 811
 812     print "SUCCESS\n";
 813
 814     chomp $sha1;
 815
 816     return $sha1;
 817 }
 818
 819 sub monitor {
 820     my $booted = 0;
 821     my $bug = 0;
 822     my $skip_call_trace = 0;
 823     my $loops;
 824
 825     wait_for_monitor 5;
 826
 827     my $line;
 828     my $full_line = "";
 829
 830     open(DMESG, "> $dmesg") or
 831         die "unable to write to $dmesg";
 832
 833     reboot_to;
 834
 835     my $success_start;
 836     my $failure_start;
 837     my $monitor_start = time;
 838     my $done = 0;
 839
 840     while (!$done) {
 841
 842         if ($booted) {
 843             $line = wait_for_input($monitor_fp, $booted_timeout);
 844         } else {
 845             $line = wait_for_input($monitor_fp);
 846         }
 847
 848         last if (!defined($line));
 849
 850         doprint $line;
 851         print DMESG $line;
 852
 853         # we are not guaranteed to get a full line
 854         $full_line .= $line;
 855
 856         if ($full_line =~ /$success_line/) {
 857             $booted = 1;
 858             $success_start = time;
 859         }
 860
 861         if ($booted && defined($stop_after_success) &&
 862             $stop_after_success >= 0) {
 863             my $now = time;
 864             if ($now - $success_start >= $stop_after_success) {
 865                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
 866                 last;
 867             }
 868         }
 869
 870         if ($full_line =~ /\[ backtrace testing \]/) {
 871             $skip_call_trace = 1;
 872         }
 873
 874         if ($full_line =~ /call trace:/i) {
 875             if (!$bug && !$skip_call_trace) {
 876                 $bug = 1;
 877                 $failure_start = time;
 878             }
 879         }
 880
 881         if ($bug && defined($stop_after_failure) &&
 882             $stop_after_failure >= 0) {
 883             my $now = time;
 884             if ($now - $failure_start >= $stop_after_failure) {
 885                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
 886                 last;
 887             }
 888         }
 889
 890         if ($full_line =~ /\[ end of backtrace testing \]/) {
 891             $skip_call_trace = 0;
 892         }
 893
 894         if ($full_line =~ /Kernel panic -/) {
 895             $failure_start = time;
 896             $bug = 1;
 897         }
 898
 899         if ($line =~ /\n/) {
 900             $full_line = "";
 901         }
 902
 903         if ($stop_test_after > 0 && !$booted && !$bug) {
 904             if (time - $monitor_start > $stop_test_after) {
 905                 doprint "STOP_TEST_AFTER ($stop_test_after seconds) timed out\n";
 906                 $done = 1;
 907             }
 908         }
 909     }
 910
 911     close(DMESG);
 912
 913     if ($bug) {
 914         return 0 if ($in_bisect);
 915         fail "failed - got a bug report" and return 0;
 916     }
 917
 918     if (!$booted) {
 919         return 0 if ($in_bisect);
 920         fail "failed - never got a boot prompt." and return 0;
 921     }
 922
 923     return 1;
 924 }
 925
 926 sub install {
 927
 928     run_scp "$outputdir/$build_target", "$target_image" or
 929         dodie "failed to copy image";
 930
 931     my $install_mods = 0;
 932
 933     # should we process modules?
 934     $install_mods = 0;
 935     open(IN, "$output_config") or dodie("Can't read config file");
 936     while (<IN>) {
 937         if (/CONFIG_MODULES(=y)?/) {
 938             $install_mods = 1 if (defined($1));
 939             last;
 940         }
 941     }
 942     close(IN);
 943
 944     if (!$install_mods) {
 945         doprint "No modules needed\n";
 946         return;
 947     }
 948
 949     run_command "$make INSTALL_MOD_PATH=$tmpdir modules_install" or
 950         dodie "Failed to install modules";
 951
 952     my $modlib = "/lib/modules/$version";
 953     my $modtar = "ktest-mods.tar.bz2";
 954
 955     run_ssh "rm -rf $modlib" or
 956         dodie "failed to remove old mods: $modlib";
 957
 958     # would be nice if scp -r did not follow symbolic links
 959     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
 960         dodie "making tarball";
 961
 962     run_scp "$tmpdir/$modtar", "/tmp" or
 963         dodie "failed to copy modules";
 964
 965     unlink "$tmpdir/$modtar";
 966
 967     run_ssh "'(cd / && tar xf /tmp/$modtar)'" or
 968         dodie "failed to tar modules";
 969
 970     run_ssh "rm -f /tmp/$modtar";
 971
 972     return if (!defined($post_install));
 973
 974     my $cp_post_install = $post_install;
 975     $cp_post_install =~ s/\$KERNEL_VERSION/$version/g;
 976     run_command "$cp_post_install" or
 977         dodie "Failed to run post install";
 978 }
 979
 980 sub check_buildlog {
 981     my ($patch) = @_;
 982
 983     my @files = `git show $patch | diffstat -l`;
 984
 985     open(IN, "git show $patch |") or
 986         dodie "failed to show $patch";
 987     while (<IN>) {
 988         if (m,^--- a/(.*),) {
 989             chomp $1;
 990             $files[$#files] = $1;
 991         }
 992     }
 993     close(IN);
 994
 995     open(IN, $buildlog) or dodie "Can't open $buildlog";
 996     while (<IN>) {
 997         if (/^\s*(.*?):.*(warning|error)/) {
 998             my $err = $1;
 999             foreach my $file (@files) {
1000                 my $fullpath = "$builddir/$file";
1001                 if ($file eq $err || $fullpath eq $err) {
1002                     fail "$file built with warnings" and return 0;
1003                 }
1004             }
1005         }
1006     }
1007     close(IN);
1008
1009     return 1;
1010 }
1011
1012 sub make_oldconfig {
1013     my ($defconfig) = @_;
1014
1015     if (!run_command "$defconfig $make oldnoconfig") {
1016         # Perhaps oldnoconfig doesn't exist in this version of the kernel
1017         # try a yes '' | oldconfig
1018         doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
1019         run_command "yes '' | $defconfig $make oldconfig" or
1020             dodie "failed make config oldconfig";
1021     }
1022 }
1023
1024 sub build {
1025     my ($type) = @_;
1026     my $defconfig = "";
1027
1028     unlink $buildlog;
1029
1030     if ($type =~ /^useconfig:(.*)/) {
1031         run_command "cp $1 $output_config" or
1032             dodie "could not copy $1 to .config";
1033
1034         $type = "oldconfig";
1035     }
1036
1037     # old config can ask questions
1038     if ($type eq "oldconfig") {
1039         $type = "oldnoconfig";
1040
1041         # allow for empty configs
1042         run_command "touch $output_config";
1043
1044         run_command "mv $output_config $outputdir/config_temp" or
1045             dodie "moving .config";
1046
1047         if (!$noclean && !run_command "$make mrproper") {
1048             dodie "make mrproper";
1049         }
1050
1051         run_command "mv $outputdir/config_temp $output_config" or
1052             dodie "moving config_temp";
1053
1054     } elsif (!$noclean) {
1055         unlink "$output_config";
1056         run_command "$make mrproper" or
1057             dodie "make mrproper";
1058     }
1059
1060     # add something to distinguish this build
1061     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
1062     print OUT "$localversion\n";
1063     close(OUT);
1064
1065     if (defined($minconfig)) {
1066         $defconfig = "KCONFIG_ALLCONFIG=$minconfig";
1067     }
1068
1069     if ($type eq "oldnoconfig") {
1070         make_oldconfig $defconfig;
1071     } else {
1072         run_command "$defconfig $make $type" or
1073             dodie "failed make config";
1074     }
1075
1076     $redirect = "$buildlog";
1077     if (!run_command "$make $build_options") {
1078         undef $redirect;
1079         # bisect may need this to pass
1080         return 0 if ($in_bisect);
1081         fail "failed build" and return 0;
1082     }
1083     undef $redirect;
1084
1085     return 1;
1086 }
1087
1088 sub halt {
1089     if (!run_ssh "halt" or defined($power_off)) {
1090         if (defined($poweroff_after_halt)) {
1091             sleep $poweroff_after_halt;
1092             run_command "$power_off";
1093         }
1094     } else {
1095         # nope? the zap it!
1096         run_command "$power_off";
1097     }
1098 }
1099
1100 sub success {
1101     my ($i) = @_;
1102
1103     $successes++;
1104
1105     doprint "\n\n*******************************************\n";
1106     doprint     "*******************************************\n";
1107     doprint     "KTEST RESULT: TEST $i SUCCESS!!!!         **\n";
1108     doprint     "*******************************************\n";
1109     doprint     "*******************************************\n";
1110
1111     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
1112         doprint "Reboot and wait $sleep_time seconds\n";
1113         reboot;
1114         start_monitor;
1115         wait_for_monitor $sleep_time;
1116         end_monitor;
1117     }
1118 }
1119
1120 sub get_version {
1121     # get the release name
1122     doprint "$make kernelrelease ... ";
1123     $version = `$make kernelrelease | tail -1`;
1124     chomp($version);
1125     doprint "$version\n";
1126 }
1127
1128 sub answer_bisect {
1129     for (;;) {
1130         doprint "Pass or fail? [p/f]";
1131         my $ans = <STDIN>;
1132         chomp $ans;
1133         if ($ans eq "p" || $ans eq "P") {
1134             return 1;
1135         } elsif ($ans eq "f" || $ans eq "F") {
1136             return 0;
1137         } else {
1138             print "Please answer 'P' or 'F'\n";
1139         }
1140     }
1141 }
1142
1143 sub child_run_test {
1144     my $failed = 0;
1145
1146     # child should have no power
1147     $reboot_on_error = 0;
1148     $poweroff_on_error = 0;
1149     $die_on_failure = 1;
1150
1151     run_command $run_test or $failed = 1;
1152     exit $failed;
1153 }
1154
1155 my $child_done;
1156
1157 sub child_finished {
1158     $child_done = 1;
1159 }
1160
1161 sub do_run_test {
1162     my $child_pid;
1163     my $child_exit;
1164     my $line;
1165     my $full_line;
1166     my $bug = 0;
1167
1168     wait_for_monitor 1;
1169
1170     doprint "run test $run_test\n";
1171
1172     $child_done = 0;
1173
1174     $SIG{CHLD} = qw(child_finished);
1175
1176     $child_pid = fork;
1177
1178     child_run_test if (!$child_pid);
1179
1180     $full_line = "";
1181
1182     do {
1183         $line = wait_for_input($monitor_fp, 1);
1184         if (defined($line)) {
1185
1186             # we are not guaranteed to get a full line
1187             $full_line .= $line;
1188             doprint $line;
1189
1190             if ($full_line =~ /call trace:/i) {
1191                 $bug = 1;
1192             }
1193
1194             if ($full_line =~ /Kernel panic -/) {
1195                 $bug = 1;
1196             }
1197
1198             if ($line =~ /\n/) {
1199                 $full_line = "";
1200             }
1201         }
1202     } while (!$child_done && !$bug);
1203
1204     if ($bug) {
1205         my $failure_start = time;
1206         my $now;
1207         do {
1208             $line = wait_for_input($monitor_fp, 1);
1209             if (defined($line)) {
1210                 doprint $line;
1211             }
1212             $now = time;
1213             if ($now - $failure_start >= $stop_after_failure) {
1214                 last;
1215             }
1216         } while (defined($line));
1217
1218         doprint "Detected kernel crash!\n";
1219         # kill the child with extreme prejudice
1220         kill 9, $child_pid;
1221     }
1222
1223     waitpid $child_pid, 0;
1224     $child_exit = $?;
1225
1226     if ($bug || $child_exit) {
1227         return 0 if $in_bisect;
1228         fail "test failed" and return 0;
1229     }
1230     return 1;
1231 }
1232
1233 sub run_git_bisect {
1234     my ($command) = @_;
1235
1236     doprint "$command ... ";
1237
1238     my $output = `$command 2>&1`;
1239     my $ret = $?;
1240
1241     logit $output;
1242
1243     if ($ret) {
1244         doprint "FAILED\n";
1245         dodie "Failed to git bisect";
1246     }
1247
1248     doprint "SUCCESS\n";
1249     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
1250         doprint "$1 [$2]\n";
1251     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
1252         $bisect_bad = $1;
1253         doprint "Found bad commit... $1\n";
1254         return 0;
1255     } else {
1256         # we already logged it, just print it now.
1257         print $output;
1258     }
1259
1260     return 1;
1261 }
1262
1263 sub bisect_reboot {
1264     doprint "Reboot and sleep $bisect_sleep_time seconds\n";
1265     reboot;
1266     start_monitor;
1267     wait_for_monitor $bisect_sleep_time;
1268     end_monitor;
1269 }
1270
1271 # returns 1 on success, 0 on failure, -1 on skip
1272 sub run_bisect_test {
1273     my ($type, $buildtype) = @_;
1274
1275     my $failed = 0;
1276     my $result;
1277     my $output;
1278     my $ret;
1279
1280     $in_bisect = 1;
1281
1282     build $buildtype or $failed = 1;
1283
1284     if ($type ne "build") {
1285         if ($failed && $bisect_skip) {
1286             $in_bisect = 0;
1287             return -1;
1288         }
1289         dodie "Failed on build" if $failed;
1290
1291         # Now boot the box
1292         get_grub_index;
1293         get_version;
1294         install;
1295
1296         start_monitor;
1297         monitor or $failed = 1;
1298
1299         if ($type ne "boot") {
1300             if ($failed && $bisect_skip) {
1301                 end_monitor;
1302                 bisect_reboot;
1303                 $in_bisect = 0;
1304                 return -1;
1305             }
1306             dodie "Failed on boot" if $failed;
1307
1308             do_run_test or $failed = 1;
1309         }
1310         end_monitor;
1311     }
1312
1313     if ($failed) {
1314         $result = 0;
1315     } else {
1316         $result = 1;
1317     }
1318
1319     # reboot the box to a kernel we can ssh to
1320     if ($type ne "build") {
1321         bisect_reboot;
1322     }
1323     $in_bisect = 0;
1324
1325     return $result;
1326 }
1327
1328 sub run_bisect {
1329     my ($type) = @_;
1330     my $buildtype = "oldconfig";
1331
1332     # We should have a minconfig to use?
1333     if (defined($minconfig)) {
1334         $buildtype = "useconfig:$minconfig";
1335     }
1336
1337     my $ret = run_bisect_test $type, $buildtype;
1338
1339     if ($bisect_manual) {
1340         $ret = answer_bisect;
1341     }
1342
1343     # Are we looking for where it worked, not failed?
1344     if ($reverse_bisect) {
1345         $ret = !$ret;
1346     }
1347
1348     if ($ret > 0) {
1349         return "good";
1350     } elsif ($ret == 0) {
1351         return  "bad";
1352     } elsif ($bisect_skip) {
1353         doprint "HIT A BAD COMMIT ... SKIPPING\n";
1354         return "skip";
1355     }
1356 }
1357
1358 sub bisect {
1359     my ($i) = @_;
1360
1361     my $result;
1362
1363     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
1364     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
1365     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
1366
1367     my $good = $opt{"BISECT_GOOD[$i]"};
1368     my $bad = $opt{"BISECT_BAD[$i]"};
1369     my $type = $opt{"BISECT_TYPE[$i]"};
1370     my $start = $opt{"BISECT_START[$i]"};
1371     my $replay = $opt{"BISECT_REPLAY[$i]"};
1372     my $start_files = $opt{"BISECT_FILES[$i]"};
1373
1374     if (defined($start_files)) {
1375         $start_files = " -- " . $start_files;
1376     } else {
1377         $start_files = "";
1378     }
1379
1380     # convert to true sha1's
1381     $good = get_sha1($good);
1382     $bad = get_sha1($bad);
1383
1384     if (defined($opt{"BISECT_REVERSE[$i]"}) &&
1385         $opt{"BISECT_REVERSE[$i]"} == 1) {
1386         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
1387         $reverse_bisect = 1;
1388     } else {
1389         $reverse_bisect = 0;
1390     }
1391
1392     # Can't have a test without having a test to run
1393     if ($type eq "test" && !defined($run_test)) {
1394         $type = "boot";
1395     }
1396
1397     my $check = $opt{"BISECT_CHECK[$i]"};
1398     if (defined($check) && $check ne "0") {
1399
1400         # get current HEAD
1401         my $head = get_sha1("HEAD");
1402
1403         if ($check ne "good") {
1404             doprint "TESTING BISECT BAD [$bad]\n";
1405             run_command "git checkout $bad" or
1406                 die "Failed to checkout $bad";
1407
1408             $result = run_bisect $type;
1409
1410             if ($result ne "bad") {
1411                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
1412             }
1413         }
1414
1415         if ($check ne "bad") {
1416             doprint "TESTING BISECT GOOD [$good]\n";
1417             run_command "git checkout $good" or
1418                 die "Failed to checkout $good";
1419
1420             $result = run_bisect $type;
1421
1422             if ($result ne "good") {
1423                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
1424             }
1425         }
1426
1427         # checkout where we started
1428         run_command "git checkout $head" or
1429             die "Failed to checkout $head";
1430     }
1431
1432     run_command "git bisect start$start_files" or
1433         dodie "could not start bisect";
1434
1435     run_command "git bisect good $good" or
1436         dodie "could not set bisect good to $good";
1437
1438     run_git_bisect "git bisect bad $bad" or
1439         dodie "could not set bisect bad to $bad";
1440
1441     if (defined($replay)) {
1442         run_command "git bisect replay $replay" or
1443             dodie "failed to run replay";
1444     }
1445
1446     if (defined($start)) {
1447         run_command "git checkout $start" or
1448             dodie "failed to checkout $start";
1449     }
1450
1451     my $test;
1452     do {
1453         $result = run_bisect $type;
1454         $test = run_git_bisect "git bisect $result";
1455     } while ($test);
1456
1457     run_command "git bisect log" or
1458         dodie "could not capture git bisect log";
1459
1460     run_command "git bisect reset" or
1461         dodie "could not reset git bisect";
1462
1463     doprint "Bad commit was [$bisect_bad]\n";
1464
1465     success $i;
1466 }
1467
1468 my %config_ignore;
1469 my %config_set;
1470
1471 my %config_list;
1472 my %null_config;
1473
1474 my %dependency;
1475
1476 sub process_config_ignore {
1477     my ($config) = @_;
1478
1479     open (IN, $config)
1480         or dodie "Failed to read $config";
1481
1482     while (<IN>) {
1483         if (/^((CONFIG\S*)=.*)/) {
1484             $config_ignore{$2} = $1;
1485         }
1486     }
1487
1488     close(IN);
1489 }
1490
1491 sub read_current_config {
1492     my ($config_ref) = @_;
1493
1494     %{$config_ref} = ();
1495     undef %{$config_ref};
1496
1497     my @key = keys %{$config_ref};
1498     if ($#key >= 0) {
1499         print "did not delete!\n";
1500         exit;
1501     }
1502     open (IN, "$output_config");
1503
1504     while (<IN>) {
1505         if (/^(CONFIG\S+)=(.*)/) {
1506             ${$config_ref}{$1} = $2;
1507         }
1508     }
1509     close(IN);
1510 }
1511
1512 sub get_dependencies {
1513     my ($config) = @_;
1514
1515     my $arr = $dependency{$config};
1516     if (!defined($arr)) {
1517         return ();
1518     }
1519
1520     my @deps = @{$arr};
1521
1522     foreach my $dep (@{$arr}) {
1523         print "ADD DEP $dep\n";
1524         @deps = (@deps, get_dependencies $dep);
1525     }
1526
1527     return @deps;
1528 }
1529
1530 sub create_config {
1531     my @configs = @_;
1532
1533     open(OUT, ">$output_config") or dodie "Can not write to $output_config";
1534
1535     foreach my $config (@configs) {
1536         print OUT "$config_set{$config}\n";
1537         my @deps = get_dependencies $config;
1538         foreach my $dep (@deps) {
1539             print OUT "$config_set{$dep}\n";
1540         }
1541     }
1542
1543     foreach my $config (keys %config_ignore) {
1544         print OUT "$config_ignore{$config}\n";
1545     }
1546     close(OUT);
1547
1548 #    exit;
1549     make_oldconfig "";
1550 }
1551
1552 sub compare_configs {
1553     my (%a, %b) = @_;
1554
1555     foreach my $item (keys %a) {
1556         if (!defined($b{$item})) {
1557             print "diff $item\n";
1558             return 1;
1559         }
1560         delete $b{$item};
1561     }
1562
1563     my @keys = keys %b;
1564     if ($#keys) {
1565         print "diff2 $keys[0]\n";
1566     }
1567     return -1 if ($#keys >= 0);
1568
1569     return 0;
1570 }
1571
1572 sub run_config_bisect_test {
1573     my ($type) = @_;
1574
1575     return run_bisect_test $type, "oldconfig";
1576 }
1577
1578 sub process_passed {
1579     my (%configs) = @_;
1580
1581     doprint "These configs had no failure: (Enabling them for further compiles)\n";
1582     # Passed! All these configs are part of a good compile.
1583     # Add them to the min options.
1584     foreach my $config (keys %configs) {
1585         if (defined($config_list{$config})) {
1586             doprint " removing $config\n";
1587             $config_ignore{$config} = $config_list{$config};
1588             delete $config_list{$config};
1589         }
1590     }
1591     doprint "config copied to $outputdir/config_good\n";
1592     run_command "cp -f $output_config $outputdir/config_good";
1593 }
1594
1595 sub process_failed {
1596     my ($config) = @_;
1597
1598     doprint "\n\n***************************************\n";
1599     doprint "Found bad config: $config\n";
1600     doprint "***************************************\n\n";
1601 }
1602
1603 sub run_config_bisect {
1604
1605     my @start_list = keys %config_list;
1606
1607     if ($#start_list < 0) {
1608         doprint "No more configs to test!!!\n";
1609         return -1;
1610     }
1611
1612     doprint "***** RUN TEST ***\n";
1613     my $type = $opt{"CONFIG_BISECT_TYPE[$iteration]"};
1614     my $ret;
1615     my %current_config;
1616
1617     my $count = $#start_list + 1;
1618     doprint "  $count configs to test\n";
1619
1620     my $half = int($#start_list / 2);
1621
1622     do {
1623         my @tophalf = @start_list[0 .. $half];
1624
1625         create_config @tophalf;
1626         read_current_config \%current_config;
1627
1628         $count = $#tophalf + 1;
1629         doprint "Testing $count configs\n";
1630         my $found = 0;
1631         # make sure we test something
1632         foreach my $config (@tophalf) {
1633             if (defined($current_config{$config})) {
1634                 logit " $config\n";
1635                 $found = 1;
1636             }
1637         }
1638         if (!$found) {
1639             # try the other half
1640             doprint "Top half produced no set configs, trying bottom half\n";
1641             @tophalf = @start_list[$half + 1 .. $#start_list];
1642             create_config @tophalf;
1643             read_current_config \%current_config;
1644             foreach my $config (@tophalf) {
1645                 if (defined($current_config{$config})) {
1646                     logit " $config\n";
1647                     $found = 1;
1648                 }
1649             }
1650             if (!$found) {
1651                 doprint "Failed: Can't make new config with current configs\n";
1652                 foreach my $config (@start_list) {
1653                     doprint "  CONFIG: $config\n";
1654                 }
1655                 return -1;
1656             }
1657             $count = $#tophalf + 1;
1658             doprint "Testing $count configs\n";
1659         }
1660
1661         $ret = run_config_bisect_test $type;
1662         if ($bisect_manual) {
1663             $ret = answer_bisect;
1664         }
1665         if ($ret) {
1666             process_passed %current_config;
1667             return 0;
1668         }
1669
1670         doprint "This config had a failure.\n";
1671         doprint "Removing these configs that were not set in this config:\n";
1672         doprint "config copied to $outputdir/config_bad\n";
1673         run_command "cp -f $output_config $outputdir/config_bad";
1674
1675         # A config exists in this group that was bad.
1676         foreach my $config (keys %config_list) {
1677             if (!defined($current_config{$config})) {
1678                 doprint " removing $config\n";
1679                 delete $config_list{$config};
1680             }
1681         }
1682
1683         @start_list = @tophalf;
1684
1685         if ($#start_list == 0) {
1686             process_failed $start_list[0];
1687             return 1;
1688         }
1689
1690         # remove half the configs we are looking at and see if
1691         # they are good.
1692         $half = int($#start_list / 2);
1693     } while ($#start_list > 0);
1694
1695     # we found a single config, try it again unless we are running manually
1696
1697     if ($bisect_manual) {
1698         process_failed $start_list[0];
1699         return 1;
1700     }
1701
1702     my @tophalf = @start_list[0 .. 0];
1703
1704     $ret = run_config_bisect_test $type;
1705     if ($ret) {
1706         process_passed %current_config;
1707         return 0;
1708     }
1709
1710     process_failed $start_list[0];
1711     return 1;
1712 }
1713
1714 sub config_bisect {
1715     my ($i) = @_;
1716
1717     my $start_config = $opt{"CONFIG_BISECT[$i]"};
1718
1719     my $tmpconfig = "$tmpdir/use_config";
1720
1721     # Make the file with the bad config and the min config
1722     if (defined($minconfig)) {
1723         # read the min config for things to ignore
1724         run_command "cp $minconfig $tmpconfig" or
1725             dodie "failed to copy $minconfig to $tmpconfig";
1726     } else {
1727         unlink $tmpconfig;
1728     }
1729
1730     # Add other configs
1731     if (defined($addconfig)) {
1732         run_command "cat $addconfig >> $tmpconfig" or
1733             dodie "failed to append $addconfig";
1734     }
1735
1736     my $defconfig = "";
1737     if (-f $tmpconfig) {
1738         $defconfig = "KCONFIG_ALLCONFIG=$tmpconfig";
1739         process_config_ignore $tmpconfig;
1740     }
1741
1742     # now process the start config
1743     run_command "cp $start_config $output_config" or
1744         dodie "failed to copy $start_config to $output_config";
1745
1746     # read directly what we want to check
1747     my %config_check;
1748     open (IN, $output_config)
1749         or dodie "faied to open $output_config";
1750
1751     while (<IN>) {
1752         if (/^((CONFIG\S*)=.*)/) {
1753             $config_check{$2} = $1;
1754         }
1755     }
1756     close(IN);
1757
1758     # Now run oldconfig with the minconfig (and addconfigs)
1759     make_oldconfig $defconfig;
1760
1761     # check to see what we lost (or gained)
1762     open (IN, $output_config)
1763         or dodie "Failed to read $start_config";
1764
1765     my %removed_configs;
1766     my %added_configs;
1767
1768     while (<IN>) {
1769         if (/^((CONFIG\S*)=.*)/) {
1770             # save off all options
1771             $config_set{$2} = $1;
1772             if (defined($config_check{$2})) {
1773                 if (defined($config_ignore{$2})) {
1774                     $removed_configs{$2} = $1;
1775                 } else {
1776                     $config_list{$2} = $1;
1777                 }
1778             } elsif (!defined($config_ignore{$2})) {
1779                 $added_configs{$2} = $1;
1780                 $config_list{$2} = $1;
1781             }
1782         }
1783     }
1784     close(IN);
1785
1786     my @confs = keys %removed_configs;
1787     if ($#confs >= 0) {
1788         doprint "Configs overridden by default configs and removed from check:\n";
1789         foreach my $config (@confs) {
1790             doprint " $config\n";
1791         }
1792     }
1793     @confs = keys %added_configs;
1794     if ($#confs >= 0) {
1795         doprint "Configs appearing in make oldconfig and added:\n";
1796         foreach my $config (@confs) {
1797             doprint " $config\n";
1798         }
1799     }
1800
1801     my %config_test;
1802     my $once = 0;
1803
1804     # Sometimes kconfig does weird things. We must make sure
1805     # that the config we autocreate has everything we need
1806     # to test, otherwise we may miss testing configs, or
1807     # may not be able to create a new config.
1808     # Here we create a config with everything set.
1809     create_config (keys %config_list);
1810     read_current_config \%config_test;
1811     foreach my $config (keys %config_list) {
1812         if (!defined($config_test{$config})) {
1813             if (!$once) {
1814                 $once = 1;
1815                 doprint "Configs not produced by kconfig (will not be checked):\n";
1816             }
1817             doprint "  $config\n";
1818             delete $config_list{$config};
1819         }
1820     }
1821     my $ret;
1822     do {
1823         $ret = run_config_bisect;
1824     } while (!$ret);
1825
1826     return $ret if ($ret < 0);
1827
1828     success $i;
1829 }
1830
1831 sub patchcheck_reboot {
1832     doprint "Reboot and sleep $patchcheck_sleep_time seconds\n";
1833     reboot;
1834     start_monitor;
1835     wait_for_monitor $patchcheck_sleep_time;
1836     end_monitor;
1837 }
1838
1839 sub patchcheck {
1840     my ($i) = @_;
1841
1842     die "PATCHCHECK_START[$i] not defined\n"
1843         if (!defined($opt{"PATCHCHECK_START[$i]"}));
1844     die "PATCHCHECK_TYPE[$i] not defined\n"
1845         if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
1846
1847     my $start = $opt{"PATCHCHECK_START[$i]"};
1848
1849     my $end = "HEAD";
1850     if (defined($opt{"PATCHCHECK_END[$i]"})) {
1851         $end = $opt{"PATCHCHECK_END[$i]"};
1852     }
1853
1854     # Get the true sha1's since we can use things like HEAD~3
1855     $start = get_sha1($start);
1856     $end = get_sha1($end);
1857
1858     my $type = $opt{"PATCHCHECK_TYPE[$i]"};
1859
1860     # Can't have a test without having a test to run
1861     if ($type eq "test" && !defined($run_test)) {
1862         $type = "boot";
1863     }
1864
1865     open (IN, "git log --pretty=oneline $end|") or
1866         dodie "could not get git list";
1867
1868     my @list;
1869
1870     while (<IN>) {
1871         chomp;
1872         $list[$#list+1] = $_;
1873         last if (/^$start/);
1874     }
1875     close(IN);
1876
1877     if ($list[$#list] !~ /^$start/) {
1878         fail "SHA1 $start not found";
1879     }
1880
1881     # go backwards in the list
1882     @list = reverse @list;
1883
1884     my $save_clean = $noclean;
1885
1886     $in_patchcheck = 1;
1887     foreach my $item (@list) {
1888         my $sha1 = $item;
1889         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
1890
1891         doprint "\nProcessing commit $item\n\n";
1892
1893         run_command "git checkout $sha1" or
1894             die "Failed to checkout $sha1";
1895
1896         # only clean on the first and last patch
1897         if ($item eq $list[0] ||
1898             $item eq $list[$#list]) {
1899             $noclean = $save_clean;
1900         } else {
1901             $noclean = 1;
1902         }
1903
1904         if (defined($minconfig)) {
1905             build "useconfig:$minconfig" or return 0;
1906         } else {
1907             # ?? no config to use?
1908             build "oldconfig" or return 0;
1909         }
1910
1911         check_buildlog $sha1 or return 0;
1912
1913         next if ($type eq "build");
1914
1915         get_grub_index;
1916         get_version;
1917         install;
1918
1919         my $failed = 0;
1920
1921         start_monitor;
1922         monitor or $failed = 1;
1923
1924         if (!$failed && $type ne "boot"){
1925             do_run_test or $failed = 1;
1926         }
1927         end_monitor;
1928         return 0 if ($failed);
1929
1930         patchcheck_reboot;
1931
1932     }
1933     $in_patchcheck = 0;
1934     success $i;
1935
1936     return 1;
1937 }
1938
1939 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
1940
1941 if ($#ARGV == 0) {
1942     $ktest_config = $ARGV[0];
1943     if (! -f $ktest_config) {
1944         print "$ktest_config does not exist.\n";
1945         my $ans;
1946         for (;;) {
1947             print "Create it? [Y/n] ";
1948             $ans = <STDIN>;
1949             chomp $ans;
1950             if ($ans =~ /^\s*$/) {
1951                 $ans = "y";
1952             }
1953             last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
1954             print "Please answer either 'y' or 'n'.\n";
1955         }
1956         if ($ans !~ /^y$/i) {
1957             exit 0;
1958         }
1959     }
1960 } else {
1961     $ktest_config = "ktest.conf";
1962 }
1963
1964 if (! -f $ktest_config) {
1965     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
1966     print OUT << "EOF"
1967 # Generated by ktest.pl
1968 #
1969 # Define each test with TEST_START
1970 # The config options below it will override the defaults
1971 TEST_START
1972
1973 DEFAULTS
1974 EOF
1975 ;
1976     close(OUT);
1977 }
1978 read_config $ktest_config;
1979
1980 # Append any configs entered in manually to the config file.
1981 my @new_configs = keys %entered_configs;
1982 if ($#new_configs >= 0) {
1983     print "\nAppending entered in configs to $ktest_config\n";
1984     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
1985     foreach my $config (@new_configs) {
1986         print OUT "$config = $entered_configs{$config}\n";
1987         $opt{$config} = $entered_configs{$config};
1988     }
1989 }
1990
1991 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
1992     unlink $opt{"LOG_FILE"};
1993 }
1994
1995 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
1996
1997 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
1998
1999     if (!$i) {
2000         doprint "DEFAULT OPTIONS:\n";
2001     } else {
2002         doprint "\nTEST $i OPTIONS";
2003         if (defined($repeat_tests{$i})) {
2004             $repeat = $repeat_tests{$i};
2005             doprint " ITERATE $repeat";
2006         }
2007         doprint "\n";
2008     }
2009
2010     foreach my $option (sort keys %opt) {
2011
2012         if ($option =~ /\[(\d+)\]$/) {
2013             next if ($i != $1);
2014         } else {
2015             next if ($i);
2016         }
2017
2018         doprint "$option = $opt{$option}\n";
2019     }
2020 }
2021
2022 sub __set_test_option {
2023     my ($name, $i) = @_;
2024
2025     my $option = "$name\[$i\]";
2026
2027     if (defined($opt{$option})) {
2028         return $opt{$option};
2029     }
2030
2031     foreach my $test (keys %repeat_tests) {
2032         if ($i >= $test &&
2033             $i < $test + $repeat_tests{$test}) {
2034             $option = "$name\[$test\]";
2035             if (defined($opt{$option})) {
2036                 return $opt{$option};
2037             }
2038         }
2039     }
2040
2041     if (defined($opt{$name})) {
2042         return $opt{$name};
2043     }
2044
2045     return undef;
2046 }
2047
2048 sub eval_option {
2049     my ($option, $i) = @_;
2050
2051     # Add space to evaluate the character before $
2052     $option = " $option";
2053     my $retval = "";
2054
2055     while ($option =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
2056         my $start = $1;
2057         my $var = $2;
2058         my $end = $3;
2059
2060         # Append beginning of line
2061         $retval = "$retval$start";
2062
2063         # If the iteration option OPT[$i] exists, then use that.
2064         # otherwise see if the default OPT (without [$i]) exists.
2065
2066         my $o = "$var\[$i\]";
2067
2068         if (defined($opt{$o})) {
2069             $o = $opt{$o};
2070             $retval = "$retval$o";
2071         } elsif (defined($opt{$var})) {
2072             $o = $opt{$var};
2073             $retval = "$retval$o";
2074         } else {
2075             $retval = "$retval\$\{$var\}";
2076         }
2077
2078         $option = $end;
2079     }
2080
2081     $retval = "$retval$option";
2082
2083     $retval =~ s/^ //;
2084
2085     return $retval;
2086 }
2087
2088 sub set_test_option {
2089     my ($name, $i) = @_;
2090
2091     my $option = __set_test_option($name, $i);
2092     return $option if (!defined($option));
2093
2094     my $prev = "";
2095
2096     # Since an option can evaluate to another option,
2097     # keep iterating until we do not evaluate any more
2098     # options.
2099     my $r = 0;
2100     while ($prev ne $option) {
2101         # Check for recursive evaluations.
2102         # 100 deep should be more than enough.
2103         if ($r++ > 100) {
2104             die "Over 100 evaluations accurred with $name\n" .
2105                 "Check for recursive variables\n";
2106         }
2107         $prev = $option;
2108         $option = eval_option($option, $i);
2109     }
2110
2111     return $option;
2112 }
2113
2114 # First we need to do is the builds
2115 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
2116
2117     $iteration = $i;
2118
2119     my $makecmd = set_test_option("MAKE_CMD", $i);
2120
2121     $machine = set_test_option("MACHINE", $i);
2122     $ssh_user = set_test_option("SSH_USER", $i);
2123     $tmpdir = set_test_option("TMP_DIR", $i);
2124     $outputdir = set_test_option("OUTPUT_DIR", $i);
2125     $builddir = set_test_option("BUILD_DIR", $i);
2126     $test_type = set_test_option("TEST_TYPE", $i);
2127     $build_type = set_test_option("BUILD_TYPE", $i);
2128     $build_options = set_test_option("BUILD_OPTIONS", $i);
2129     $power_cycle = set_test_option("POWER_CYCLE", $i);
2130     $reboot = set_test_option("REBOOT", $i);
2131     $noclean = set_test_option("BUILD_NOCLEAN", $i);
2132     $minconfig = set_test_option("MIN_CONFIG", $i);
2133     $run_test = set_test_option("TEST", $i);
2134     $addconfig = set_test_option("ADD_CONFIG", $i);
2135     $reboot_type = set_test_option("REBOOT_TYPE", $i);
2136     $grub_menu = set_test_option("GRUB_MENU", $i);
2137     $post_install = set_test_option("POST_INSTALL", $i);
2138     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
2139     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
2140     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
2141     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
2142     $power_off = set_test_option("POWER_OFF", $i);
2143     $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
2144     $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
2145     $sleep_time = set_test_option("SLEEP_TIME", $i);
2146     $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
2147     $patchcheck_sleep_time = set_test_option("PATCHCHECK_SLEEP_TIME", $i);
2148     $bisect_manual = set_test_option("BISECT_MANUAL", $i);
2149     $bisect_skip = set_test_option("BISECT_SKIP", $i);
2150     $store_failures = set_test_option("STORE_FAILURES", $i);
2151     $timeout = set_test_option("TIMEOUT", $i);
2152     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
2153     $console = set_test_option("CONSOLE", $i);
2154     $success_line = set_test_option("SUCCESS_LINE", $i);
2155     $stop_after_success = set_test_option("STOP_AFTER_SUCCESS", $i);
2156     $stop_after_failure = set_test_option("STOP_AFTER_FAILURE", $i);
2157     $stop_test_after = set_test_option("STOP_TEST_AFTER", $i);
2158     $build_target = set_test_option("BUILD_TARGET", $i);
2159     $ssh_exec = set_test_option("SSH_EXEC", $i);
2160     $scp_to_target = set_test_option("SCP_TO_TARGET", $i);
2161     $target_image = set_test_option("TARGET_IMAGE", $i);
2162     $localversion = set_test_option("LOCALVERSION", $i);
2163
2164     chdir $builddir || die "can't change directory to $builddir";
2165
2166     if (!-d $tmpdir) {
2167         mkpath($tmpdir) or
2168             die "can't create $tmpdir";
2169     }
2170
2171     $ENV{"SSH_USER"} = $ssh_user;
2172     $ENV{"MACHINE"} = $machine;
2173
2174     $target = "$ssh_user\@$machine";
2175
2176     $buildlog = "$tmpdir/buildlog-$machine";
2177     $dmesg = "$tmpdir/dmesg-$machine";
2178     $make = "$makecmd O=$outputdir";
2179     $output_config = "$outputdir/.config";
2180
2181     if ($reboot_type eq "grub") {
2182         dodie "GRUB_MENU not defined" if (!defined($grub_menu));
2183     } elsif (!defined($reboot_script)) {
2184         dodie "REBOOT_SCRIPT not defined"
2185     }
2186
2187     my $run_type = $build_type;
2188     if ($test_type eq "patchcheck") {
2189         $run_type = $opt{"PATCHCHECK_TYPE[$i]"};
2190     } elsif ($test_type eq "bisect") {
2191         $run_type = $opt{"BISECT_TYPE[$i]"};
2192     } elsif ($test_type eq "config_bisect") {
2193         $run_type = $opt{"CONFIG_BISECT_TYPE[$i]"};
2194     }
2195
2196     # mistake in config file?
2197     if (!defined($run_type)) {
2198         $run_type = "ERROR";
2199     }
2200
2201     doprint "\n\n";
2202     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type\n\n";
2203
2204     unlink $dmesg;
2205     unlink $buildlog;
2206
2207     if (!defined($minconfig)) {
2208         $minconfig = $addconfig;
2209
2210     } elsif (defined($addconfig)) {
2211         run_command "cat $addconfig $minconfig > $tmpdir/add_config" or
2212             dodie "Failed to create temp config";
2213         $minconfig = "$tmpdir/add_config";
2214     }
2215
2216     my $checkout = $opt{"CHECKOUT[$i]"};
2217     if (defined($checkout)) {
2218         run_command "git checkout $checkout" or
2219             die "failed to checkout $checkout";
2220     }
2221
2222     if ($test_type eq "bisect") {
2223         bisect $i;
2224         next;
2225     } elsif ($test_type eq "config_bisect") {
2226         config_bisect $i;
2227         next;
2228     } elsif ($test_type eq "patchcheck") {
2229         patchcheck $i;
2230         next;
2231     }
2232
2233     if ($build_type ne "nobuild") {
2234         build $build_type or next;
2235     }
2236
2237     if ($test_type ne "build") {
2238         get_grub_index;
2239         get_version;
2240         install;
2241
2242         my $failed = 0;
2243         start_monitor;
2244         monitor or $failed = 1;;
2245
2246         if (!$failed && $test_type ne "boot" && defined($run_test)) {
2247             do_run_test or $failed = 1;
2248         }
2249         end_monitor;
2250         next if ($failed);
2251     }
2252
2253     success $i;
2254 }
2255
2256 if ($opt{"POWEROFF_ON_SUCCESS"}) {
2257     halt;
2258 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
2259     reboot;
2260 }
2261
2262 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
2263
2264 exit 0;