tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2 #
   3 # Copyright 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
   4 # Licensed under the terms of the GNU GPL License version 2
   5 #
   6
   7 use strict;
   8 use IPC::Open2;
   9 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
  10 use File::Path qw(mkpath);
  11 use File::Copy qw(cp);
  12 use FileHandle;
  13
  14 my $VERSION = "0.2";
  15
  16 $| = 1;
  17
  18 my %opt;
  19 my %repeat_tests;
  20 my %repeats;
  21 my %default;
  22
  23 #default opts
  24 $default{"NUM_TESTS"}           = 1;
  25 $default{"REBOOT_TYPE"}         = "grub";
  26 $default{"TEST_TYPE"}           = "test";
  27 $default{"BUILD_TYPE"}          = "randconfig";
  28 $default{"MAKE_CMD"}            = "make";
  29 $default{"TIMEOUT"}             = 120;
  30 $default{"TMP_DIR"}             = "/tmp/ktest/\${MACHINE}";
  31 $default{"SLEEP_TIME"}          = 60;   # sleep time between tests
  32 $default{"BUILD_NOCLEAN"}       = 0;
  33 $default{"REBOOT_ON_ERROR"}     = 0;
  34 $default{"POWEROFF_ON_ERROR"}   = 0;
  35 $default{"REBOOT_ON_SUCCESS"}   = 1;
  36 $default{"POWEROFF_ON_SUCCESS"} = 0;
  37 $default{"BUILD_OPTIONS"}       = "";
  38 $default{"BISECT_SLEEP_TIME"}   = 60;   # sleep time between bisects
  39 $default{"PATCHCHECK_SLEEP_TIME"} = 60; # sleep time between patch checks
  40 $default{"CLEAR_LOG"}           = 0;
  41 $default{"BISECT_MANUAL"}       = 0;
  42 $default{"BISECT_SKIP"}         = 1;
  43 $default{"SUCCESS_LINE"}        = "login:";
  44 $default{"DETECT_TRIPLE_FAULT"} = 1;
  45 $default{"NO_INSTALL"}          = 0;
  46 $default{"BOOTED_TIMEOUT"}      = 1;
  47 $default{"DIE_ON_FAILURE"}      = 1;
  48 $default{"SSH_EXEC"}            = "ssh \$SSH_USER\@\$MACHINE \$SSH_COMMAND";
  49 $default{"SCP_TO_TARGET"}       = "scp \$SRC_FILE \$SSH_USER\@\$MACHINE:\$DST_FILE";
  50 $default{"REBOOT"}              = "ssh \$SSH_USER\@\$MACHINE reboot";
  51 $default{"STOP_AFTER_SUCCESS"}  = 10;
  52 $default{"STOP_AFTER_FAILURE"}  = 60;
  53 $default{"STOP_TEST_AFTER"}     = 600;
  54 $default{"LOCALVERSION"}        = "-test";
  55
  56 my $ktest_config;
  57 my $version;
  58 my $machine;
  59 my $ssh_user;
  60 my $tmpdir;
  61 my $builddir;
  62 my $outputdir;
  63 my $output_config;
  64 my $test_type;
  65 my $build_type;
  66 my $build_options;
  67 my $pre_build;
  68 my $post_build;
  69 my $pre_build_die;
  70 my $post_build_die;
  71 my $reboot_type;
  72 my $reboot_script;
  73 my $power_cycle;
  74 my $reboot;
  75 my $reboot_on_error;
  76 my $poweroff_on_error;
  77 my $die_on_failure;
  78 my $powercycle_after_reboot;
  79 my $poweroff_after_halt;
  80 my $ssh_exec;
  81 my $scp_to_target;
  82 my $power_off;
  83 my $grub_menu;
  84 my $grub_number;
  85 my $target;
  86 my $make;
  87 my $post_install;
  88 my $no_install;
  89 my $noclean;
  90 my $minconfig;
  91 my $start_minconfig;
  92 my $start_minconfig_defined;
  93 my $output_minconfig;
  94 my $ignore_config;
  95 my $addconfig;
  96 my $in_bisect = 0;
  97 my $bisect_bad = "";
  98 my $reverse_bisect;
  99 my $bisect_manual;
 100 my $bisect_skip;
 101 my $config_bisect_good;
 102 my $in_patchcheck = 0;
 103 my $run_test;
 104 my $redirect;
 105 my $buildlog;
 106 my $dmesg;
 107 my $monitor_fp;
 108 my $monitor_pid;
 109 my $monitor_cnt = 0;
 110 my $sleep_time;
 111 my $bisect_sleep_time;
 112 my $patchcheck_sleep_time;
 113 my $ignore_warnings;
 114 my $store_failures;
 115 my $test_name;
 116 my $timeout;
 117 my $booted_timeout;
 118 my $detect_triplefault;
 119 my $console;
 120 my $reboot_success_line;
 121 my $success_line;
 122 my $stop_after_success;
 123 my $stop_after_failure;
 124 my $stop_test_after;
 125 my $build_target;
 126 my $target_image;
 127 my $localversion;
 128 my $iteration = 0;
 129 my $successes = 0;
 130
 131 my %entered_configs;
 132 my %config_help;
 133 my %variable;
 134 my %force_config;
 135
 136 # do not force reboots on config problems
 137 my $no_reboot = 1;
 138
 139 $config_help{"MACHINE"} = << "EOF"
 140  The machine hostname that you will test.
 141 EOF
 142     ;
 143 $config_help{"SSH_USER"} = << "EOF"
 144  The box is expected to have ssh on normal bootup, provide the user
 145   (most likely root, since you need privileged operations)
 146 EOF
 147     ;
 148 $config_help{"BUILD_DIR"} = << "EOF"
 149  The directory that contains the Linux source code (full path).
 150 EOF
 151     ;
 152 $config_help{"OUTPUT_DIR"} = << "EOF"
 153  The directory that the objects will be built (full path).
 154  (can not be same as BUILD_DIR)
 155 EOF
 156     ;
 157 $config_help{"BUILD_TARGET"} = << "EOF"
 158  The location of the compiled file to copy to the target.
 159  (relative to OUTPUT_DIR)
 160 EOF
 161     ;
 162 $config_help{"TARGET_IMAGE"} = << "EOF"
 163  The place to put your image on the test machine.
 164 EOF
 165     ;
 166 $config_help{"POWER_CYCLE"} = << "EOF"
 167  A script or command to reboot the box.
 168
 169  Here is a digital loggers power switch example
 170  POWER_CYCLE = wget --no-proxy -O /dev/null -q  --auth-no-challenge 'http://admin:admin\@power/outlet?5=CCL'
 171
 172  Here is an example to reboot a virtual box on the current host
 173  with the name "Guest".
 174  POWER_CYCLE = virsh destroy Guest; sleep 5; virsh start Guest
 175 EOF
 176     ;
 177 $config_help{"CONSOLE"} = << "EOF"
 178  The script or command that reads the console
 179
 180   If you use ttywatch server, something like the following would work.
 181 CONSOLE = nc -d localhost 3001
 182
 183  For a virtual machine with guest name "Guest".
 184 CONSOLE =  virsh console Guest
 185 EOF
 186     ;
 187 $config_help{"LOCALVERSION"} = << "EOF"
 188  Required version ending to differentiate the test
 189  from other linux builds on the system.
 190 EOF
 191     ;
 192 $config_help{"REBOOT_TYPE"} = << "EOF"
 193  Way to reboot the box to the test kernel.
 194  Only valid options so far are "grub" and "script".
 195
 196  If you specify grub, it will assume grub version 1
 197  and will search in /boot/grub/menu.lst for the title \$GRUB_MENU
 198  and select that target to reboot to the kernel. If this is not
 199  your setup, then specify "script" and have a command or script
 200  specified in REBOOT_SCRIPT to boot to the target.
 201
 202  The entry in /boot/grub/menu.lst must be entered in manually.
 203  The test will not modify that file.
 204 EOF
 205     ;
 206 $config_help{"GRUB_MENU"} = << "EOF"
 207  The grub title name for the test kernel to boot
 208  (Only mandatory if REBOOT_TYPE = grub)
 209
 210  Note, ktest.pl will not update the grub menu.lst, you need to
 211  manually add an option for the test. ktest.pl will search
 212  the grub menu.lst for this option to find what kernel to
 213  reboot into.
 214
 215  For example, if in the /boot/grub/menu.lst the test kernel title has:
 216  title Test Kernel
 217  kernel vmlinuz-test
 218  GRUB_MENU = Test Kernel
 219 EOF
 220     ;
 221 $config_help{"REBOOT_SCRIPT"} = << "EOF"
 222  A script to reboot the target into the test kernel
 223  (Only mandatory if REBOOT_TYPE = script)
 224 EOF
 225     ;
 226
 227 sub read_yn {
 228     my ($prompt) = @_;
 229
 230     my $ans;
 231
 232     for (;;) {
 233         print "$prompt [Y/n] ";
 234         $ans = <STDIN>;
 235         chomp $ans;
 236         if ($ans =~ /^\s*$/) {
 237             $ans = "y";
 238         }
 239         last if ($ans =~ /^y$/i || $ans =~ /^n$/i);
 240         print "Please answer either 'y' or 'n'.\n";
 241     }
 242     if ($ans !~ /^y$/i) {
 243         return 0;
 244     }
 245     return 1;
 246 }
 247
 248 sub get_ktest_config {
 249     my ($config) = @_;
 250
 251     return if (defined($opt{$config}));
 252
 253     if (defined($config_help{$config})) {
 254         print "\n";
 255         print $config_help{$config};
 256     }
 257
 258     for (;;) {
 259         print "$config = ";
 260         if (defined($default{$config})) {
 261             print "\[$default{$config}\] ";
 262         }
 263         $entered_configs{$config} = <STDIN>;
 264         $entered_configs{$config} =~ s/^\s*(.*\S)\s*$/$1/;
 265         if ($entered_configs{$config} =~ /^\s*$/) {
 266             if ($default{$config}) {
 267                 $entered_configs{$config} = $default{$config};
 268             } else {
 269                 print "Your answer can not be blank\n";
 270                 next;
 271             }
 272         }
 273         last;
 274     }
 275 }
 276
 277 sub get_ktest_configs {
 278     get_ktest_config("MACHINE");
 279     get_ktest_config("SSH_USER");
 280     get_ktest_config("BUILD_DIR");
 281     get_ktest_config("OUTPUT_DIR");
 282     get_ktest_config("BUILD_TARGET");
 283     get_ktest_config("TARGET_IMAGE");
 284     get_ktest_config("POWER_CYCLE");
 285     get_ktest_config("CONSOLE");
 286     get_ktest_config("LOCALVERSION");
 287
 288     my $rtype = $opt{"REBOOT_TYPE"};
 289
 290     if (!defined($rtype)) {
 291         if (!defined($opt{"GRUB_MENU"})) {
 292             get_ktest_config("REBOOT_TYPE");
 293             $rtype = $entered_configs{"REBOOT_TYPE"};
 294         } else {
 295             $rtype = "grub";
 296         }
 297     }
 298
 299     if ($rtype eq "grub") {
 300         get_ktest_config("GRUB_MENU");
 301     } else {
 302         get_ktest_config("REBOOT_SCRIPT");
 303     }
 304 }
 305
 306 sub process_variables {
 307     my ($value) = @_;
 308     my $retval = "";
 309
 310     # We want to check for '\', and it is just easier
 311     # to check the previous characet of '$' and not need
 312     # to worry if '$' is the first character. By adding
 313     # a space to $value, we can just check [^\\]\$ and
 314     # it will still work.
 315     $value = " $value";
 316
 317     while ($value =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 318         my $begin = $1;
 319         my $var = $2;
 320         my $end = $3;
 321         # append beginning of value to retval
 322         $retval = "$retval$begin";
 323         if (defined($variable{$var})) {
 324             $retval = "$retval$variable{$var}";
 325         } else {
 326             # put back the origin piece.
 327             $retval = "$retval\$\{$var\}";
 328         }
 329         $value = $end;
 330     }
 331     $retval = "$retval$value";
 332
 333     # remove the space added in the beginning
 334     $retval =~ s/ //;
 335
 336     return "$retval"
 337 }
 338
 339 sub set_value {
 340     my ($lvalue, $rvalue, $override, $overrides, $name) = @_;
 341
 342     if (defined($opt{$lvalue})) {
 343         if (!$override || defined(${$overrides}{$lvalue})) {
 344             my $extra = "";
 345             if ($override) {
 346                 $extra = "In the same override section!\n";
 347             }
 348             die "$name: $.: Option $lvalue defined more than once!\n$extra";
 349         }
 350         ${$overrides}{$lvalue} = $rvalue;
 351     }
 352     if ($rvalue =~ /^\s*$/) {
 353         delete $opt{$lvalue};
 354     } else {
 355         $rvalue = process_variables($rvalue);
 356         $opt{$lvalue} = $rvalue;
 357     }
 358 }
 359
 360 sub set_variable {
 361     my ($lvalue, $rvalue) = @_;
 362
 363     if ($rvalue =~ /^\s*$/) {
 364         delete $variable{$lvalue};
 365     } else {
 366         $rvalue = process_variables($rvalue);
 367         $variable{$lvalue} = $rvalue;
 368     }
 369 }
 370
 371 sub process_compare {
 372     my ($lval, $cmp, $rval) = @_;
 373
 374     # remove whitespace
 375
 376     $lval =~ s/^\s*//;
 377     $lval =~ s/\s*$//;
 378
 379     $rval =~ s/^\s*//;
 380     $rval =~ s/\s*$//;
 381
 382     if ($cmp eq "==") {
 383         return $lval eq $rval;
 384     } elsif ($cmp eq "!=") {
 385         return $lval ne $rval;
 386     }
 387
 388     my $statement = "$lval $cmp $rval";
 389     my $ret = eval $statement;
 390
 391     # $@ stores error of eval
 392     if ($@) {
 393         return -1;
 394     }
 395
 396     return $ret;
 397 }
 398
 399 sub value_defined {
 400     my ($val) = @_;
 401
 402     return defined($variable{$2}) ||
 403         defined($opt{$2});
 404 }
 405
 406 sub process_if {
 407     my ($name, $value) = @_;
 408
 409     my $val = process_variables($value);
 410
 411     if ($val =~ /(.*)(==|\!=|>=|<=|>|<)(.*)/) {
 412         my $ret = process_compare($1, $2, $3);
 413         if ($ret < 0) {
 414             die "$name: $.: Unable to process comparison\n";
 415         }
 416         return $ret;
 417     }
 418
 419     if ($val =~ /^\s*(NOT\s*)?DEFINED\s+(\S+)\s*$/) {
 420         if (defined $1) {
 421             return !value_defined($2);
 422         } else {
 423             return value_defined($2);
 424         }
 425     }
 426
 427     if ($val =~ /^\s*0\s*$/) {
 428         return 0;
 429     } elsif ($val =~ /^\s*\d+\s*$/) {
 430         return 1;
 431     }
 432
 433     die ("$name: $.: Undefined content $val in if statement\n");
 434     return 1;
 435 }
 436
 437 sub __read_config {
 438     my ($config, $current_test_num) = @_;
 439
 440     my $in;
 441     open($in, $config) || die "can't read file $config";
 442
 443     my $name = $config;
 444     $name =~ s,.*/(.*),$1,;
 445
 446     my $test_num = $$current_test_num;
 447     my $default = 1;
 448     my $repeat = 1;
 449     my $num_tests_set = 0;
 450     my $skip = 0;
 451     my $rest;
 452     my $test_case = 0;
 453     my $if = 0;
 454     my $if_set = 0;
 455     my $override = 0;
 456
 457     my %overrides;
 458
 459     while (<$in>) {
 460
 461         # ignore blank lines and comments
 462         next if (/^\s*$/ || /\s*\#/);
 463
 464         if (/^\s*(TEST_START|DEFAULTS)\b(.*)/) {
 465
 466             my $type = $1;
 467             $rest = $2;
 468
 469             my $old_test_num;
 470             my $old_repeat;
 471             $override = 0;
 472
 473             if ($type eq "TEST_START") {
 474
 475                 if ($num_tests_set) {
 476                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 477                 }
 478
 479                 $old_test_num = $test_num;
 480                 $old_repeat = $repeat;
 481
 482                 $test_num += $repeat;
 483                 $default = 0;
 484                 $repeat = 1;
 485             } else {
 486                 $default = 1;
 487             }
 488
 489             if ($rest =~ /\s+SKIP\b(.*)/) {
 490                 $rest = $1;
 491                 $skip = 1;
 492             } else {
 493                 $test_case = 1;
 494                 $skip = 0;
 495             }
 496
 497             if (!$skip) {
 498                 if ($type eq "TEST_START") {
 499                     if ($rest =~ /\s+ITERATE\s+(\d+)(.*)$/) {
 500                         $repeat = $1;
 501                         $rest = $2;
 502                         $repeat_tests{"$test_num"} = $repeat;
 503                     }
 504                 } elsif ($rest =~ /\sOVERRIDE\b(.*)/) {
 505                     # DEFAULT only
 506                     $rest = $1;
 507                     $override = 1;
 508                     # Clear previous overrides
 509                     %overrides = ();
 510                 }
 511             }
 512
 513             if ($rest =~ /\sIF\s+(.*)/) {
 514                 $rest = "";
 515                 if (process_if($name, $1)) {
 516                     $if_set = 1;
 517                 } else {
 518                     $skip = 1;
 519                 }
 520                 $if = 1;
 521             } else {
 522                 $if = 0;
 523             }
 524
 525             if ($rest !~ /^\s*$/) {
 526                 die "$name: $.: Gargbage found after $type\n$_";
 527             }
 528
 529             if ($skip && $type eq "TEST_START") {
 530                 $test_num = $old_test_num;
 531                 $repeat = $old_repeat;
 532             }
 533
 534         } elsif (/^\s*ELSE\b(.*)$/) {
 535             if (!$if) {
 536                 die "$name: $.: ELSE found with out matching IF section\n$_";
 537             }
 538             $rest = $1;
 539             if ($if_set) {
 540                 $skip = 1;
 541                 $rest = "";
 542             } else {
 543                 $skip = 0;
 544
 545                 if ($rest =~ /\sIF\s+(.*)/) {
 546                     # May be a ELSE IF section.
 547                     if (!process_if($name, $1)) {
 548                         $skip = 1;
 549                     }
 550                     $rest = "";
 551                 } else {
 552                     $if = 0;
 553                 }
 554             }
 555
 556             if ($rest !~ /^\s*$/) {
 557                 die "$name: $.: Gargbage found after DEFAULTS\n$_";
 558             }
 559
 560         } elsif (/^\s*INCLUDE\s+(\S+)/) {
 561
 562             next if ($skip);
 563
 564             if (!$default) {
 565                 die "$name: $.: INCLUDE can only be done in default sections\n$_";
 566             }
 567
 568             my $file = process_variables($1);
 569
 570             if ($file !~ m,^/,) {
 571                 # check the path of the config file first
 572                 if ($config =~ m,(.*)/,) {
 573                     if (-f "$1/$file") {
 574                         $file = "$1/$file";
 575                     }
 576                 }
 577             }
 578
 579             if ( ! -r $file ) {
 580                 die "$name: $.: Can't read file $file\n$_";
 581             }
 582
 583             if (__read_config($file, \$test_num)) {
 584                 $test_case = 1;
 585             }
 586
 587         } elsif (/^\s*([A-Z_\[\]\d]+)\s*=\s*(.*?)\s*$/) {
 588
 589             next if ($skip);
 590
 591             my $lvalue = $1;
 592             my $rvalue = $2;
 593
 594             if (!$default &&
 595                 ($lvalue eq "NUM_TESTS" ||
 596                  $lvalue eq "LOG_FILE" ||
 597                  $lvalue eq "CLEAR_LOG")) {
 598                 die "$name: $.: $lvalue must be set in DEFAULTS section\n";
 599             }
 600
 601             if ($lvalue eq "NUM_TESTS") {
 602                 if ($test_num) {
 603                     die "$name: $.: Can not specify both NUM_TESTS and TEST_START\n";
 604                 }
 605                 if (!$default) {
 606                     die "$name: $.: NUM_TESTS must be set in default section\n";
 607                 }
 608                 $num_tests_set = 1;
 609             }
 610
 611             if ($default || $lvalue =~ /\[\d+\]$/) {
 612                 set_value($lvalue, $rvalue, $override, \%overrides, $name);
 613             } else {
 614                 my $val = "$lvalue\[$test_num\]";
 615                 set_value($val, $rvalue, $override, \%overrides, $name);
 616
 617                 if ($repeat > 1) {
 618                     $repeats{$val} = $repeat;
 619                 }
 620             }
 621         } elsif (/^\s*([A-Z_\[\]\d]+)\s*:=\s*(.*?)\s*$/) {
 622             next if ($skip);
 623
 624             my $lvalue = $1;
 625             my $rvalue = $2;
 626
 627             # process config variables.
 628             # Config variables are only active while reading the
 629             # config and can be defined anywhere. They also ignore
 630             # TEST_START and DEFAULTS, but are skipped if they are in
 631             # on of these sections that have SKIP defined.
 632             # The save variable can be
 633             # defined multiple times and the new one simply overrides
 634             # the prevous one.
 635             set_variable($lvalue, $rvalue);
 636
 637         } else {
 638             die "$name: $.: Garbage found in config\n$_";
 639         }
 640     }
 641
 642     if ($test_num) {
 643         $test_num += $repeat - 1;
 644         $opt{"NUM_TESTS"} = $test_num;
 645     }
 646
 647     close($in);
 648
 649     $$current_test_num = $test_num;
 650
 651     return $test_case;
 652 }
 653
 654 sub read_config {
 655     my ($config) = @_;
 656
 657     my $test_case;
 658     my $test_num = 0;
 659
 660     $test_case = __read_config $config, \$test_num;
 661
 662     # make sure we have all mandatory configs
 663     get_ktest_configs;
 664
 665     # was a test specified?
 666     if (!$test_case) {
 667         print "No test case specified.\n";
 668         print "What test case would you like to run?\n";
 669         my $ans = <STDIN>;
 670         chomp $ans;
 671         $default{"TEST_TYPE"} = $ans;
 672     }
 673
 674     # set any defaults
 675
 676     foreach my $default (keys %default) {
 677         if (!defined($opt{$default})) {
 678             $opt{$default} = $default{$default};
 679         }
 680     }
 681 }
 682
 683 sub __eval_option {
 684     my ($option, $i) = @_;
 685
 686     # Add space to evaluate the character before $
 687     $option = " $option";
 688     my $retval = "";
 689
 690     while ($option =~ /(.*?[^\\])\$\{(.*?)\}(.*)/) {
 691         my $start = $1;
 692         my $var = $2;
 693         my $end = $3;
 694
 695         # Append beginning of line
 696         $retval = "$retval$start";
 697
 698         # If the iteration option OPT[$i] exists, then use that.
 699         # otherwise see if the default OPT (without [$i]) exists.
 700
 701         my $o = "$var\[$i\]";
 702
 703         if (defined($opt{$o})) {
 704             $o = $opt{$o};
 705             $retval = "$retval$o";
 706         } elsif (defined($opt{$var})) {
 707             $o = $opt{$var};
 708             $retval = "$retval$o";
 709         } else {
 710             $retval = "$retval\$\{$var\}";
 711         }
 712
 713         $option = $end;
 714     }
 715
 716     $retval = "$retval$option";
 717
 718     $retval =~ s/^ //;
 719
 720     return $retval;
 721 }
 722
 723 sub eval_option {
 724     my ($option, $i) = @_;
 725
 726     my $prev = "";
 727
 728     # Since an option can evaluate to another option,
 729     # keep iterating until we do not evaluate any more
 730     # options.
 731     my $r = 0;
 732     while ($prev ne $option) {
 733         # Check for recursive evaluations.
 734         # 100 deep should be more than enough.
 735         if ($r++ > 100) {
 736             die "Over 100 evaluations accurred with $option\n" .
 737                 "Check for recursive variables\n";
 738         }
 739         $prev = $option;
 740         $option = __eval_option($option, $i);
 741     }
 742
 743     return $option;
 744 }
 745
 746 sub _logit {
 747     if (defined($opt{"LOG_FILE"})) {
 748         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
 749         print OUT @_;
 750         close(OUT);
 751     }
 752 }
 753
 754 sub logit {
 755     if (defined($opt{"LOG_FILE"})) {
 756         _logit @_;
 757     } else {
 758         print @_;
 759     }
 760 }
 761
 762 sub doprint {
 763     print @_;
 764     _logit @_;
 765 }
 766
 767 sub run_command;
 768 sub start_monitor;
 769 sub end_monitor;
 770 sub wait_for_monitor;
 771
 772 sub reboot {
 773     my ($time) = @_;
 774
 775     if (defined($time)) {
 776         start_monitor;
 777         # flush out current monitor
 778         # May contain the reboot success line
 779         wait_for_monitor 1;
 780     }
 781
 782     # try to reboot normally
 783     if (run_command $reboot) {
 784         if (defined($powercycle_after_reboot)) {
 785             sleep $powercycle_after_reboot;
 786             run_command "$power_cycle";
 787         }
 788     } else {
 789         # nope? power cycle it.
 790         run_command "$power_cycle";
 791     }
 792
 793     if (defined($time)) {
 794         wait_for_monitor($time, $reboot_success_line);
 795         end_monitor;
 796     }
 797 }
 798
 799 sub do_not_reboot {
 800     my $i = $iteration;
 801
 802     return $test_type eq "build" || $no_reboot ||
 803         ($test_type eq "patchcheck" && $opt{"PATCHCHECK_TYPE[$i]"} eq "build") ||
 804         ($test_type eq "bisect" && $opt{"BISECT_TYPE[$i]"} eq "build");
 805 }
 806
 807 sub dodie {
 808     doprint "CRITICAL FAILURE... ", @_, "\n";
 809
 810     my $i = $iteration;
 811
 812     if ($reboot_on_error && !do_not_reboot) {
 813
 814         doprint "REBOOTING\n";
 815         reboot;
 816
 817     } elsif ($poweroff_on_error && defined($power_off)) {
 818         doprint "POWERING OFF\n";
 819         `$power_off`;
 820     }
 821
 822     if (defined($opt{"LOG_FILE"})) {
 823         print " See $opt{LOG_FILE} for more info.\n";
 824     }
 825
 826     die @_, "\n";
 827 }
 828
 829 sub open_console {
 830     my ($fp) = @_;
 831
 832     my $flags;
 833
 834     my $pid = open($fp, "$console|") or
 835         dodie "Can't open console $console";
 836
 837     $flags = fcntl($fp, F_GETFL, 0) or
 838         dodie "Can't get flags for the socket: $!";
 839     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 840         dodie "Can't set flags for the socket: $!";
 841
 842     return $pid;
 843 }
 844
 845 sub close_console {
 846     my ($fp, $pid) = @_;
 847
 848     doprint "kill child process $pid\n";
 849     kill 2, $pid;
 850
 851     print "closing!\n";
 852     close($fp);
 853 }
 854
 855 sub start_monitor {
 856     if ($monitor_cnt++) {
 857         return;
 858     }
 859     $monitor_fp = \*MONFD;
 860     $monitor_pid = open_console $monitor_fp;
 861
 862     return;
 863
 864     open(MONFD, "Stop perl from warning about single use of MONFD");
 865 }
 866
 867 sub end_monitor {
 868     if (--$monitor_cnt) {
 869         return;
 870     }
 871     close_console($monitor_fp, $monitor_pid);
 872 }
 873
 874 sub wait_for_monitor {
 875     my ($time, $stop) = @_;
 876     my $full_line = "";
 877     my $line;
 878     my $booted = 0;
 879
 880     doprint "** Wait for monitor to settle down **\n";
 881
 882     # read the monitor and wait for the system to calm down
 883     while (!$booted) {
 884         $line = wait_for_input($monitor_fp, $time);
 885         last if (!defined($line));
 886         print "$line";
 887         $full_line .= $line;
 888
 889         if (defined($stop) && $full_line =~ /$stop/) {
 890             doprint "wait for monitor detected $stop\n";
 891             $booted = 1;
 892         }
 893
 894         if ($line =~ /\n/) {
 895             $full_line = "";
 896         }
 897     }
 898     print "** Monitor flushed **\n";
 899 }
 900
 901 sub fail {
 902
 903         if ($die_on_failure) {
 904                 dodie @_;
 905         }
 906
 907         doprint "FAILED\n";
 908
 909         my $i = $iteration;
 910
 911         # no need to reboot for just building.
 912         if (!do_not_reboot) {
 913             doprint "REBOOTING\n";
 914             reboot $sleep_time;
 915         }
 916
 917         my $name = "";
 918
 919         if (defined($test_name)) {
 920             $name = " ($test_name)";
 921         }
 922
 923         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 924         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 925         doprint "KTEST RESULT: TEST $i$name Failed: ", @_, "\n";
 926         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 927         doprint "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%\n";
 928
 929         return 1 if (!defined($store_failures));
 930
 931         my @t = localtime;
 932         my $date = sprintf "%04d%02d%02d%02d%02d%02d",
 933                 1900+$t[5],$t[4],$t[3],$t[2],$t[1],$t[0];
 934
 935         my $type = $build_type;
 936         if ($type =~ /useconfig/) {
 937             $type = "useconfig";
 938         }
 939
 940         my $dir = "$machine-$test_type-$type-fail-$date";
 941         my $faildir = "$store_failures/$dir";
 942
 943         if (!-d $faildir) {
 944             mkpath($faildir) or
 945                 die "can't create $faildir";
 946         }
 947         if (-f "$output_config") {
 948             cp "$output_config", "$faildir/config" or
 949                 die "failed to copy .config";
 950         }
 951         if (-f $buildlog) {
 952             cp $buildlog, "$faildir/buildlog" or
 953                 die "failed to move $buildlog";
 954         }
 955         if (-f $dmesg) {
 956             cp $dmesg, "$faildir/dmesg" or
 957                 die "failed to move $dmesg";
 958         }
 959
 960         doprint "*** Saved info to $faildir ***\n";
 961
 962         return 1;
 963 }
 964
 965 sub run_command {
 966     my ($command) = @_;
 967     my $dolog = 0;
 968     my $dord = 0;
 969     my $pid;
 970
 971     $command =~ s/\$SSH_USER/$ssh_user/g;
 972     $command =~ s/\$MACHINE/$machine/g;
 973
 974     doprint("$command ... ");
 975
 976     $pid = open(CMD, "$command 2>&1 |") or
 977         (fail "unable to exec $command" and return 0);
 978
 979     if (defined($opt{"LOG_FILE"})) {
 980         open(LOG, ">>$opt{LOG_FILE}") or
 981             dodie "failed to write to log";
 982         $dolog = 1;
 983     }
 984
 985     if (defined($redirect)) {
 986         open (RD, ">$redirect") or
 987             dodie "failed to write to redirect $redirect";
 988         $dord = 1;
 989     }
 990
 991     while (<CMD>) {
 992         print LOG if ($dolog);
 993         print RD  if ($dord);
 994     }
 995
 996     waitpid($pid, 0);
 997     my $failed = $?;
 998
 999     close(CMD);
1000     close(LOG) if ($dolog);
1001     close(RD)  if ($dord);
1002
1003     if ($failed) {
1004         doprint "FAILED!\n";
1005     } else {
1006         doprint "SUCCESS\n";
1007     }
1008
1009     return !$failed;
1010 }
1011
1012 sub run_ssh {
1013     my ($cmd) = @_;
1014     my $cp_exec = $ssh_exec;
1015
1016     $cp_exec =~ s/\$SSH_COMMAND/$cmd/g;
1017     return run_command "$cp_exec";
1018 }
1019
1020 sub run_scp {
1021     my ($src, $dst) = @_;
1022     my $cp_scp = $scp_to_target;
1023
1024     $cp_scp =~ s/\$SRC_FILE/$src/g;
1025     $cp_scp =~ s/\$DST_FILE/$dst/g;
1026
1027     return run_command "$cp_scp";
1028 }
1029
1030 sub get_grub_index {
1031
1032     if ($reboot_type ne "grub") {
1033         return;
1034     }
1035     return if (defined($grub_number));
1036
1037     doprint "Find grub menu ... ";
1038     $grub_number = -1;
1039
1040     my $ssh_grub = $ssh_exec;
1041     $ssh_grub =~ s,\$SSH_COMMAND,cat /boot/grub/menu.lst,g;
1042
1043     open(IN, "$ssh_grub |")
1044         or die "unable to get menu.lst";
1045
1046     my $found = 0;
1047
1048     while (<IN>) {
1049         if (/^\s*title\s+$grub_menu\s*$/) {
1050             $grub_number++;
1051             $found = 1;
1052             last;
1053         } elsif (/^\s*title\s/) {
1054             $grub_number++;
1055         }
1056     }
1057     close(IN);
1058
1059     die "Could not find '$grub_menu' in /boot/grub/menu on $machine"
1060         if (!$found);
1061     doprint "$grub_number\n";
1062 }
1063
1064 sub wait_for_input
1065 {
1066     my ($fp, $time) = @_;
1067     my $rin;
1068     my $ready;
1069     my $line;
1070     my $ch;
1071
1072     if (!defined($time)) {
1073         $time = $timeout;
1074     }
1075
1076     $rin = '';
1077     vec($rin, fileno($fp), 1) = 1;
1078     $ready = select($rin, undef, undef, $time);
1079
1080     $line = "";
1081
1082     # try to read one char at a time
1083     while (sysread $fp, $ch, 1) {
1084         $line .= $ch;
1085         last if ($ch eq "\n");
1086     }
1087
1088     if (!length($line)) {
1089         return undef;
1090     }
1091
1092     return $line;
1093 }
1094
1095 sub reboot_to {
1096     if ($reboot_type eq "grub") {
1097         run_ssh "'(echo \"savedefault --default=$grub_number --once\" | grub --batch && reboot)'";
1098         return;
1099     }
1100
1101     run_command "$reboot_script";
1102 }
1103
1104 sub get_sha1 {
1105     my ($commit) = @_;
1106
1107     doprint "git rev-list --max-count=1 $commit ... ";
1108     my $sha1 = `git rev-list --max-count=1 $commit`;
1109     my $ret = $?;
1110
1111     logit $sha1;
1112
1113     if ($ret) {
1114         doprint "FAILED\n";
1115         dodie "Failed to get git $commit";
1116     }
1117
1118     print "SUCCESS\n";
1119
1120     chomp $sha1;
1121
1122     return $sha1;
1123 }
1124
1125 sub monitor {
1126     my $booted = 0;
1127     my $bug = 0;
1128     my $skip_call_trace = 0;
1129     my $loops;
1130
1131     wait_for_monitor 5;
1132
1133     my $line;
1134     my $full_line = "";
1135
1136     open(DMESG, "> $dmesg") or
1137         die "unable to write to $dmesg";
1138
1139     reboot_to;
1140
1141     my $success_start;
1142     my $failure_start;
1143     my $monitor_start = time;
1144     my $done = 0;
1145     my $version_found = 0;
1146
1147     while (!$done) {
1148
1149         if ($bug && defined($stop_after_failure) &&
1150             $stop_after_failure >= 0) {
1151             my $time = $stop_after_failure - (time - $failure_start);
1152             $line = wait_for_input($monitor_fp, $time);
1153             if (!defined($line)) {
1154                 doprint "bug timed out after $booted_timeout seconds\n";
1155                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
1156                 last;
1157             }
1158         } elsif ($booted) {
1159             $line = wait_for_input($monitor_fp, $booted_timeout);
1160             if (!defined($line)) {
1161                 my $s = $booted_timeout == 1 ? "" : "s";
1162                 doprint "Successful boot found: break after $booted_timeout second$s\n";
1163                 last;
1164             }
1165         } else {
1166             $line = wait_for_input($monitor_fp);
1167             if (!defined($line)) {
1168                 my $s = $timeout == 1 ? "" : "s";
1169                 doprint "Timed out after $timeout second$s\n";
1170                 last;
1171             }
1172         }
1173
1174         doprint $line;
1175         print DMESG $line;
1176
1177         # we are not guaranteed to get a full line
1178         $full_line .= $line;
1179
1180         if ($full_line =~ /$success_line/) {
1181             $booted = 1;
1182             $success_start = time;
1183         }
1184
1185         if ($booted && defined($stop_after_success) &&
1186             $stop_after_success >= 0) {
1187             my $now = time;
1188             if ($now - $success_start >= $stop_after_success) {
1189                 doprint "Test forced to stop after $stop_after_success seconds after success\n";
1190                 last;
1191             }
1192         }
1193
1194         if ($full_line =~ /\[ backtrace testing \]/) {
1195             $skip_call_trace = 1;
1196         }
1197
1198         if ($full_line =~ /call trace:/i) {
1199             if (!$bug && !$skip_call_trace) {
1200                 $bug = 1;
1201                 $failure_start = time;
1202             }
1203         }
1204
1205         if ($bug && defined($stop_after_failure) &&
1206             $stop_after_failure >= 0) {
1207             my $now = time;
1208             if ($now - $failure_start >= $stop_after_failure) {
1209                 doprint "Test forced to stop after $stop_after_failure seconds after failure\n";
1210                 last;
1211             }
1212         }
1213
1214         if ($full_line =~ /\[ end of backtrace testing \]/) {
1215             $skip_call_trace = 0;
1216         }
1217
1218         if ($full_line =~ /Kernel panic -/) {
1219             $failure_start = time;
1220             $bug = 1;
1221         }
1222
1223         # Detect triple faults by testing the banner
1224         if ($full_line =~ /\bLinux version (\S+).*\n/) {
1225             if ($1 eq $version) {
1226                 $version_found = 1;
1227             } elsif ($version_found && $detect_triplefault) {
1228                 # We already booted into the kernel we are testing,
1229                 # but now we booted into another kernel?
1230                 # Consider this a triple fault.
1231                 doprint "Aleady booted in Linux kernel $version, but now\n";
1232                 doprint "we booted into Linux kernel $1.\n";
1233                 doprint "Assuming that this is a triple fault.\n";
1234                 doprint "To disable this: set DETECT_TRIPLE_FAULT to 0\n";
1235                 last;
1236             }
1237         }
1238
1239         if ($line =~ /\n/) {
1240             $full_line = "";
1241         }
1242
1243         if ($stop_test_after > 0 && !$booted && !$bug) {
1244             if (time - $monitor_start > $stop_test_after) {
1245                 doprint "STOP_TEST_AFTER ($stop_test_after seconds) timed out\n";
1246                 $done = 1;
1247             }
1248         }
1249     }
1250
1251     close(DMESG);
1252
1253     if ($bug) {
1254         return 0 if ($in_bisect);
1255         fail "failed - got a bug report" and return 0;
1256     }
1257
1258     if (!$booted) {
1259         return 0 if ($in_bisect);
1260         fail "failed - never got a boot prompt." and return 0;
1261     }
1262
1263     return 1;
1264 }
1265
1266 sub do_post_install {
1267
1268     return if (!defined($post_install));
1269
1270     my $cp_post_install = $post_install;
1271     $cp_post_install =~ s/\$KERNEL_VERSION/$version/g;
1272     run_command "$cp_post_install" or
1273         dodie "Failed to run post install";
1274 }
1275
1276 sub install {
1277
1278     return if ($no_install);
1279
1280     run_scp "$outputdir/$build_target", "$target_image" or
1281         dodie "failed to copy image";
1282
1283     my $install_mods = 0;
1284
1285     # should we process modules?
1286     $install_mods = 0;
1287     open(IN, "$output_config") or dodie("Can't read config file");
1288     while (<IN>) {
1289         if (/CONFIG_MODULES(=y)?/) {
1290             $install_mods = 1 if (defined($1));
1291             last;
1292         }
1293     }
1294     close(IN);
1295
1296     if (!$install_mods) {
1297         do_post_install;
1298         doprint "No modules needed\n";
1299         return;
1300     }
1301
1302     run_command "$make INSTALL_MOD_PATH=$tmpdir modules_install" or
1303         dodie "Failed to install modules";
1304
1305     my $modlib = "/lib/modules/$version";
1306     my $modtar = "ktest-mods.tar.bz2";
1307
1308     run_ssh "rm -rf $modlib" or
1309         dodie "failed to remove old mods: $modlib";
1310
1311     # would be nice if scp -r did not follow symbolic links
1312     run_command "cd $tmpdir && tar -cjf $modtar lib/modules/$version" or
1313         dodie "making tarball";
1314
1315     run_scp "$tmpdir/$modtar", "/tmp" or
1316         dodie "failed to copy modules";
1317
1318     unlink "$tmpdir/$modtar";
1319
1320     run_ssh "'(cd / && tar xjf /tmp/$modtar)'" or
1321         dodie "failed to tar modules";
1322
1323     run_ssh "rm -f /tmp/$modtar";
1324
1325     do_post_install;
1326 }
1327
1328 sub get_version {
1329     # get the release name
1330     doprint "$make kernelrelease ... ";
1331     $version = `$make kernelrelease | tail -1`;
1332     chomp($version);
1333     doprint "$version\n";
1334 }
1335
1336 sub start_monitor_and_boot {
1337     get_grub_index;
1338     get_version;
1339     install;
1340
1341     start_monitor;
1342     return monitor;
1343 }
1344
1345 sub check_buildlog {
1346     my ($patch) = @_;
1347
1348     my @files = `git show $patch | diffstat -l`;
1349
1350     open(IN, "git show $patch |") or
1351         dodie "failed to show $patch";
1352     while (<IN>) {
1353         if (m,^--- a/(.*),) {
1354             chomp $1;
1355             $files[$#files] = $1;
1356         }
1357     }
1358     close(IN);
1359
1360     open(IN, $buildlog) or dodie "Can't open $buildlog";
1361     while (<IN>) {
1362         if (/^\s*(.*?):.*(warning|error)/) {
1363             my $err = $1;
1364             foreach my $file (@files) {
1365                 my $fullpath = "$builddir/$file";
1366                 if ($file eq $err || $fullpath eq $err) {
1367                     fail "$file built with warnings" and return 0;
1368                 }
1369             }
1370         }
1371     }
1372     close(IN);
1373
1374     return 1;
1375 }
1376
1377 sub apply_min_config {
1378     my $outconfig = "$output_config.new";
1379
1380     # Read the config file and remove anything that
1381     # is in the force_config hash (from minconfig and others)
1382     # then add the force config back.
1383
1384     doprint "Applying minimum configurations into $output_config.new\n";
1385
1386     open (OUT, ">$outconfig") or
1387         dodie "Can't create $outconfig";
1388
1389     if (-f $output_config) {
1390         open (IN, $output_config) or
1391             dodie "Failed to open $output_config";
1392         while (<IN>) {
1393             if (/^(# )?(CONFIG_[^\s=]*)/) {
1394                 next if (defined($force_config{$2}));
1395             }
1396             print OUT;
1397         }
1398         close IN;
1399     }
1400     foreach my $config (keys %force_config) {
1401         print OUT "$force_config{$config}\n";
1402     }
1403     close OUT;
1404
1405     run_command "mv $outconfig $output_config";
1406 }
1407
1408 sub make_oldconfig {
1409
1410     my @force_list = keys %force_config;
1411
1412     if ($#force_list >= 0) {
1413         apply_min_config;
1414     }
1415
1416     if (!run_command "$make oldnoconfig") {
1417         # Perhaps oldnoconfig doesn't exist in this version of the kernel
1418         # try a yes '' | oldconfig
1419         doprint "oldnoconfig failed, trying yes '' | make oldconfig\n";
1420         run_command "yes '' | $make oldconfig" or
1421             dodie "failed make config oldconfig";
1422     }
1423 }
1424
1425 # read a config file and use this to force new configs.
1426 sub load_force_config {
1427     my ($config) = @_;
1428
1429     open(IN, $config) or
1430         dodie "failed to read $config";
1431     while (<IN>) {
1432         chomp;
1433         if (/^(CONFIG[^\s=]*)(\s*=.*)/) {
1434             $force_config{$1} = $_;
1435         } elsif (/^# (CONFIG_\S*) is not set/) {
1436             $force_config{$1} = $_;
1437         }
1438     }
1439     close IN;
1440 }
1441
1442 sub build {
1443     my ($type) = @_;
1444
1445     unlink $buildlog;
1446
1447     # Failed builds should not reboot the target
1448     my $save_no_reboot = $no_reboot;
1449     $no_reboot = 1;
1450
1451     if (defined($pre_build)) {
1452         my $ret = run_command $pre_build;
1453         if (!$ret && defined($pre_build_die) &&
1454             $pre_build_die) {
1455             dodie "failed to pre_build\n";
1456         }
1457     }
1458
1459     if ($type =~ /^useconfig:(.*)/) {
1460         run_command "cp $1 $output_config" or
1461             dodie "could not copy $1 to .config";
1462
1463         $type = "oldconfig";
1464     }
1465
1466     # old config can ask questions
1467     if ($type eq "oldconfig") {
1468         $type = "oldnoconfig";
1469
1470         # allow for empty configs
1471         run_command "touch $output_config";
1472
1473         if (!$noclean) {
1474             run_command "mv $output_config $outputdir/config_temp" or
1475                 dodie "moving .config";
1476
1477             run_command "$make mrproper" or dodie "make mrproper";
1478
1479             run_command "mv $outputdir/config_temp $output_config" or
1480                 dodie "moving config_temp";
1481         }
1482
1483     } elsif (!$noclean) {
1484         unlink "$output_config";
1485         run_command "$make mrproper" or
1486             dodie "make mrproper";
1487     }
1488
1489     # add something to distinguish this build
1490     open(OUT, "> $outputdir/localversion") or dodie("Can't make localversion file");
1491     print OUT "$localversion\n";
1492     close(OUT);
1493
1494     if (defined($minconfig)) {
1495         load_force_config($minconfig);
1496     }
1497
1498     if ($type ne "oldnoconfig") {
1499         run_command "$make $type" or
1500             dodie "failed make config";
1501     }
1502     # Run old config regardless, to enforce min configurations
1503     make_oldconfig;
1504
1505     $redirect = "$buildlog";
1506     my $build_ret = run_command "$make $build_options";
1507     undef $redirect;
1508
1509     if (defined($post_build)) {
1510         my $ret = run_command $post_build;
1511         if (!$ret && defined($post_build_die) &&
1512             $post_build_die) {
1513             dodie "failed to post_build\n";
1514         }
1515     }
1516
1517     if (!$build_ret) {
1518         # bisect may need this to pass
1519         if ($in_bisect) {
1520             $no_reboot = $save_no_reboot;
1521             return 0;
1522         }
1523         fail "failed build" and return 0;
1524     }
1525
1526     $no_reboot = $save_no_reboot;
1527
1528     return 1;
1529 }
1530
1531 sub halt {
1532     if (!run_ssh "halt" or defined($power_off)) {
1533         if (defined($poweroff_after_halt)) {
1534             sleep $poweroff_after_halt;
1535             run_command "$power_off";
1536         }
1537     } else {
1538         # nope? the zap it!
1539         run_command "$power_off";
1540     }
1541 }
1542
1543 sub success {
1544     my ($i) = @_;
1545
1546     $successes++;
1547
1548     my $name = "";
1549
1550     if (defined($test_name)) {
1551         $name = " ($test_name)";
1552     }
1553
1554     doprint "\n\n*******************************************\n";
1555     doprint     "*******************************************\n";
1556     doprint     "KTEST RESULT: TEST $i$name SUCCESS!!!!         **\n";
1557     doprint     "*******************************************\n";
1558     doprint     "*******************************************\n";
1559
1560     if ($i != $opt{"NUM_TESTS"} && !do_not_reboot) {
1561         doprint "Reboot and wait $sleep_time seconds\n";
1562         reboot $sleep_time;
1563     }
1564 }
1565
1566 sub answer_bisect {
1567     for (;;) {
1568         doprint "Pass or fail? [p/f]";
1569         my $ans = <STDIN>;
1570         chomp $ans;
1571         if ($ans eq "p" || $ans eq "P") {
1572             return 1;
1573         } elsif ($ans eq "f" || $ans eq "F") {
1574             return 0;
1575         } else {
1576             print "Please answer 'P' or 'F'\n";
1577         }
1578     }
1579 }
1580
1581 sub child_run_test {
1582     my $failed = 0;
1583
1584     # child should have no power
1585     $reboot_on_error = 0;
1586     $poweroff_on_error = 0;
1587     $die_on_failure = 1;
1588
1589     run_command $run_test or $failed = 1;
1590     exit $failed;
1591 }
1592
1593 my $child_done;
1594
1595 sub child_finished {
1596     $child_done = 1;
1597 }
1598
1599 sub do_run_test {
1600     my $child_pid;
1601     my $child_exit;
1602     my $line;
1603     my $full_line;
1604     my $bug = 0;
1605
1606     wait_for_monitor 1;
1607
1608     doprint "run test $run_test\n";
1609
1610     $child_done = 0;
1611
1612     $SIG{CHLD} = qw(child_finished);
1613
1614     $child_pid = fork;
1615
1616     child_run_test if (!$child_pid);
1617
1618     $full_line = "";
1619
1620     do {
1621         $line = wait_for_input($monitor_fp, 1);
1622         if (defined($line)) {
1623
1624             # we are not guaranteed to get a full line
1625             $full_line .= $line;
1626             doprint $line;
1627
1628             if ($full_line =~ /call trace:/i) {
1629                 $bug = 1;
1630             }
1631
1632             if ($full_line =~ /Kernel panic -/) {
1633                 $bug = 1;
1634             }
1635
1636             if ($line =~ /\n/) {
1637                 $full_line = "";
1638             }
1639         }
1640     } while (!$child_done && !$bug);
1641
1642     if ($bug) {
1643         my $failure_start = time;
1644         my $now;
1645         do {
1646             $line = wait_for_input($monitor_fp, 1);
1647             if (defined($line)) {
1648                 doprint $line;
1649             }
1650             $now = time;
1651             if ($now - $failure_start >= $stop_after_failure) {
1652                 last;
1653             }
1654         } while (defined($line));
1655
1656         doprint "Detected kernel crash!\n";
1657         # kill the child with extreme prejudice
1658         kill 9, $child_pid;
1659     }
1660
1661     waitpid $child_pid, 0;
1662     $child_exit = $?;
1663
1664     if ($bug || $child_exit) {
1665         return 0 if $in_bisect;
1666         fail "test failed" and return 0;
1667     }
1668     return 1;
1669 }
1670
1671 sub run_git_bisect {
1672     my ($command) = @_;
1673
1674     doprint "$command ... ";
1675
1676     my $output = `$command 2>&1`;
1677     my $ret = $?;
1678
1679     logit $output;
1680
1681     if ($ret) {
1682         doprint "FAILED\n";
1683         dodie "Failed to git bisect";
1684     }
1685
1686     doprint "SUCCESS\n";
1687     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
1688         doprint "$1 [$2]\n";
1689     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
1690         $bisect_bad = $1;
1691         doprint "Found bad commit... $1\n";
1692         return 0;
1693     } else {
1694         # we already logged it, just print it now.
1695         print $output;
1696     }
1697
1698     return 1;
1699 }
1700
1701 sub bisect_reboot {
1702     doprint "Reboot and sleep $bisect_sleep_time seconds\n";
1703     reboot $bisect_sleep_time;
1704 }
1705
1706 # returns 1 on success, 0 on failure, -1 on skip
1707 sub run_bisect_test {
1708     my ($type, $buildtype) = @_;
1709
1710     my $failed = 0;
1711     my $result;
1712     my $output;
1713     my $ret;
1714
1715     $in_bisect = 1;
1716
1717     build $buildtype or $failed = 1;
1718
1719     if ($type ne "build") {
1720         if ($failed && $bisect_skip) {
1721             $in_bisect = 0;
1722             return -1;
1723         }
1724         dodie "Failed on build" if $failed;
1725
1726         # Now boot the box
1727         start_monitor_and_boot or $failed = 1;
1728
1729         if ($type ne "boot") {
1730             if ($failed && $bisect_skip) {
1731                 end_monitor;
1732                 bisect_reboot;
1733                 $in_bisect = 0;
1734                 return -1;
1735             }
1736             dodie "Failed on boot" if $failed;
1737
1738             do_run_test or $failed = 1;
1739         }
1740         end_monitor;
1741     }
1742
1743     if ($failed) {
1744         $result = 0;
1745     } else {
1746         $result = 1;
1747     }
1748
1749     # reboot the box to a kernel we can ssh to
1750     if ($type ne "build") {
1751         bisect_reboot;
1752     }
1753     $in_bisect = 0;
1754
1755     return $result;
1756 }
1757
1758 sub run_bisect {
1759     my ($type) = @_;
1760     my $buildtype = "oldconfig";
1761
1762     # We should have a minconfig to use?
1763     if (defined($minconfig)) {
1764         $buildtype = "useconfig:$minconfig";
1765     }
1766
1767     my $ret = run_bisect_test $type, $buildtype;
1768
1769     if ($bisect_manual) {
1770         $ret = answer_bisect;
1771     }
1772
1773     # Are we looking for where it worked, not failed?
1774     if ($reverse_bisect) {
1775         $ret = !$ret;
1776     }
1777
1778     if ($ret > 0) {
1779         return "good";
1780     } elsif ($ret == 0) {
1781         return  "bad";
1782     } elsif ($bisect_skip) {
1783         doprint "HIT A BAD COMMIT ... SKIPPING\n";
1784         return "skip";
1785     }
1786 }
1787
1788 sub bisect {
1789     my ($i) = @_;
1790
1791     my $result;
1792
1793     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
1794     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
1795     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
1796
1797     my $good = $opt{"BISECT_GOOD[$i]"};
1798     my $bad = $opt{"BISECT_BAD[$i]"};
1799     my $type = $opt{"BISECT_TYPE[$i]"};
1800     my $start = $opt{"BISECT_START[$i]"};
1801     my $replay = $opt{"BISECT_REPLAY[$i]"};
1802     my $start_files = $opt{"BISECT_FILES[$i]"};
1803
1804     if (defined($start_files)) {
1805         $start_files = " -- " . $start_files;
1806     } else {
1807         $start_files = "";
1808     }
1809
1810     # convert to true sha1's
1811     $good = get_sha1($good);
1812     $bad = get_sha1($bad);
1813
1814     if (defined($opt{"BISECT_REVERSE[$i]"}) &&
1815         $opt{"BISECT_REVERSE[$i]"} == 1) {
1816         doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
1817         $reverse_bisect = 1;
1818     } else {
1819         $reverse_bisect = 0;
1820     }
1821
1822     # Can't have a test without having a test to run
1823     if ($type eq "test" && !defined($run_test)) {
1824         $type = "boot";
1825     }
1826
1827     my $check = $opt{"BISECT_CHECK[$i]"};
1828     if (defined($check) && $check ne "0") {
1829
1830         # get current HEAD
1831         my $head = get_sha1("HEAD");
1832
1833         if ($check ne "good") {
1834             doprint "TESTING BISECT BAD [$bad]\n";
1835             run_command "git checkout $bad" or
1836                 die "Failed to checkout $bad";
1837
1838             $result = run_bisect $type;
1839
1840             if ($result ne "bad") {
1841                 fail "Tested BISECT_BAD [$bad] and it succeeded" and return 0;
1842             }
1843         }
1844
1845         if ($check ne "bad") {
1846             doprint "TESTING BISECT GOOD [$good]\n";
1847             run_command "git checkout $good" or
1848                 die "Failed to checkout $good";
1849
1850             $result = run_bisect $type;
1851
1852             if ($result ne "good") {
1853                 fail "Tested BISECT_GOOD [$good] and it failed" and return 0;
1854             }
1855         }
1856
1857         # checkout where we started
1858         run_command "git checkout $head" or
1859             die "Failed to checkout $head";
1860     }
1861
1862     run_command "git bisect start$start_files" or
1863         dodie "could not start bisect";
1864
1865     run_command "git bisect good $good" or
1866         dodie "could not set bisect good to $good";
1867
1868     run_git_bisect "git bisect bad $bad" or
1869         dodie "could not set bisect bad to $bad";
1870
1871     if (defined($replay)) {
1872         run_command "git bisect replay $replay" or
1873             dodie "failed to run replay";
1874     }
1875
1876     if (defined($start)) {
1877         run_command "git checkout $start" or
1878             dodie "failed to checkout $start";
1879     }
1880
1881     my $test;
1882     do {
1883         $result = run_bisect $type;
1884         $test = run_git_bisect "git bisect $result";
1885     } while ($test);
1886
1887     run_command "git bisect log" or
1888         dodie "could not capture git bisect log";
1889
1890     run_command "git bisect reset" or
1891         dodie "could not reset git bisect";
1892
1893     doprint "Bad commit was [$bisect_bad]\n";
1894
1895     success $i;
1896 }
1897
1898 my %config_ignore;
1899 my %config_set;
1900
1901 my %config_list;
1902 my %null_config;
1903
1904 my %dependency;
1905
1906 sub assign_configs {
1907     my ($hash, $config) = @_;
1908
1909     open (IN, $config)
1910         or dodie "Failed to read $config";
1911
1912     while (<IN>) {
1913         if (/^((CONFIG\S*)=.*)/) {
1914             ${$hash}{$2} = $1;
1915         }
1916     }
1917
1918     close(IN);
1919 }
1920
1921 sub process_config_ignore {
1922     my ($config) = @_;
1923
1924     assign_configs \%config_ignore, $config;
1925 }
1926
1927 sub read_current_config {
1928     my ($config_ref) = @_;
1929
1930     %{$config_ref} = ();
1931     undef %{$config_ref};
1932
1933     my @key = keys %{$config_ref};
1934     if ($#key >= 0) {
1935         print "did not delete!\n";
1936         exit;
1937     }
1938     open (IN, "$output_config");
1939
1940     while (<IN>) {
1941         if (/^(CONFIG\S+)=(.*)/) {
1942             ${$config_ref}{$1} = $2;
1943         }
1944     }
1945     close(IN);
1946 }
1947
1948 sub get_dependencies {
1949     my ($config) = @_;
1950
1951     my $arr = $dependency{$config};
1952     if (!defined($arr)) {
1953         return ();
1954     }
1955
1956     my @deps = @{$arr};
1957
1958     foreach my $dep (@{$arr}) {
1959         print "ADD DEP $dep\n";
1960         @deps = (@deps, get_dependencies $dep);
1961     }
1962
1963     return @deps;
1964 }
1965
1966 sub create_config {
1967     my @configs = @_;
1968
1969     open(OUT, ">$output_config") or dodie "Can not write to $output_config";
1970
1971     foreach my $config (@configs) {
1972         print OUT "$config_set{$config}\n";
1973         my @deps = get_dependencies $config;
1974         foreach my $dep (@deps) {
1975             print OUT "$config_set{$dep}\n";
1976         }
1977     }
1978
1979     foreach my $config (keys %config_ignore) {
1980         print OUT "$config_ignore{$config}\n";
1981     }
1982     close(OUT);
1983
1984 #    exit;
1985     make_oldconfig;
1986 }
1987
1988 sub compare_configs {
1989     my (%a, %b) = @_;
1990
1991     foreach my $item (keys %a) {
1992         if (!defined($b{$item})) {
1993             print "diff $item\n";
1994             return 1;
1995         }
1996         delete $b{$item};
1997     }
1998
1999     my @keys = keys %b;
2000     if ($#keys) {
2001         print "diff2 $keys[0]\n";
2002     }
2003     return -1 if ($#keys >= 0);
2004
2005     return 0;
2006 }
2007
2008 sub run_config_bisect_test {
2009     my ($type) = @_;
2010
2011     return run_bisect_test $type, "oldconfig";
2012 }
2013
2014 sub process_passed {
2015     my (%configs) = @_;
2016
2017     doprint "These configs had no failure: (Enabling them for further compiles)\n";
2018     # Passed! All these configs are part of a good compile.
2019     # Add them to the min options.
2020     foreach my $config (keys %configs) {
2021         if (defined($config_list{$config})) {
2022             doprint " removing $config\n";
2023             $config_ignore{$config} = $config_list{$config};
2024             delete $config_list{$config};
2025         }
2026     }
2027     doprint "config copied to $outputdir/config_good\n";
2028     run_command "cp -f $output_config $outputdir/config_good";
2029 }
2030
2031 sub process_failed {
2032     my ($config) = @_;
2033
2034     doprint "\n\n***************************************\n";
2035     doprint "Found bad config: $config\n";
2036     doprint "***************************************\n\n";
2037 }
2038
2039 sub run_config_bisect {
2040
2041     my @start_list = keys %config_list;
2042
2043     if ($#start_list < 0) {
2044         doprint "No more configs to test!!!\n";
2045         return -1;
2046     }
2047
2048     doprint "***** RUN TEST ***\n";
2049     my $type = $opt{"CONFIG_BISECT_TYPE[$iteration]"};
2050     my $ret;
2051     my %current_config;
2052
2053     my $count = $#start_list + 1;
2054     doprint "  $count configs to test\n";
2055
2056     my $half = int($#start_list / 2);
2057
2058     do {
2059         my @tophalf = @start_list[0 .. $half];
2060
2061         create_config @tophalf;
2062         read_current_config \%current_config;
2063
2064         $count = $#tophalf + 1;
2065         doprint "Testing $count configs\n";
2066         my $found = 0;
2067         # make sure we test something
2068         foreach my $config (@tophalf) {
2069             if (defined($current_config{$config})) {
2070                 logit " $config\n";
2071                 $found = 1;
2072             }
2073         }
2074         if (!$found) {
2075             # try the other half
2076             doprint "Top half produced no set configs, trying bottom half\n";
2077             @tophalf = @start_list[$half + 1 .. $#start_list];
2078             create_config @tophalf;
2079             read_current_config \%current_config;
2080             foreach my $config (@tophalf) {
2081                 if (defined($current_config{$config})) {
2082                     logit " $config\n";
2083                     $found = 1;
2084                 }
2085             }
2086             if (!$found) {
2087                 doprint "Failed: Can't make new config with current configs\n";
2088                 foreach my $config (@start_list) {
2089                     doprint "  CONFIG: $config\n";
2090                 }
2091                 return -1;
2092             }
2093             $count = $#tophalf + 1;
2094             doprint "Testing $count configs\n";
2095         }
2096
2097         $ret = run_config_bisect_test $type;
2098         if ($bisect_manual) {
2099             $ret = answer_bisect;
2100         }
2101         if ($ret) {
2102             process_passed %current_config;
2103             return 0;
2104         }
2105
2106         doprint "This config had a failure.\n";
2107         doprint "Removing these configs that were not set in this config:\n";
2108         doprint "config copied to $outputdir/config_bad\n";
2109         run_command "cp -f $output_config $outputdir/config_bad";
2110
2111         # A config exists in this group that was bad.
2112         foreach my $config (keys %config_list) {
2113             if (!defined($current_config{$config})) {
2114                 doprint " removing $config\n";
2115                 delete $config_list{$config};
2116             }
2117         }
2118
2119         @start_list = @tophalf;
2120
2121         if ($#start_list == 0) {
2122             process_failed $start_list[0];
2123             return 1;
2124         }
2125
2126         # remove half the configs we are looking at and see if
2127         # they are good.
2128         $half = int($#start_list / 2);
2129     } while ($#start_list > 0);
2130
2131     # we found a single config, try it again unless we are running manually
2132
2133     if ($bisect_manual) {
2134         process_failed $start_list[0];
2135         return 1;
2136     }
2137
2138     my @tophalf = @start_list[0 .. 0];
2139
2140     $ret = run_config_bisect_test $type;
2141     if ($ret) {
2142         process_passed %current_config;
2143         return 0;
2144     }
2145
2146     process_failed $start_list[0];
2147     return 1;
2148 }
2149
2150 sub config_bisect {
2151     my ($i) = @_;
2152
2153     my $start_config = $opt{"CONFIG_BISECT[$i]"};
2154
2155     my $tmpconfig = "$tmpdir/use_config";
2156
2157     if (defined($config_bisect_good)) {
2158         process_config_ignore $config_bisect_good;
2159     }
2160
2161     # Make the file with the bad config and the min config
2162     if (defined($minconfig)) {
2163         # read the min config for things to ignore
2164         run_command "cp $minconfig $tmpconfig" or
2165             dodie "failed to copy $minconfig to $tmpconfig";
2166     } else {
2167         unlink $tmpconfig;
2168     }
2169
2170     if (-f $tmpconfig) {
2171         load_force_config($tmpconfig);
2172         process_config_ignore $tmpconfig;
2173     }
2174
2175     # now process the start config
2176     run_command "cp $start_config $output_config" or
2177         dodie "failed to copy $start_config to $output_config";
2178
2179     # read directly what we want to check
2180     my %config_check;
2181     open (IN, $output_config)
2182         or dodie "faied to open $output_config";
2183
2184     while (<IN>) {
2185         if (/^((CONFIG\S*)=.*)/) {
2186             $config_check{$2} = $1;
2187         }
2188     }
2189     close(IN);
2190
2191     # Now run oldconfig with the minconfig
2192     make_oldconfig;
2193
2194     # check to see what we lost (or gained)
2195     open (IN, $output_config)
2196         or dodie "Failed to read $start_config";
2197
2198     my %removed_configs;
2199     my %added_configs;
2200
2201     while (<IN>) {
2202         if (/^((CONFIG\S*)=.*)/) {
2203             # save off all options
2204             $config_set{$2} = $1;
2205             if (defined($config_check{$2})) {
2206                 if (defined($config_ignore{$2})) {
2207                     $removed_configs{$2} = $1;
2208                 } else {
2209                     $config_list{$2} = $1;
2210                 }
2211             } elsif (!defined($config_ignore{$2})) {
2212                 $added_configs{$2} = $1;
2213                 $config_list{$2} = $1;
2214             }
2215         }
2216     }
2217     close(IN);
2218
2219     my @confs = keys %removed_configs;
2220     if ($#confs >= 0) {
2221         doprint "Configs overridden by default configs and removed from check:\n";
2222         foreach my $config (@confs) {
2223             doprint " $config\n";
2224         }
2225     }
2226     @confs = keys %added_configs;
2227     if ($#confs >= 0) {
2228         doprint "Configs appearing in make oldconfig and added:\n";
2229         foreach my $config (@confs) {
2230             doprint " $config\n";
2231         }
2232     }
2233
2234     my %config_test;
2235     my $once = 0;
2236
2237     # Sometimes kconfig does weird things. We must make sure
2238     # that the config we autocreate has everything we need
2239     # to test, otherwise we may miss testing configs, or
2240     # may not be able to create a new config.
2241     # Here we create a config with everything set.
2242     create_config (keys %config_list);
2243     read_current_config \%config_test;
2244     foreach my $config (keys %config_list) {
2245         if (!defined($config_test{$config})) {
2246             if (!$once) {
2247                 $once = 1;
2248                 doprint "Configs not produced by kconfig (will not be checked):\n";
2249             }
2250             doprint "  $config\n";
2251             delete $config_list{$config};
2252         }
2253     }
2254     my $ret;
2255     do {
2256         $ret = run_config_bisect;
2257     } while (!$ret);
2258
2259     return $ret if ($ret < 0);
2260
2261     success $i;
2262 }
2263
2264 sub patchcheck_reboot {
2265     doprint "Reboot and sleep $patchcheck_sleep_time seconds\n";
2266     reboot $patchcheck_sleep_time;
2267 }
2268
2269 sub patchcheck {
2270     my ($i) = @_;
2271
2272     die "PATCHCHECK_START[$i] not defined\n"
2273         if (!defined($opt{"PATCHCHECK_START[$i]"}));
2274     die "PATCHCHECK_TYPE[$i] not defined\n"
2275         if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
2276
2277     my $start = $opt{"PATCHCHECK_START[$i]"};
2278
2279     my $end = "HEAD";
2280     if (defined($opt{"PATCHCHECK_END[$i]"})) {
2281         $end = $opt{"PATCHCHECK_END[$i]"};
2282     }
2283
2284     # Get the true sha1's since we can use things like HEAD~3
2285     $start = get_sha1($start);
2286     $end = get_sha1($end);
2287
2288     my $type = $opt{"PATCHCHECK_TYPE[$i]"};
2289
2290     # Can't have a test without having a test to run
2291     if ($type eq "test" && !defined($run_test)) {
2292         $type = "boot";
2293     }
2294
2295     open (IN, "git log --pretty=oneline $end|") or
2296         dodie "could not get git list";
2297
2298     my @list;
2299
2300     while (<IN>) {
2301         chomp;
2302         $list[$#list+1] = $_;
2303         last if (/^$start/);
2304     }
2305     close(IN);
2306
2307     if ($list[$#list] !~ /^$start/) {
2308         fail "SHA1 $start not found";
2309     }
2310
2311     # go backwards in the list
2312     @list = reverse @list;
2313
2314     my $save_clean = $noclean;
2315     my %ignored_warnings;
2316
2317     if (defined($ignore_warnings)) {
2318         foreach my $sha1 (split /\s+/, $ignore_warnings) {
2319             $ignored_warnings{$sha1} = 1;
2320         }
2321     }
2322
2323     $in_patchcheck = 1;
2324     foreach my $item (@list) {
2325         my $sha1 = $item;
2326         $sha1 =~ s/^([[:xdigit:]]+).*/$1/;
2327
2328         doprint "\nProcessing commit $item\n\n";
2329
2330         run_command "git checkout $sha1" or
2331             die "Failed to checkout $sha1";
2332
2333         # only clean on the first and last patch
2334         if ($item eq $list[0] ||
2335             $item eq $list[$#list]) {
2336             $noclean = $save_clean;
2337         } else {
2338             $noclean = 1;
2339         }
2340
2341         if (defined($minconfig)) {
2342             build "useconfig:$minconfig" or return 0;
2343         } else {
2344             # ?? no config to use?
2345             build "oldconfig" or return 0;
2346         }
2347
2348
2349         if (!defined($ignored_warnings{$sha1})) {
2350             check_buildlog $sha1 or return 0;
2351         }
2352
2353         next if ($type eq "build");
2354
2355         my $failed = 0;
2356
2357         start_monitor_and_boot or $failed = 1;
2358
2359         if (!$failed && $type ne "boot"){
2360             do_run_test or $failed = 1;
2361         }
2362         end_monitor;
2363         return 0 if ($failed);
2364
2365         patchcheck_reboot;
2366
2367     }
2368     $in_patchcheck = 0;
2369     success $i;
2370
2371     return 1;
2372 }
2373
2374 my %depends;
2375 my %depcount;
2376 my $iflevel = 0;
2377 my @ifdeps;
2378
2379 # prevent recursion
2380 my %read_kconfigs;
2381
2382 sub add_dep {
2383     # $config depends on $dep
2384     my ($config, $dep) = @_;
2385
2386     if (defined($depends{$config})) {
2387         $depends{$config} .= " " . $dep;
2388     } else {
2389         $depends{$config} = $dep;
2390     }
2391
2392     # record the number of configs depending on $dep
2393     if (defined $depcount{$dep}) {
2394         $depcount{$dep}++;
2395     } else {
2396         $depcount{$dep} = 1;
2397     }
2398 }
2399
2400 # taken from streamline_config.pl
2401 sub read_kconfig {
2402     my ($kconfig) = @_;
2403
2404     my $state = "NONE";
2405     my $config;
2406     my @kconfigs;
2407
2408     my $cont = 0;
2409     my $line;
2410
2411
2412     if (! -f $kconfig) {
2413         doprint "file $kconfig does not exist, skipping\n";
2414         return;
2415     }
2416
2417     open(KIN, "$kconfig")
2418         or die "Can't open $kconfig";
2419     while (<KIN>) {
2420         chomp;
2421
2422         # Make sure that lines ending with \ continue
2423         if ($cont) {
2424             $_ = $line . " " . $_;
2425         }
2426
2427         if (s/\\$//) {
2428             $cont = 1;
2429             $line = $_;
2430             next;
2431         }
2432
2433         $cont = 0;
2434
2435         # collect any Kconfig sources
2436         if (/^source\s*"(.*)"/) {
2437             $kconfigs[$#kconfigs+1] = $1;
2438         }
2439
2440         # configs found
2441         if (/^\s*(menu)?config\s+(\S+)\s*$/) {
2442             $state = "NEW";
2443             $config = $2;
2444
2445             for (my $i = 0; $i < $iflevel; $i++) {
2446                 add_dep $config, $ifdeps[$i];
2447             }
2448
2449         # collect the depends for the config
2450         } elsif ($state eq "NEW" && /^\s*depends\s+on\s+(.*)$/) {
2451
2452             add_dep $config, $1;
2453
2454         # Get the configs that select this config
2455         } elsif ($state eq "NEW" && /^\s*select\s+(\S+)/) {
2456
2457             # selected by depends on config
2458             add_dep $1, $config;
2459
2460         # Check for if statements
2461         } elsif (/^if\s+(.*\S)\s*$/) {
2462             my $deps = $1;
2463             # remove beginning and ending non text
2464             $deps =~ s/^[^a-zA-Z0-9_]*//;
2465             $deps =~ s/[^a-zA-Z0-9_]*$//;
2466
2467             my @deps = split /[^a-zA-Z0-9_]+/, $deps;
2468
2469             $ifdeps[$iflevel++] = join ':', @deps;
2470
2471         } elsif (/^endif/) {
2472
2473             $iflevel-- if ($iflevel);
2474
2475         # stop on "help"
2476         } elsif (/^\s*help\s*$/) {
2477             $state = "NONE";
2478         }
2479     }
2480     close(KIN);
2481
2482     # read in any configs that were found.
2483     foreach $kconfig (@kconfigs) {
2484         if (!defined($read_kconfigs{$kconfig})) {
2485             $read_kconfigs{$kconfig} = 1;
2486             read_kconfig("$builddir/$kconfig");
2487         }
2488     }
2489 }
2490
2491 sub read_depends {
2492     # find out which arch this is by the kconfig file
2493     open (IN, $output_config)
2494         or dodie "Failed to read $output_config";
2495     my $arch;
2496     while (<IN>) {
2497         if (m,Linux/(\S+)\s+\S+\s+Kernel Configuration,) {
2498             $arch = $1;
2499             last;
2500         }
2501     }
2502     close IN;
2503
2504     if (!defined($arch)) {
2505         doprint "Could not find arch from config file\n";
2506         doprint "no dependencies used\n";
2507         return;
2508     }
2509
2510     # arch is really the subarch, we need to know
2511     # what directory to look at.
2512     if ($arch eq "i386" || $arch eq "x86_64") {
2513         $arch = "x86";
2514     } elsif ($arch =~ /^tile/) {
2515         $arch = "tile";
2516     }
2517
2518     my $kconfig = "$builddir/arch/$arch/Kconfig";
2519
2520     if (! -f $kconfig && $arch =~ /\d$/) {
2521         my $orig = $arch;
2522         # some subarchs have numbers, truncate them
2523         $arch =~ s/\d*$//;
2524         $kconfig = "$builddir/arch/$arch/Kconfig";
2525         if (! -f $kconfig) {
2526             doprint "No idea what arch dir $orig is for\n";
2527             doprint "no dependencies used\n";
2528             return;
2529         }
2530     }
2531
2532     read_kconfig($kconfig);
2533 }
2534
2535 sub read_config_list {
2536     my ($config) = @_;
2537
2538     open (IN, $config)
2539         or dodie "Failed to read $config";
2540
2541     while (<IN>) {
2542         if (/^((CONFIG\S*)=.*)/) {
2543             if (!defined($config_ignore{$2})) {
2544                 $config_list{$2} = $1;
2545             }
2546         }
2547     }
2548
2549     close(IN);
2550 }
2551
2552 sub read_output_config {
2553     my ($config) = @_;
2554
2555     assign_configs \%config_ignore, $config;
2556 }
2557
2558 sub make_new_config {
2559     my @configs = @_;
2560
2561     open (OUT, ">$output_config")
2562         or dodie "Failed to write $output_config";
2563
2564     foreach my $config (@configs) {
2565         print OUT "$config\n";
2566     }
2567     close OUT;
2568 }
2569
2570 sub chomp_config {
2571     my ($config) = @_;
2572
2573     $config =~ s/CONFIG_//;
2574
2575     return $config;
2576 }
2577
2578 sub get_depends {
2579     my ($dep) = @_;
2580
2581     my $kconfig = chomp_config $dep;
2582
2583     $dep = $depends{"$kconfig"};
2584
2585     # the dep string we have saves the dependencies as they
2586     # were found, including expressions like ! && ||. We
2587     # want to split this out into just an array of configs.
2588
2589     my $valid = "A-Za-z_0-9";
2590
2591     my @configs;
2592
2593     while ($dep =~ /[$valid]/) {
2594
2595         if ($dep =~ /^[^$valid]*([$valid]+)/) {
2596             my $conf = "CONFIG_" . $1;
2597
2598             $configs[$#configs + 1] = $conf;
2599
2600             $dep =~ s/^[^$valid]*[$valid]+//;
2601         } else {
2602             die "this should never happen";
2603         }
2604     }
2605
2606     return @configs;
2607 }
2608
2609 my %min_configs;
2610 my %keep_configs;
2611 my %save_configs;
2612 my %processed_configs;
2613 my %nochange_config;
2614
2615 sub test_this_config {
2616     my ($config) = @_;
2617
2618     my $found;
2619
2620     # if we already processed this config, skip it
2621     if (defined($processed_configs{$config})) {
2622         return undef;
2623     }
2624     $processed_configs{$config} = 1;
2625
2626     # if this config failed during this round, skip it
2627     if (defined($nochange_config{$config})) {
2628         return undef;
2629     }
2630
2631     my $kconfig = chomp_config $config;
2632
2633     # Test dependencies first
2634     if (defined($depends{"$kconfig"})) {
2635         my @parents = get_depends $config;
2636         foreach my $parent (@parents) {
2637             # if the parent is in the min config, check it first
2638             next if (!defined($min_configs{$parent}));
2639             $found = test_this_config($parent);
2640             if (defined($found)) {
2641                 return $found;
2642             }
2643         }
2644     }
2645
2646     # Remove this config from the list of configs
2647     # do a make oldnoconfig and then read the resulting
2648     # .config to make sure it is missing the config that
2649     # we had before
2650     my %configs = %min_configs;
2651     delete $configs{$config};
2652     make_new_config ((values %configs), (values %keep_configs));
2653     make_oldconfig;
2654     undef %configs;
2655     assign_configs \%configs, $output_config;
2656
2657     return $config if (!defined($configs{$config}));
2658
2659     doprint "disabling config $config did not change .config\n";
2660
2661     $nochange_config{$config} = 1;
2662
2663     return undef;
2664 }
2665
2666 sub make_min_config {
2667     my ($i) = @_;
2668
2669     if (!defined($output_minconfig)) {
2670         fail "OUTPUT_MIN_CONFIG not defined" and return;
2671     }
2672
2673     # If output_minconfig exists, and the start_minconfig
2674     # came from min_config, than ask if we should use
2675     # that instead.
2676     if (-f $output_minconfig && !$start_minconfig_defined) {
2677         print "$output_minconfig exists\n";
2678         if (read_yn " Use it as minconfig?") {
2679             $start_minconfig = $output_minconfig;
2680         }
2681     }
2682
2683     if (!defined($start_minconfig)) {
2684         fail "START_MIN_CONFIG or MIN_CONFIG not defined" and return;
2685     }
2686
2687     my $temp_config = "$tmpdir/temp_config";
2688
2689     # First things first. We build an allnoconfig to find
2690     # out what the defaults are that we can't touch.
2691     # Some are selections, but we really can't handle selections.
2692
2693     my $save_minconfig = $minconfig;
2694     undef $minconfig;
2695
2696     run_command "$make allnoconfig" or return 0;
2697
2698     read_depends;
2699
2700     process_config_ignore $output_config;
2701
2702     undef %save_configs;
2703     undef %min_configs;
2704
2705     if (defined($ignore_config)) {
2706         # make sure the file exists
2707         `touch $ignore_config`;
2708         assign_configs \%save_configs, $ignore_config;
2709     }
2710
2711     %keep_configs = %save_configs;
2712
2713     doprint "Load initial configs from $start_minconfig\n";
2714
2715     # Look at the current min configs, and save off all the
2716     # ones that were set via the allnoconfig
2717     assign_configs \%min_configs, $start_minconfig;
2718
2719     my @config_keys = keys %min_configs;
2720
2721     # All configs need a depcount
2722     foreach my $config (@config_keys) {
2723         my $kconfig = chomp_config $config;
2724         if (!defined $depcount{$kconfig}) {
2725                 $depcount{$kconfig} = 0;
2726         }
2727     }
2728
2729     # Remove anything that was set by the make allnoconfig
2730     # we shouldn't need them as they get set for us anyway.
2731     foreach my $config (@config_keys) {
2732         # Remove anything in the ignore_config
2733         if (defined($keep_configs{$config})) {
2734             my $file = $ignore_config;
2735             $file =~ s,.*/(.*?)$,$1,;
2736             doprint "$config set by $file ... ignored\n";
2737             delete $min_configs{$config};
2738             next;
2739         }
2740         # But make sure the settings are the same. If a min config
2741         # sets a selection, we do not want to get rid of it if
2742         # it is not the same as what we have. Just move it into
2743         # the keep configs.
2744         if (defined($config_ignore{$config})) {
2745             if ($config_ignore{$config} ne $min_configs{$config}) {
2746                 doprint "$config is in allnoconfig as '$config_ignore{$config}'";
2747                 doprint " but it is '$min_configs{$config}' in minconfig .. keeping\n";
2748                 $keep_configs{$config} = $min_configs{$config};
2749             } else {
2750                 doprint "$config set by allnoconfig ... ignored\n";
2751             }
2752             delete $min_configs{$config};
2753         }
2754     }
2755
2756     my $done = 0;
2757     my $take_two = 0;
2758
2759     while (!$done) {
2760
2761         my $config;
2762         my $found;
2763
2764         # Now disable each config one by one and do a make oldconfig
2765         # till we find a config that changes our list.
2766
2767         my @test_configs = keys %min_configs;
2768
2769         # Sort keys by who is most dependent on
2770         @test_configs = sort  { $depcount{chomp_config($b)} <=> $depcount{chomp_config($a)} }
2771                           @test_configs ;
2772
2773         # Put configs that did not modify the config at the end.
2774         my $reset = 1;
2775         for (my $i = 0; $i < $#test_configs; $i++) {
2776             if (!defined($nochange_config{$test_configs[0]})) {
2777                 $reset = 0;
2778                 last;
2779             }
2780             # This config didn't change the .config last time.
2781             # Place it at the end
2782             my $config = shift @test_configs;
2783             push @test_configs, $config;
2784         }
2785
2786         # if every test config has failed to modify the .config file
2787         # in the past, then reset and start over.
2788         if ($reset) {
2789             undef %nochange_config;
2790         }
2791
2792         undef %processed_configs;
2793
2794         foreach my $config (@test_configs) {
2795
2796             $found = test_this_config $config;
2797
2798             last if (defined($found));
2799
2800             # oh well, try another config
2801         }
2802
2803         if (!defined($found)) {
2804             # we could have failed due to the nochange_config hash
2805             # reset and try again
2806             if (!$take_two) {
2807                 undef %nochange_config;
2808                 $take_two = 1;
2809                 next;
2810             }
2811             doprint "No more configs found that we can disable\n";
2812             $done = 1;
2813             last;
2814         }
2815         $take_two = 0;
2816
2817         $config = $found;
2818
2819         doprint "Test with $config disabled\n";
2820
2821         # set in_bisect to keep build and monitor from dieing
2822         $in_bisect = 1;
2823
2824         my $failed = 0;
2825         build "oldconfig";
2826         start_monitor_and_boot or $failed = 1;
2827         end_monitor;
2828
2829         $in_bisect = 0;
2830
2831         if ($failed) {
2832             doprint "$min_configs{$config} is needed to boot the box... keeping\n";
2833             # this config is needed, add it to the ignore list.
2834             $keep_configs{$config} = $min_configs{$config};
2835             $save_configs{$config} = $min_configs{$config};
2836             delete $min_configs{$config};
2837
2838             # update new ignore configs
2839             if (defined($ignore_config)) {
2840                 open (OUT, ">$temp_config")
2841                     or die "Can't write to $temp_config";
2842                 foreach my $config (keys %save_configs) {
2843                     print OUT "$save_configs{$config}\n";
2844                 }
2845                 close OUT;
2846                 run_command "mv $temp_config $ignore_config" or
2847                     dodie "failed to copy update to $ignore_config";
2848             }
2849
2850         } else {
2851             # We booted without this config, remove it from the minconfigs.
2852             doprint "$config is not needed, disabling\n";
2853
2854             delete $min_configs{$config};
2855
2856             # Also disable anything that is not enabled in this config
2857             my %configs;
2858             assign_configs \%configs, $output_config;
2859             my @config_keys = keys %min_configs;
2860             foreach my $config (@config_keys) {
2861                 if (!defined($configs{$config})) {
2862                     doprint "$config is not set, disabling\n";
2863                     delete $min_configs{$config};
2864                 }
2865             }
2866
2867             # Save off all the current mandidory configs
2868             open (OUT, ">$temp_config")
2869                 or die "Can't write to $temp_config";
2870             foreach my $config (keys %keep_configs) {
2871                 print OUT "$keep_configs{$config}\n";
2872             }
2873             foreach my $config (keys %min_configs) {
2874                 print OUT "$min_configs{$config}\n";
2875             }
2876             close OUT;
2877
2878             run_command "mv $temp_config $output_minconfig" or
2879                 dodie "failed to copy update to $output_minconfig";
2880         }
2881
2882         doprint "Reboot and wait $sleep_time seconds\n";
2883         reboot $sleep_time;
2884     }
2885
2886     success $i;
2887     return 1;
2888 }
2889
2890 $#ARGV < 1 or die "ktest.pl version: $VERSION\n   usage: ktest.pl config-file\n";
2891
2892 if ($#ARGV == 0) {
2893     $ktest_config = $ARGV[0];
2894     if (! -f $ktest_config) {
2895         print "$ktest_config does not exist.\n";
2896         if (!read_yn "Create it?") {
2897             exit 0;
2898         }
2899     }
2900 } else {
2901     $ktest_config = "ktest.conf";
2902 }
2903
2904 if (! -f $ktest_config) {
2905     open(OUT, ">$ktest_config") or die "Can not create $ktest_config";
2906     print OUT << "EOF"
2907 # Generated by ktest.pl
2908 #
2909 # Define each test with TEST_START
2910 # The config options below it will override the defaults
2911 TEST_START
2912
2913 DEFAULTS
2914 EOF
2915 ;
2916     close(OUT);
2917 }
2918 read_config $ktest_config;
2919
2920 if (defined($opt{"LOG_FILE"})) {
2921     $opt{"LOG_FILE"} = eval_option($opt{"LOG_FILE"}, -1);
2922 }
2923
2924 # Append any configs entered in manually to the config file.
2925 my @new_configs = keys %entered_configs;
2926 if ($#new_configs >= 0) {
2927     print "\nAppending entered in configs to $ktest_config\n";
2928     open(OUT, ">>$ktest_config") or die "Can not append to $ktest_config";
2929     foreach my $config (@new_configs) {
2930         print OUT "$config = $entered_configs{$config}\n";
2931         $opt{$config} = $entered_configs{$config};
2932     }
2933 }
2934
2935 if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
2936     unlink $opt{"LOG_FILE"};
2937 }
2938
2939 doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
2940
2941 for (my $i = 0, my $repeat = 1; $i <= $opt{"NUM_TESTS"}; $i += $repeat) {
2942
2943     if (!$i) {
2944         doprint "DEFAULT OPTIONS:\n";
2945     } else {
2946         doprint "\nTEST $i OPTIONS";
2947         if (defined($repeat_tests{$i})) {
2948             $repeat = $repeat_tests{$i};
2949             doprint " ITERATE $repeat";
2950         }
2951         doprint "\n";
2952     }
2953
2954     foreach my $option (sort keys %opt) {
2955
2956         if ($option =~ /\[(\d+)\]$/) {
2957             next if ($i != $1);
2958         } else {
2959             next if ($i);
2960         }
2961
2962         doprint "$option = $opt{$option}\n";
2963     }
2964 }
2965
2966 sub __set_test_option {
2967     my ($name, $i) = @_;
2968
2969     my $option = "$name\[$i\]";
2970
2971     if (defined($opt{$option})) {
2972         return $opt{$option};
2973     }
2974
2975     foreach my $test (keys %repeat_tests) {
2976         if ($i >= $test &&
2977             $i < $test + $repeat_tests{$test}) {
2978             $option = "$name\[$test\]";
2979             if (defined($opt{$option})) {
2980                 return $opt{$option};
2981             }
2982         }
2983     }
2984
2985     if (defined($opt{$name})) {
2986         return $opt{$name};
2987     }
2988
2989     return undef;
2990 }
2991
2992 sub set_test_option {
2993     my ($name, $i) = @_;
2994
2995     my $option = __set_test_option($name, $i);
2996     return $option if (!defined($option));
2997
2998     return eval_option($option, $i);
2999 }
3000
3001 # First we need to do is the builds
3002 for (my $i = 1; $i <= $opt{"NUM_TESTS"}; $i++) {
3003
3004     # Do not reboot on failing test options
3005     $no_reboot = 1;
3006
3007     $iteration = $i;
3008
3009     my $makecmd = set_test_option("MAKE_CMD", $i);
3010
3011     $machine = set_test_option("MACHINE", $i);
3012     $ssh_user = set_test_option("SSH_USER", $i);
3013     $tmpdir = set_test_option("TMP_DIR", $i);
3014     $outputdir = set_test_option("OUTPUT_DIR", $i);
3015     $builddir = set_test_option("BUILD_DIR", $i);
3016     $test_type = set_test_option("TEST_TYPE", $i);
3017     $build_type = set_test_option("BUILD_TYPE", $i);
3018     $build_options = set_test_option("BUILD_OPTIONS", $i);
3019     $pre_build = set_test_option("PRE_BUILD", $i);
3020     $post_build = set_test_option("POST_BUILD", $i);
3021     $pre_build_die = set_test_option("PRE_BUILD_DIE", $i);
3022     $post_build_die = set_test_option("POST_BUILD_DIE", $i);
3023     $power_cycle = set_test_option("POWER_CYCLE", $i);
3024     $reboot = set_test_option("REBOOT", $i);
3025     $noclean = set_test_option("BUILD_NOCLEAN", $i);
3026     $minconfig = set_test_option("MIN_CONFIG", $i);
3027     $output_minconfig = set_test_option("OUTPUT_MIN_CONFIG", $i);
3028     $start_minconfig = set_test_option("START_MIN_CONFIG", $i);
3029     $ignore_config = set_test_option("IGNORE_CONFIG", $i);
3030     $run_test = set_test_option("TEST", $i);
3031     $addconfig = set_test_option("ADD_CONFIG", $i);
3032     $reboot_type = set_test_option("REBOOT_TYPE", $i);
3033     $grub_menu = set_test_option("GRUB_MENU", $i);
3034     $post_install = set_test_option("POST_INSTALL", $i);
3035     $no_install = set_test_option("NO_INSTALL", $i);
3036     $reboot_script = set_test_option("REBOOT_SCRIPT", $i);
3037     $reboot_on_error = set_test_option("REBOOT_ON_ERROR", $i);
3038     $poweroff_on_error = set_test_option("POWEROFF_ON_ERROR", $i);
3039     $die_on_failure = set_test_option("DIE_ON_FAILURE", $i);
3040     $power_off = set_test_option("POWER_OFF", $i);
3041     $powercycle_after_reboot = set_test_option("POWERCYCLE_AFTER_REBOOT", $i);
3042     $poweroff_after_halt = set_test_option("POWEROFF_AFTER_HALT", $i);
3043     $sleep_time = set_test_option("SLEEP_TIME", $i);
3044     $bisect_sleep_time = set_test_option("BISECT_SLEEP_TIME", $i);
3045     $patchcheck_sleep_time = set_test_option("PATCHCHECK_SLEEP_TIME", $i);
3046     $ignore_warnings = set_test_option("IGNORE_WARNINGS", $i);
3047     $bisect_manual = set_test_option("BISECT_MANUAL", $i);
3048     $bisect_skip = set_test_option("BISECT_SKIP", $i);
3049     $config_bisect_good = set_test_option("CONFIG_BISECT_GOOD", $i);
3050     $store_failures = set_test_option("STORE_FAILURES", $i);
3051     $test_name = set_test_option("TEST_NAME", $i);
3052     $timeout = set_test_option("TIMEOUT", $i);
3053     $booted_timeout = set_test_option("BOOTED_TIMEOUT", $i);
3054     $console = set_test_option("CONSOLE", $i);
3055     $detect_triplefault = set_test_option("DETECT_TRIPLE_FAULT", $i);
3056     $success_line = set_test_option("SUCCESS_LINE", $i);
3057     $reboot_success_line = set_test_option("REBOOT_SUCCESS_LINE", $i);
3058     $stop_after_success = set_test_option("STOP_AFTER_SUCCESS", $i);
3059     $stop_after_failure = set_test_option("STOP_AFTER_FAILURE", $i);
3060     $stop_test_after = set_test_option("STOP_TEST_AFTER", $i);
3061     $build_target = set_test_option("BUILD_TARGET", $i);
3062     $ssh_exec = set_test_option("SSH_EXEC", $i);
3063     $scp_to_target = set_test_option("SCP_TO_TARGET", $i);
3064     $target_image = set_test_option("TARGET_IMAGE", $i);
3065     $localversion = set_test_option("LOCALVERSION", $i);
3066
3067     $start_minconfig_defined = 1;
3068
3069     if (!defined($start_minconfig)) {
3070         $start_minconfig_defined = 0;
3071         $start_minconfig = $minconfig;
3072     }
3073
3074     chdir $builddir || die "can't change directory to $builddir";
3075
3076     foreach my $dir ($tmpdir, $outputdir) {
3077         if (!-d $dir) {
3078             mkpath($dir) or
3079                 die "can't create $dir";
3080         }
3081     }
3082
3083     $ENV{"SSH_USER"} = $ssh_user;
3084     $ENV{"MACHINE"} = $machine;
3085
3086     $target = "$ssh_user\@$machine";
3087
3088     $buildlog = "$tmpdir/buildlog-$machine";
3089     $dmesg = "$tmpdir/dmesg-$machine";
3090     $make = "$makecmd O=$outputdir";
3091     $output_config = "$outputdir/.config";
3092
3093     if ($reboot_type eq "grub") {
3094         dodie "GRUB_MENU not defined" if (!defined($grub_menu));
3095     } elsif (!defined($reboot_script)) {
3096         dodie "REBOOT_SCRIPT not defined"
3097     }
3098
3099     my $run_type = $build_type;
3100     if ($test_type eq "patchcheck") {
3101         $run_type = $opt{"PATCHCHECK_TYPE[$i]"};
3102     } elsif ($test_type eq "bisect") {
3103         $run_type = $opt{"BISECT_TYPE[$i]"};
3104     } elsif ($test_type eq "config_bisect") {
3105         $run_type = $opt{"CONFIG_BISECT_TYPE[$i]"};
3106     }
3107
3108     if ($test_type eq "make_min_config") {
3109         $run_type = "";
3110     }
3111
3112     # mistake in config file?
3113     if (!defined($run_type)) {
3114         $run_type = "ERROR";
3115     }
3116
3117     my $installme = "";
3118     $installme = " no_install" if ($no_install);
3119
3120     doprint "\n\n";
3121     doprint "RUNNING TEST $i of $opt{NUM_TESTS} with option $test_type $run_type$installme\n\n";
3122
3123     unlink $dmesg;
3124     unlink $buildlog;
3125
3126     if (defined($addconfig)) {
3127         my $min = $minconfig;
3128         if (!defined($minconfig)) {
3129             $min = "";
3130         }
3131         run_command "cat $addconfig $min > $tmpdir/add_config" or
3132             dodie "Failed to create temp config";
3133         $minconfig = "$tmpdir/add_config";
3134     }
3135
3136     my $checkout = $opt{"CHECKOUT[$i]"};
3137     if (defined($checkout)) {
3138         run_command "git checkout $checkout" or
3139             die "failed to checkout $checkout";
3140     }
3141
3142     $no_reboot = 0;
3143
3144
3145     if ($test_type eq "bisect") {
3146         bisect $i;
3147         next;
3148     } elsif ($test_type eq "config_bisect") {
3149         config_bisect $i;
3150         next;
3151     } elsif ($test_type eq "patchcheck") {
3152         patchcheck $i;
3153         next;
3154     } elsif ($test_type eq "make_min_config") {
3155         make_min_config $i;
3156         next;
3157     }
3158
3159     if ($build_type ne "nobuild") {
3160         build $build_type or next;
3161     }
3162
3163     if ($test_type eq "install") {
3164         get_version;
3165         install;
3166         success $i;
3167         next;
3168     }
3169
3170     if ($test_type ne "build") {
3171         my $failed = 0;
3172         start_monitor_and_boot or $failed = 1;
3173
3174         if (!$failed && $test_type ne "boot" && defined($run_test)) {
3175             do_run_test or $failed = 1;
3176         }
3177         end_monitor;
3178         next if ($failed);
3179     }
3180
3181     success $i;
3182 }
3183
3184 if ($opt{"POWEROFF_ON_SUCCESS"}) {
3185     halt;
3186 } elsif ($opt{"REBOOT_ON_SUCCESS"} && !do_not_reboot) {
3187     reboot;
3188 }
3189
3190 doprint "\n    $successes of $opt{NUM_TESTS} tests were successful\n\n";
3191
3192 exit 0;