tools/testing/ktest/ktest.pl

   1 #!/usr/bin/perl -w
   2
   3 use strict;
   4 use IPC::Open2;
   5 use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
   6 use FileHandle;
   7
   8 $#ARGV >= 0 || die "usage: autotest.pl config-file\n";
   9
  10 $| = 1;
  11
  12 my %opt;
  13
  14 #default opts
  15 $opt{"NUM_BUILDS"}              = 5;
  16 $opt{"DEFAULT_BUILD_TYPE"}      = "randconfig";
  17 $opt{"MAKE_CMD"}                = "make";
  18 $opt{"TIMEOUT"}                 = 50;
  19 $opt{"TMP_DIR"}                 = "/tmp/autotest";
  20 $opt{"SLEEP_TIME"}              = 60;   # sleep time between tests
  21 $opt{"BUILD_NOCLEAN"}           = 0;
  22 $opt{"REBOOT_ON_ERROR"}         = 0;
  23 $opt{"POWEROFF_ON_ERROR"}       = 0;
  24 $opt{"POWEROFF_ON_SUCCESS"}     = 0;
  25 $opt{"BUILD_OPTIONS"}           = "";
  26 $opt{"BISECT_SLEEP_TIME"}       = 10;   # sleep time between bisects
  27
  28 my $version;
  29 my $grub_number;
  30 my $target;
  31 my $make;
  32 my $noclean;
  33 my $minconfig;
  34 my $in_bisect = 0;
  35 my $bisect_bad = "";
  36 my $run_test;
  37
  38 sub read_config {
  39     my ($config) = @_;
  40
  41     open(IN, $config) || die "can't read file $config";
  42
  43     while (<IN>) {
  44
  45         # ignore blank lines and comments
  46         next if (/^\s*$/ || /\s*\#/);
  47
  48         if (/^\s*(\S+)\s*=\s*(.*?)\s*$/) {
  49             my $lvalue = $1;
  50             my $rvalue = $2;
  51
  52             $opt{$lvalue} = $rvalue;
  53         }
  54     }
  55
  56     close(IN);
  57 }
  58
  59 sub logit {
  60     if (defined($opt{"LOG_FILE"})) {
  61         open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
  62         print OUT @_;
  63         close(OUT);
  64     }
  65 }
  66
  67 sub doprint {
  68     print @_;
  69     logit @_;
  70 }
  71
  72 sub dodie {
  73     doprint "CRITICAL FAILURE... ", @_, "\n";
  74
  75     if ($opt{"REBOOT_ON_ERROR"}) {
  76         doprint "REBOOTING\n";
  77         `$opt{"POWER_CYCLE"}`;
  78
  79     } elsif ($opt{"POWEROFF_ON_ERROR"} && defined($opt{"POWER_OFF"})) {
  80         doprint "POWERING OFF\n";
  81         `$opt{"POWER_OFF"}`;
  82     }
  83
  84     die @_;
  85 }
  86
  87 sub run_command {
  88     my ($command) = @_;
  89     my $redirect_log = "";
  90
  91     if (defined($opt{"LOG_FILE"})) {
  92         $redirect_log = " >> $opt{LOG_FILE} 2>&1";
  93     }
  94
  95     doprint "$command ... ";
  96     `$command $redirect_log`;
  97
  98     my $failed = $?;
  99
 100     if ($failed) {
 101         doprint "FAILED!\n";
 102     } else {
 103         doprint "SUCCESS\n";
 104     }
 105
 106     return !$failed;
 107 }
 108
 109 sub get_grub_index {
 110
 111     return if (defined($grub_number));
 112
 113     doprint "Find grub menu ... ";
 114     $grub_number = -1;
 115     open(IN, "ssh $target cat /boot/grub/menu.lst |")
 116         or die "unable to get menu.lst";
 117     while (<IN>) {
 118         if (/^\s*title\s+$opt{GRUB_MENU}\s*$/) {
 119             $grub_number++;
 120             last;
 121         } elsif (/^\s*title\s/) {
 122             $grub_number++;
 123         }
 124     }
 125     close(IN);
 126
 127     die "Could not find '$opt{GRUB_MENU}' in /boot/grub/menu on $opt{MACHINE}"
 128         if ($grub_number < 0);
 129     doprint "$grub_number\n";
 130 }
 131
 132 my $timeout = $opt{"TIMEOUT"};
 133
 134 sub wait_for_input
 135 {
 136     my ($fp, $time) = @_;
 137     my $rin;
 138     my $ready;
 139     my $line;
 140     my $ch;
 141
 142     if (!defined($time)) {
 143         $time = $timeout;
 144     }
 145
 146     $rin = '';
 147     vec($rin, fileno($fp), 1) = 1;
 148     $ready = select($rin, undef, undef, $time);
 149
 150     $line = "";
 151
 152     # try to read one char at a time
 153     while (sysread $fp, $ch, 1) {
 154         $line .= $ch;
 155         last if ($ch eq "\n");
 156     }
 157
 158     if (!length($line)) {
 159         return undef;
 160     }
 161
 162     return $line;
 163 }
 164
 165 sub reboot_to {
 166     run_command "ssh $target '(echo \"savedefault --default=$grub_number --once\" | grub --batch; reboot)'";
 167 }
 168
 169 sub open_console {
 170     my ($fp) = @_;
 171
 172     my $flags;
 173
 174     my $pid = open($fp, "$opt{CONSOLE}|") or
 175         dodie "Can't open console $opt{CONSOLE}";
 176
 177     $flags = fcntl($fp, F_GETFL, 0) or
 178         dodie "Can't get flags for the socket: $!\n";
 179     $flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
 180         dodie "Can't set flags for the socket: $!\n";
 181
 182     return $pid;
 183 }
 184
 185 sub close_console {
 186     my ($fp, $pid) = @_;
 187
 188     doprint "kill child process $pid\n";
 189     kill 2, $pid;
 190
 191     print "closing!\n";
 192     close($fp);
 193 }
 194
 195 sub monitor {
 196     my $booted = 0;
 197     my $bug = 0;
 198     my $pid;
 199     my $skip_call_trace = 0;
 200     my $fp = \*IN;
 201
 202     $pid = open_console($fp);
 203
 204     my $line;
 205     my $full_line = "";
 206
 207     doprint "Wait for monitor to settle down.\n";
 208     # read the monitor and wait for the system to calm down
 209     do {
 210         $line = wait_for_input($fp, 5);
 211     } while (defined($line));
 212
 213     reboot_to;
 214
 215     for (;;) {
 216
 217         $line = wait_for_input($fp);
 218
 219         last if (!defined($line));
 220
 221         doprint $line;
 222
 223         # we are not guaranteed to get a full line
 224         $full_line .= $line;
 225
 226         if ($full_line =~ /login:/) {
 227             $booted = 1;
 228         }
 229
 230         if ($full_line =~ /\[ backtrace testing \]/) {
 231             $skip_call_trace = 1;
 232         }
 233
 234         if ($full_line =~ /call trace:/i) {
 235             $bug = 1 if (!$skip_call_trace);
 236         }
 237
 238         if ($full_line =~ /\[ end of backtrace testing \]/) {
 239             $skip_call_trace = 0;
 240         }
 241
 242         if ($full_line =~ /Kernel panic -/) {
 243             $bug = 1;
 244         }
 245
 246         if ($line =~ /\n/) {
 247             $full_line = "";
 248         }
 249     }
 250
 251     close_console($fp, $pid);
 252
 253     if (!$booted) {
 254         return 1 if ($in_bisect);
 255         dodie "failed - never got a boot prompt.\n";
 256     }
 257
 258     if ($bug) {
 259         return 1 if ($in_bisect);
 260         dodie "failed - got a bug report\n";
 261     }
 262
 263     return 0;
 264 }
 265
 266 sub install {
 267
 268     run_command "scp $opt{OUTPUT_DIR}/$opt{BUILD_TARGET} $target:$opt{TARGET_IMAGE}" or
 269         dodie "failed to copy image";
 270
 271     my $install_mods = 0;
 272
 273     # should we process modules?
 274     $install_mods = 0;
 275     open(IN, "$opt{OUTPUT_DIR}/.config") or dodie("Can't read config file");
 276     while (<IN>) {
 277         if (/CONFIG_MODULES(=y)?/) {
 278             $install_mods = 1 if (defined($1));
 279             last;
 280         }
 281     }
 282     close(IN);
 283
 284     if (!$install_mods) {
 285         doprint "No modules needed\n";
 286         return;
 287     }
 288
 289     run_command "$make INSTALL_MOD_PATH=$opt{TMP_DIR} modules_install" or
 290         dodie "Failed to install modules";
 291
 292     my $modlib = "/lib/modules/$version";
 293     my $modtar = "autotest-mods.tar.bz2";
 294
 295     run_command "ssh $target rm -rf $modlib" or
 296         dodie "failed to remove old mods: $modlib";
 297
 298     # would be nice if scp -r did not follow symbolic links
 299     run_command "cd $opt{TMP_DIR} && tar -cjf $modtar lib/modules/$version" or
 300         dodie "making tarball";
 301
 302     run_command "scp $opt{TMP_DIR}/$modtar $target:/tmp" or
 303         dodie "failed to copy modules";
 304
 305     unlink "$opt{TMP_DIR}/$modtar";
 306
 307     run_command "ssh $target '(cd / && tar xf /tmp/$modtar)'" or
 308         dodie "failed to tar modules";
 309
 310     run_command "ssh $target rm -f /tmp/$modtar";
 311 }
 312
 313 sub build {
 314     my ($type) = @_;
 315     my $defconfig = "";
 316     my $append = "";
 317
 318     if ($type =~ /^useconfig:(.*)/) {
 319         run_command "cp $1 $opt{OUTPUT_DIR}/.config" or
 320             dodie "could not copy $1 to .config";
 321
 322         $type = "oldconfig";
 323     }
 324
 325     # old config can ask questions
 326     if ($type eq "oldconfig") {
 327         $append = "yes ''|";
 328
 329         # allow for empty configs
 330         run_command "touch $opt{OUTPUT_DIR}/.config";
 331
 332         run_command "mv $opt{OUTPUT_DIR}/.config $opt{OUTPUT_DIR}/config_temp" or
 333             dodie "moving .config";
 334
 335         if (!$noclean && !run_command "$make mrproper") {
 336             dodie "make mrproper";
 337         }
 338
 339         run_command "mv $opt{OUTPUT_DIR}/config_temp $opt{OUTPUT_DIR}/.config" or
 340             dodie "moving config_temp";
 341
 342     } elsif (!$noclean) {
 343         unlink "$opt{OUTPUT_DIR}/.config";
 344         run_command "$make mrproper" or
 345             dodie "make mrproper";
 346     }
 347
 348     # add something to distinguish this build
 349     open(OUT, "> $opt{OUTPUT_DIR}/localversion") or dodie("Can't make localversion file");
 350     print OUT "$opt{LOCALVERSION}\n";
 351     close(OUT);
 352
 353     if (defined($minconfig)) {
 354         $defconfig = "KCONFIG_ALLCONFIG=$minconfig";
 355     }
 356
 357     run_command "$defconfig $append $make $type" or
 358         dodie "failed make config";
 359
 360     if (!run_command "$make $opt{BUILD_OPTIONS}") {
 361         # bisect may need this to pass
 362         return 1 if ($in_bisect);
 363         dodie "failed build";
 364     }
 365
 366     return 0;
 367 }
 368
 369 sub reboot {
 370     # try to reboot normally
 371     if (!run_command "ssh $target reboot") {
 372         # nope? power cycle it.
 373         run_command "$opt{POWER_CYCLE}";
 374     }
 375 }
 376
 377 sub halt {
 378     if (!run_command "ssh $target halt" or defined($opt{"POWER_OFF"})) {
 379         # nope? the zap it!
 380         run_command "$opt{POWER_OFF}";
 381     }
 382 }
 383
 384 sub success {
 385     my ($i) = @_;
 386
 387     doprint "\n\n*******************************************\n";
 388     doprint     "*******************************************\n";
 389     doprint     "**            SUCCESS!!!!                **\n";
 390     doprint     "*******************************************\n";
 391     doprint     "*******************************************\n";
 392
 393     if ($i != $opt{"NUM_BUILDS"}) {
 394         reboot;
 395         doprint "Sleeping $opt{SLEEP_TIME} seconds\n";
 396         sleep "$opt{SLEEP_TIME}";
 397     }
 398 }
 399
 400 sub get_version {
 401     # get the release name
 402     doprint "$make kernelrelease ... ";
 403     $version = `$make kernelrelease | tail -1`;
 404     chomp($version);
 405     doprint "$version\n";
 406 }
 407
 408 sub child_run_test {
 409     my $failed;
 410
 411     $failed = !run_command $run_test;
 412     exit $failed;
 413 }
 414
 415 my $child_done;
 416
 417 sub child_finished {
 418     $child_done = 1;
 419 }
 420
 421 sub do_run_test {
 422     my $child_pid;
 423     my $child_exit;
 424     my $pid;
 425     my $line;
 426     my $full_line;
 427     my $bug = 0;
 428     my $fp = \*IN;
 429
 430     $pid = open_console($fp);
 431
 432     # read the monitor and wait for the system to calm down
 433     do {
 434         $line = wait_for_input($fp, 1);
 435     } while (defined($line));
 436
 437     $child_done = 0;
 438
 439     $SIG{CHLD} = qw(child_finished);
 440
 441     $child_pid = fork;
 442
 443     child_run_test if (!$child_pid);
 444
 445     $full_line = "";
 446
 447     do {
 448         $line = wait_for_input($fp, 1);
 449         if (defined($line)) {
 450
 451             # we are not guaranteed to get a full line
 452             $full_line .= $line;
 453
 454             if ($full_line =~ /call trace:/i) {
 455                 $bug = 1;
 456             }
 457
 458             if ($full_line =~ /Kernel panic -/) {
 459                 $bug = 1;
 460             }
 461
 462             if ($line =~ /\n/) {
 463                 $full_line = "";
 464             }
 465         }
 466     } while (!$child_done && !$bug);
 467
 468     if ($bug) {
 469         doprint "Detected kernel crash!\n";
 470         # kill the child with extreme prejudice
 471         kill 9, $child_pid;
 472     }
 473
 474     waitpid $child_pid, 0;
 475     $child_exit = $?;
 476
 477     close_console($fp, $pid);
 478
 479     if ($bug || $child_exit) {
 480         return 1 if $in_bisect;
 481         dodie "test failed";
 482     }
 483     return 0;
 484 }
 485
 486 sub run_bisect {
 487     my ($type) = @_;
 488
 489     my $failed;
 490     my $result;
 491     my $output;
 492     my $ret;
 493
 494
 495     if (defined($minconfig)) {
 496         $failed = build "useconfig:$minconfig";
 497     } else {
 498         # ?? no config to use?
 499         $failed = build "oldconfig";
 500     }
 501
 502     if ($type ne "build") {
 503         dodie "Failed on build" if $failed;
 504
 505         # Now boot the box
 506         get_grub_index;
 507         get_version;
 508         install;
 509         $failed = monitor;
 510
 511         if ($type ne "boot") {
 512             dodie "Failed on boot" if $failed;
 513
 514             $failed = do_run_test;
 515         }
 516     }
 517
 518     if ($failed) {
 519         $result = "bad";
 520
 521         # reboot the box to a good kernel
 522         if ($type eq "boot") {
 523             reboot;
 524             doprint "sleep a little for reboot\n";
 525             sleep $opt{"BISECT_SLEEP_TIME"};
 526         }
 527     } else {
 528         $result = "good";
 529     }
 530
 531     doprint "git bisect $result ... ";
 532     $output = `git bisect $result 2>&1`;
 533     $ret = $?;
 534
 535     logit $output;
 536
 537     if ($ret) {
 538         doprint "FAILED\n";
 539         dodie "Failed to git bisect";
 540     }
 541
 542     doprint "SUCCESS\n";
 543     if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
 544         doprint "$1 [$2]\n";
 545     } elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
 546         $bisect_bad = $1;
 547         doprint "Found bad commit... $1\n";
 548         return 0;
 549     } else {
 550         # we already logged it, just print it now.
 551         print $output;
 552     }
 553
 554
 555     return 1;
 556 }
 557
 558 sub bisect {
 559     my ($i) = @_;
 560
 561     my $result;
 562
 563     die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
 564     die "BISECT_BAD[$i] not defined\n"  if (!defined($opt{"BISECT_BAD[$i]"}));
 565     die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
 566
 567     my $good = $opt{"BISECT_GOOD[$i]"};
 568     my $bad = $opt{"BISECT_BAD[$i]"};
 569     my $type = $opt{"BISECT_TYPE[$i]"};
 570
 571     $in_bisect = 1;
 572
 573     run_command "git bisect start" or
 574         dodie "could not start bisect";
 575
 576     run_command "git bisect good $good" or
 577         dodie "could not set bisect good to $good";
 578
 579     run_command "git bisect bad $bad" or
 580         dodie "could not set bisect good to $bad";
 581
 582     # Can't have a test without having a test to run
 583     if ($type eq "test" && !defined($run_test)) {
 584         $type = "boot";
 585     }
 586
 587     do {
 588         $result = run_bisect $type;
 589     } while ($result);
 590
 591     run_command "git bisect log" or
 592         dodie "could not capture git bisect log";
 593
 594     run_command "git bisect reset" or
 595         dodie "could not reset git bisect";
 596
 597     doprint "Bad commit was [$bisect_bad]\n";
 598
 599     $in_bisect = 0;
 600
 601     success $i;
 602 }
 603
 604 read_config $ARGV[0];
 605
 606 # mandatory configs
 607 die "MACHINE not defined\n"             if (!defined($opt{"MACHINE"}));
 608 die "SSH_USER not defined\n"            if (!defined($opt{"SSH_USER"}));
 609 die "BUILD_DIR not defined\n"           if (!defined($opt{"BUILD_DIR"}));
 610 die "OUTPUT_DIR not defined\n"          if (!defined($opt{"OUTPUT_DIR"}));
 611 die "BUILD_TARGET not defined\n"        if (!defined($opt{"BUILD_TARGET"}));
 612 die "TARGET_IMAGE not defined\n"        if (!defined($opt{"TARGET_IMAGE"}));
 613 die "POWER_CYCLE not defined\n"         if (!defined($opt{"POWER_CYCLE"}));
 614 die "CONSOLE not defined\n"             if (!defined($opt{"CONSOLE"}));
 615 die "LOCALVERSION not defined\n"        if (!defined($opt{"LOCALVERSION"}));
 616 die "GRUB_MENU not defined\n"           if (!defined($opt{"GRUB_MENU"}));
 617
 618 chdir $opt{"BUILD_DIR"} || die "can't change directory to $opt{BUILD_DIR}";
 619
 620 $target = "$opt{SSH_USER}\@$opt{MACHINE}";
 621
 622 doprint "\n\nSTARTING AUTOMATED TESTS\n";
 623
 624
 625 $make = "$opt{MAKE_CMD} O=$opt{OUTPUT_DIR}";
 626
 627 sub set_build_option {
 628     my ($name, $i) = @_;
 629
 630     my $option = "$name\[$i\]";
 631
 632     if (defined($opt{$option})) {
 633         return $opt{$option};
 634     }
 635
 636     if (defined($opt{$name})) {
 637         return $opt{$name};
 638     }
 639
 640     return undef;
 641 }
 642
 643 # First we need to do is the builds
 644 for (my $i = 1; $i <= $opt{"NUM_BUILDS"}; $i++) {
 645     my $type = "BUILD_TYPE[$i]";
 646
 647     $noclean = set_build_option("BUILD_NOCLEAN", $i);
 648     $minconfig = set_build_option("MIN_CONFIG", $i);
 649     $run_test = set_build_option("TEST", $i);
 650
 651     doprint "\n\n";
 652     doprint "RUNNING TEST $i of $opt{NUM_BUILDS} with option $opt{$type}\n\n";
 653
 654     if ($opt{$type} eq "bisect") {
 655         bisect $i;
 656         next;
 657     }
 658
 659     if ($opt{$type} ne "nobuild") {
 660         build $opt{$type};
 661     }
 662
 663     get_grub_index;
 664     get_version;
 665     install;
 666     monitor;
 667
 668     if (defined($run_test)) {
 669         do_run_test;
 670     }
 671
 672     success $i;
 673 }
 674
 675 if ($opt{"POWEROFF_ON_SUCCESS"}) {
 676     halt;
 677 } else {
 678     reboot;
 679 }
 680
 681 exit 0;