server/autotest.py

   1 # Copyright 2007 Google Inc. Released under the GPL v2
   2
   3 import re, os, sys, traceback, subprocess, time, pickle, glob, tempfile
   4 import logging, getpass
   5 from autotest_lib.server import installable_object, prebuild, utils
   6 from autotest_lib.client.common_lib import base_job, log, error, autotemp
   7 from autotest_lib.client.common_lib import global_config, packages
   8 from autotest_lib.client.common_lib import utils as client_utils
   9
  10 AUTOTEST_SVN  = 'svn://test.kernel.org/autotest/trunk/client'
  11 AUTOTEST_HTTP = 'http://test.kernel.org/svn/autotest/trunk/client'
  12
  13 # Timeouts for powering down and up respectively
  14 HALT_TIME = 300
  15 BOOT_TIME = 1800
  16 CRASH_RECOVERY_TIME = 9000
  17
  18
  19 get_value = global_config.global_config.get_config_value
  20 autoserv_prebuild = get_value('AUTOSERV', 'enable_server_prebuild',
  21                               type=bool, default=False)
  22
  23
  24 class AutodirNotFoundError(Exception):
  25     """No Autotest installation could be found."""
  26
  27
  28 class BaseAutotest(installable_object.InstallableObject):
  29     """
  30     This class represents the Autotest program.
  31
  32     Autotest is used to run tests automatically and collect the results.
  33     It also supports profilers.
  34
  35     Implementation details:
  36     This is a leaf class in an abstract class hierarchy, it must
  37     implement the unimplemented methods in parent classes.
  38     """
  39
  40     def __init__(self, host = None):
  41         self.host = host
  42         self.got = False
  43         self.installed = False
  44         self.serverdir = utils.get_server_dir()
  45         super(BaseAutotest, self).__init__()
  46
  47
  48     install_in_tmpdir = False
  49     @classmethod
  50     def set_install_in_tmpdir(cls, flag):
  51         """ Sets a flag that controls whether or not Autotest should by
  52         default be installed in a "standard" directory (e.g.
  53         /home/autotest, /usr/local/autotest) or a temporary directory. """
  54         cls.install_in_tmpdir = flag
  55
  56
  57     @classmethod
  58     def get_client_autodir_paths(cls, host):
  59         return global_config.global_config.get_config_value(
  60                 'AUTOSERV', 'client_autodir_paths', type=list)
  61
  62
  63     @classmethod
  64     def get_installed_autodir(cls, host):
  65         """
  66         Find where the Autotest client is installed on the host.
  67         @returns an absolute path to an installed Autotest client root.
  68         @raises AutodirNotFoundError if no Autotest installation can be found.
  69         """
  70         autodir = host.get_autodir()
  71         if autodir:
  72             logging.debug('Using existing host autodir: %s', autodir)
  73             return autodir
  74
  75         for path in Autotest.get_client_autodir_paths(host):
  76             try:
  77                 autotest_binary = os.path.join(path, 'bin', 'autotest')
  78                 host.run('test -x %s' % utils.sh_escape(autotest_binary))
  79                 host.run('test -w %s' % utils.sh_escape(path))
  80                 logging.debug('Found existing autodir at %s', path)
  81                 return path
  82             except error.AutoservRunError:
  83                 logging.debug('%s does not exist on %s', autotest_binary,
  84                               host.hostname)
  85         raise AutodirNotFoundError
  86
  87
  88     @classmethod
  89     def get_install_dir(cls, host):
  90         """
  91         Determines the location where autotest should be installed on
  92         host. If self.install_in_tmpdir is set, it will return a unique
  93         temporary directory that autotest can be installed in. Otherwise, looks
  94         for an existing installation to use; if none is found, looks for a
  95         usable directory in the global config client_autodir_paths.
  96         """
  97         try:
  98             install_dir = cls.get_installed_autodir(host)
  99         except AutodirNotFoundError:
 100             install_dir = cls._find_installable_dir(host)
 101
 102         if cls.install_in_tmpdir:
 103             return host.get_tmp_dir(parent=install_dir)
 104         return install_dir
 105
 106
 107     @classmethod
 108     def _find_installable_dir(cls, host):
 109         client_autodir_paths = cls.get_client_autodir_paths(host)
 110         for path in client_autodir_paths:
 111             try:
 112                 host.run('mkdir -p %s' % utils.sh_escape(path))
 113                 host.run('test -w %s' % utils.sh_escape(path))
 114                 return path
 115             except error.AutoservRunError:
 116                 logging.debug('Failed to create %s', path)
 117         raise error.AutoservInstallError(
 118                 'Unable to find a place to install Autotest; tried %s',
 119                 ', '.join(client_autodir_paths))
 120
 121
 122     def install(self, host=None, autodir=None):
 123         self._install(host=host, autodir=autodir)
 124
 125
 126     def install_full_client(self, host=None, autodir=None):
 127         self._install(host=host, autodir=autodir, use_autoserv=False,
 128                       use_packaging=False)
 129
 130
 131     def install_no_autoserv(self, host=None, autodir=None):
 132         self._install(host=host, autodir=autodir, use_autoserv=False)
 133
 134
 135     def _install_using_packaging(self, host, autodir):
 136         c = global_config.global_config
 137         repos = c.get_config_value("PACKAGES", 'fetch_location', type=list,
 138                                    default=[])
 139         repos.reverse()
 140         if not repos:
 141             raise error.PackageInstallError("No repos to install an "
 142                                             "autotest client from")
 143         pkgmgr = packages.PackageManager(autodir, hostname=host.hostname,
 144                                          repo_urls=repos,
 145                                          do_locking=False,
 146                                          run_function=host.run,
 147                                          run_function_dargs=dict(timeout=600))
 148         # The packages dir is used to store all the packages that
 149         # are fetched on that client. (for the tests,deps etc.
 150         # too apart from the client)
 151         pkg_dir = os.path.join(autodir, 'packages')
 152         # clean up the autodir except for the packages directory
 153         host.run('cd %s && ls | grep -v "^packages$"'
 154                  ' | xargs rm -rf && rm -rf .[^.]*' % autodir)
 155         pkgmgr.install_pkg('autotest', 'client', pkg_dir, autodir,
 156                            preserve_install_dir=True)
 157         self.installed = True
 158
 159
 160     def _install_using_send_file(self, host, autodir):
 161         dirs_to_exclude = set(["tests", "site_tests", "deps", "profilers"])
 162         light_files = [os.path.join(self.source_material, f)
 163                        for f in os.listdir(self.source_material)
 164                        if f not in dirs_to_exclude]
 165         host.send_file(light_files, autodir, delete_dest=True)
 166
 167         # create empty dirs for all the stuff we excluded
 168         commands = []
 169         for path in dirs_to_exclude:
 170             abs_path = os.path.join(autodir, path)
 171             abs_path = utils.sh_escape(abs_path)
 172             commands.append("mkdir -p '%s'" % abs_path)
 173             commands.append("touch '%s'/__init__.py" % abs_path)
 174         host.run(';'.join(commands))
 175
 176
 177     def _install(self, host=None, autodir=None, use_autoserv=True,
 178                  use_packaging=True):
 179         """
 180         Install autotest.  If get() was not called previously, an
 181         attempt will be made to install from the autotest svn
 182         repository.
 183
 184         @param host A Host instance on which autotest will be installed
 185         @param autodir Location on the remote host to install to
 186         @param use_autoserv Enable install modes that depend on the client
 187             running with the autoserv harness
 188         @param use_packaging Enable install modes that use the packaging system
 189
 190         @exception AutoservError if a tarball was not specified and
 191             the target host does not have svn installed in its path
 192         """
 193         if not host:
 194             host = self.host
 195         if not self.got:
 196             self.get()
 197         host.wait_up(timeout=30)
 198         host.setup()
 199         logging.info("Installing autotest on %s", host.hostname)
 200
 201         # set up the autotest directory on the remote machine
 202         if not autodir:
 203             autodir = self.get_install_dir(host)
 204         logging.info('Using installation dir %s', autodir)
 205         host.set_autodir(autodir)
 206         host.run('mkdir -p %s' % utils.sh_escape(autodir))
 207
 208         # make sure there are no files in $AUTODIR/results
 209         results_path = os.path.join(autodir, 'results')
 210         host.run('rm -rf %s/*' % utils.sh_escape(results_path),
 211                  ignore_status=True)
 212
 213         # Fetch the autotest client from the nearest repository
 214         if use_packaging:
 215             try:
 216                 self._install_using_packaging(host, autodir)
 217                 return
 218             except (error.PackageInstallError, error.AutoservRunError,
 219                     global_config.ConfigError), e:
 220                 logging.info("Could not install autotest using the packaging "
 221                              "system: %s. Trying other methods",  e)
 222
 223         # try to install from file or directory
 224         if self.source_material:
 225             c = global_config.global_config
 226             supports_autoserv_packaging = c.get_config_value(
 227                 "PACKAGES", "serve_packages_from_autoserv", type=bool)
 228             # Copy autotest recursively
 229             if supports_autoserv_packaging and use_autoserv:
 230                 self._install_using_send_file(host, autodir)
 231             else:
 232                 host.send_file(self.source_material, autodir, delete_dest=True)
 233             logging.info("Installation of autotest completed")
 234             self.installed = True
 235             return
 236
 237         # if that fails try to install using svn
 238         if utils.run('which svn').exit_status:
 239             raise error.AutoservError('svn not found on target machine: %s'
 240                                                                    % host.name)
 241         try:
 242             host.run('svn checkout %s %s' % (AUTOTEST_SVN, autodir))
 243         except error.AutoservRunError, e:
 244             host.run('svn checkout %s %s' % (AUTOTEST_HTTP, autodir))
 245         logging.info("Installation of autotest completed")
 246         self.installed = True
 247
 248
 249     def uninstall(self, host=None):
 250         """
 251         Uninstall (i.e. delete) autotest. Removes the autotest client install
 252         from the specified host.
 253
 254         @params host a Host instance from which the client will be removed
 255         """
 256         if not self.installed:
 257             return
 258         if not host:
 259             host = self.host
 260         autodir = host.get_autodir()
 261         if not autodir:
 262             return
 263
 264         # perform the actual uninstall
 265         host.run("rm -rf %s" % utils.sh_escape(autodir), ignore_status=True)
 266         host.set_autodir(None)
 267         self.installed = False
 268
 269
 270     def get(self, location = None):
 271         if not location:
 272             location = os.path.join(self.serverdir, '../client')
 273             location = os.path.abspath(location)
 274         # If there's stuff run on our client directory already, it
 275         # can cause problems. Try giving it a quick clean first.
 276         cwd = os.getcwd()
 277         os.chdir(location)
 278         try:
 279             utils.system('tools/make_clean', ignore_status=True)
 280         finally:
 281             os.chdir(cwd)
 282         super(BaseAutotest, self).get(location)
 283         self.got = True
 284
 285
 286     def run(self, control_file, results_dir='.', host=None, timeout=None,
 287             tag=None, parallel_flag=False, background=False,
 288             client_disconnect_timeout=1800):
 289         """
 290         Run an autotest job on the remote machine.
 291
 292         @param control_file: An open file-like-obj of the control file.
 293         @param results_dir: A str path where the results should be stored
 294                 on the local filesystem.
 295         @param host: A Host instance on which the control file should
 296                 be run.
 297         @param timeout: Maximum number of seconds to wait for the run or None.
 298         @param tag: Tag name for the client side instance of autotest.
 299         @param parallel_flag: Flag set when multiple jobs are run at the
 300                 same time.
 301         @param background: Indicates that the client should be launched as
 302                 a background job; the code calling run will be responsible
 303                 for monitoring the client and collecting the results.
 304         @param client_disconnect_timeout: Seconds to wait for the remote host
 305                 to come back after a reboot.  [default: 30 minutes]
 306
 307         @raises AutotestRunError: If there is a problem executing
 308                 the control file.
 309         """
 310         host = self._get_host_and_setup(host)
 311         results_dir = os.path.abspath(results_dir)
 312
 313         if tag:
 314             results_dir = os.path.join(results_dir, tag)
 315
 316         atrun = _Run(host, results_dir, tag, parallel_flag, background)
 317         self._do_run(control_file, results_dir, host, atrun, timeout,
 318                      client_disconnect_timeout)
 319
 320
 321     def _get_host_and_setup(self, host):
 322         if not host:
 323             host = self.host
 324         if not self.installed:
 325             self.install(host)
 326
 327         host.wait_up(timeout=30)
 328         return host
 329
 330
 331     def _do_run(self, control_file, results_dir, host, atrun, timeout,
 332                 client_disconnect_timeout):
 333         try:
 334             atrun.verify_machine()
 335         except:
 336             logging.error("Verify failed on %s. Reinstalling autotest",
 337                           host.hostname)
 338             self.install(host)
 339         atrun.verify_machine()
 340         debug = os.path.join(results_dir, 'debug')
 341         try:
 342             os.makedirs(debug)
 343         except Exception:
 344             pass
 345
 346         delete_file_list = [atrun.remote_control_file,
 347                             atrun.remote_control_file + '.state',
 348                             atrun.manual_control_file,
 349                             atrun.manual_control_file + '.state']
 350         cmd = ';'.join('rm -f ' + control for control in delete_file_list)
 351         host.run(cmd, ignore_status=True)
 352
 353         tmppath = utils.get(control_file)
 354
 355         # build up the initialization prologue for the control file
 356         prologue_lines = []
 357
 358         # Add the additional user arguments
 359         prologue_lines.append("args = %r\n" % self.job.args)
 360
 361         # If the packaging system is being used, add the repository list.
 362         repos = None
 363         try:
 364             c = global_config.global_config
 365             repos = c.get_config_value("PACKAGES", 'fetch_location', type=list)
 366             repos.reverse()  # high priority packages should be added last
 367             pkgmgr = packages.PackageManager('autotest', hostname=host.hostname,
 368                                              repo_urls=repos)
 369             prologue_lines.append('job.add_repository(%s)\n' % repos)
 370         except global_config.ConfigError, e:
 371             # If repos is defined packaging is enabled so log the error
 372             if repos:
 373                 logging.error(e)
 374
 375         # on full-size installs, turn on any profilers the server is using
 376         if not atrun.background:
 377             running_profilers = host.job.profilers.add_log.iteritems()
 378             for profiler, (args, dargs) in running_profilers:
 379                 call_args = [repr(profiler)]
 380                 call_args += [repr(arg) for arg in args]
 381                 call_args += ["%s=%r" % item for item in dargs.iteritems()]
 382                 prologue_lines.append("job.profilers.add(%s)\n"
 383                                       % ", ".join(call_args))
 384         cfile = "".join(prologue_lines)
 385
 386         cfile += open(tmppath).read()
 387         open(tmppath, "w").write(cfile)
 388
 389         # Create and copy state file to remote_control_file + '.state'
 390         state_file = host.job.preprocess_client_state()
 391         host.send_file(state_file, atrun.remote_control_file + '.init.state')
 392         os.remove(state_file)
 393
 394         # Copy control_file to remote_control_file on the host
 395         host.send_file(tmppath, atrun.remote_control_file)
 396         if os.path.abspath(tmppath) != os.path.abspath(control_file):
 397             os.remove(tmppath)
 398
 399         atrun.execute_control(
 400                 timeout=timeout,
 401                 client_disconnect_timeout=client_disconnect_timeout)
 402
 403
 404     def run_timed_test(self, test_name, results_dir='.', host=None,
 405                        timeout=None, *args, **dargs):
 406         """
 407         Assemble a tiny little control file to just run one test,
 408         and run it as an autotest client-side test
 409         """
 410         if not host:
 411             host = self.host
 412         if not self.installed:
 413             self.install(host)
 414         opts = ["%s=%s" % (o[0], repr(o[1])) for o in dargs.items()]
 415         cmd = ", ".join([repr(test_name)] + map(repr, args) + opts)
 416         control = "job.run_test(%s)\n" % cmd
 417         self.run(control, results_dir, host, timeout=timeout)
 418
 419
 420     def run_test(self, test_name, results_dir='.', host=None, *args, **dargs):
 421         self.run_timed_test(test_name, results_dir, host, timeout=None,
 422                             *args, **dargs)
 423
 424
 425 class _BaseRun(object):
 426     """
 427     Represents a run of autotest control file.  This class maintains
 428     all the state necessary as an autotest control file is executed.
 429
 430     It is not intended to be used directly, rather control files
 431     should be run using the run method in Autotest.
 432     """
 433     def __init__(self, host, results_dir, tag, parallel_flag, background):
 434         self.host = host
 435         self.results_dir = results_dir
 436         self.env = host.env
 437         self.tag = tag
 438         self.parallel_flag = parallel_flag
 439         self.background = background
 440         self.autodir = Autotest.get_installed_autodir(self.host)
 441         control = os.path.join(self.autodir, 'control')
 442         if tag:
 443             control += '.' + tag
 444         self.manual_control_file = control
 445         self.remote_control_file = control + '.autoserv'
 446         self.config_file = os.path.join(self.autodir, 'global_config.ini')
 447
 448
 449     def verify_machine(self):
 450         binary = os.path.join(self.autodir, 'bin/autotest')
 451         try:
 452             self.host.run('ls %s > /dev/null 2>&1' % binary)
 453         except:
 454             raise error.AutoservInstallError(
 455                 "Autotest does not appear to be installed")
 456
 457         if not self.parallel_flag:
 458             tmpdir = os.path.join(self.autodir, 'tmp')
 459             download = os.path.join(self.autodir, 'tests/download')
 460             self.host.run('umount %s' % tmpdir, ignore_status=True)
 461             self.host.run('umount %s' % download, ignore_status=True)
 462
 463
 464     def get_base_cmd_args(self, section):
 465         args = ['--verbose']
 466         if section > 0:
 467             args.append('-c')
 468         if self.tag:
 469             args.append('-t %s' % self.tag)
 470         if self.host.job.use_external_logging():
 471             args.append('-l')
 472         if self.host.hostname:
 473             args.append('--hostname=%s' % self.host.hostname)
 474         args.append('--user=%s' % self.host.job.user)
 475
 476         args.append(self.remote_control_file)
 477         return args
 478
 479
 480     def get_background_cmd(self, section):
 481         cmd = ['nohup', os.path.join(self.autodir, 'bin/autotest_client')]
 482         cmd += self.get_base_cmd_args(section)
 483         cmd += ['>/dev/null', '2>/dev/null', '&']
 484         return ' '.join(cmd)
 485
 486
 487     def get_daemon_cmd(self, section, monitor_dir):
 488         cmd = ['nohup', os.path.join(self.autodir, 'bin/autotestd'),
 489                monitor_dir, '-H autoserv']
 490         cmd += self.get_base_cmd_args(section)
 491         cmd += ['>/dev/null', '2>/dev/null', '&']
 492         return ' '.join(cmd)
 493
 494
 495     def get_monitor_cmd(self, monitor_dir, stdout_read, stderr_read):
 496         cmd = [os.path.join(self.autodir, 'bin', 'autotestd_monitor'),
 497                monitor_dir, str(stdout_read), str(stderr_read)]
 498         return ' '.join(cmd)
 499
 500
 501     def get_client_log(self):
 502         """Find what the "next" client.* prefix should be
 503
 504         @returns A string of the form client.INTEGER that should be prefixed
 505             to all client debug log files.
 506         """
 507         max_digit = -1
 508         debug_dir = os.path.join(self.results_dir, 'debug')
 509         client_logs = glob.glob(os.path.join(debug_dir, 'client.*.*'))
 510         for log in client_logs:
 511             _, number, _ = log.split('.', 2)
 512             if number.isdigit():
 513                 max_digit = max(max_digit, int(number))
 514         return 'client.%d' % (max_digit + 1)
 515
 516
 517     def copy_client_config_file(self, client_log_prefix=None):
 518         """
 519         Create and copy the client config file based on the server config.
 520
 521         @param client_log_prefix: Optional prefix to prepend to log files.
 522         """
 523         client_config_file = self._create_client_config_file(client_log_prefix)
 524         self.host.send_file(client_config_file, self.config_file)
 525         os.remove(client_config_file)
 526
 527
 528     def _create_client_config_file(self, client_log_prefix=None):
 529         """
 530         Create a temporary file with the [CLIENT] section configuration values
 531         taken from the server global_config.ini.
 532
 533         @param client_log_prefix: Optional prefix to prepend to log files.
 534
 535         @return: Path of the temporary file generated.
 536         """
 537         config = global_config.global_config.get_section_values('CLIENT')
 538         if client_log_prefix:
 539             config.set('CLIENT', 'default_logging_name', client_log_prefix)
 540         return self._create_aux_file(config.write)
 541
 542
 543     def _create_aux_file(self, func, *args):
 544         """
 545         Creates a temporary file and writes content to it according to a
 546         content creation function. The file object is appended to *args, which
 547         is then passed to the content creation function
 548
 549         @param func: Function that will be used to write content to the
 550                 temporary file.
 551         @param *args: List of parameters that func takes.
 552         @return: Path to the temporary file that was created.
 553         """
 554         fd, path = tempfile.mkstemp(dir=self.host.job.tmpdir)
 555         aux_file = os.fdopen(fd, "w")
 556         try:
 557             list_args = list(args)
 558             list_args.append(aux_file)
 559             func(*list_args)
 560         finally:
 561             aux_file.close()
 562         return path
 563
 564
 565     @staticmethod
 566     def is_client_job_finished(last_line):
 567         return bool(re.match(r'^END .*\t----\t----\t.*$', last_line))
 568
 569
 570     @staticmethod
 571     def is_client_job_rebooting(last_line):
 572         return bool(re.match(r'^\t*GOOD\t----\treboot\.start.*$', last_line))
 573
 574
 575     def log_unexpected_abort(self, stderr_redirector):
 576         stderr_redirector.flush_all_buffers()
 577         msg = "Autotest client terminated unexpectedly"
 578         self.host.job.record("END ABORT", None, None, msg)
 579
 580
 581     def _execute_in_background(self, section, timeout):
 582         full_cmd = self.get_background_cmd(section)
 583         devnull = open(os.devnull, "w")
 584
 585         self.copy_client_config_file(self.get_client_log())
 586
 587         self.host.job.push_execution_context(self.results_dir)
 588         try:
 589             result = self.host.run(full_cmd, ignore_status=True,
 590                                    timeout=timeout,
 591                                    stdout_tee=devnull,
 592                                    stderr_tee=devnull)
 593         finally:
 594             self.host.job.pop_execution_context()
 595
 596         return result
 597
 598
 599     @staticmethod
 600     def _strip_stderr_prologue(stderr):
 601         """Strips the 'standard' prologue that get pre-pended to every
 602         remote command and returns the text that was actually written to
 603         stderr by the remote command."""
 604         stderr_lines = stderr.split("\n")[1:]
 605         if not stderr_lines:
 606             return ""
 607         elif stderr_lines[0].startswith("NOTE: autotestd_monitor"):
 608             del stderr_lines[0]
 609         return "\n".join(stderr_lines)
 610
 611
 612     def _execute_daemon(self, section, timeout, stderr_redirector,
 613                         client_disconnect_timeout):
 614         monitor_dir = self.host.get_tmp_dir()
 615         daemon_cmd = self.get_daemon_cmd(section, monitor_dir)
 616
 617         # grab the location for the server-side client log file
 618         client_log_prefix = self.get_client_log()
 619         client_log_path = os.path.join(self.results_dir, 'debug',
 620                                        client_log_prefix + '.log')
 621         client_log = open(client_log_path, 'w', 0)
 622         self.copy_client_config_file(client_log_prefix)
 623
 624         stdout_read = stderr_read = 0
 625         self.host.job.push_execution_context(self.results_dir)
 626         try:
 627             self.host.run(daemon_cmd, ignore_status=True, timeout=timeout)
 628             disconnect_warnings = []
 629             while True:
 630                 monitor_cmd = self.get_monitor_cmd(monitor_dir, stdout_read,
 631                                                    stderr_read)
 632                 try:
 633                     result = self.host.run(monitor_cmd, ignore_status=True,
 634                                            timeout=timeout,
 635                                            stdout_tee=client_log,
 636                                            stderr_tee=stderr_redirector)
 637                 except error.AutoservRunError, e:
 638                     result = e.result_obj
 639                     result.exit_status = None
 640                     disconnect_warnings.append(e.description)
 641
 642                     stderr_redirector.log_warning(
 643                         "Autotest client was disconnected: %s" % e.description,
 644                         "NETWORK")
 645                 except error.AutoservSSHTimeout:
 646                     result = utils.CmdResult(monitor_cmd, "", "", None, 0)
 647                     stderr_redirector.log_warning(
 648                         "Attempt to connect to Autotest client timed out",
 649                         "NETWORK")
 650
 651                 stdout_read += len(result.stdout)
 652                 stderr_read += len(self._strip_stderr_prologue(result.stderr))
 653
 654                 if result.exit_status is not None:
 655                     return result
 656                 elif not self.host.wait_up(client_disconnect_timeout):
 657                     raise error.AutoservSSHTimeout(
 658                         "client was disconnected, reconnect timed out")
 659         finally:
 660             client_log.close()
 661             self.host.job.pop_execution_context()
 662
 663
 664     def execute_section(self, section, timeout, stderr_redirector,
 665                         client_disconnect_timeout):
 666         logging.info("Executing %s/bin/autotest %s/control phase %d",
 667                      self.autodir, self.autodir, section)
 668
 669         if self.background:
 670             result = self._execute_in_background(section, timeout)
 671         else:
 672             result = self._execute_daemon(section, timeout, stderr_redirector,
 673                                           client_disconnect_timeout)
 674
 675         last_line = stderr_redirector.last_line
 676
 677         # check if we failed hard enough to warrant an exception
 678         if result.exit_status == 1:
 679             err = error.AutotestRunError("client job was aborted")
 680         elif not self.background and not result.stderr:
 681             err = error.AutotestRunError(
 682                 "execute_section %s failed to return anything\n"
 683                 "stdout:%s\n" % (section, result.stdout))
 684         else:
 685             err = None
 686
 687         # log something if the client failed AND never finished logging
 688         if err and not self.is_client_job_finished(last_line):
 689             self.log_unexpected_abort(stderr_redirector)
 690
 691         if err:
 692             raise err
 693         else:
 694             return stderr_redirector.last_line
 695
 696
 697     def _wait_for_reboot(self, old_boot_id):
 698         logging.info("Client is rebooting")
 699         logging.info("Waiting for client to halt")
 700         if not self.host.wait_down(HALT_TIME, old_boot_id=old_boot_id):
 701             err = "%s failed to shutdown after %d"
 702             err %= (self.host.hostname, HALT_TIME)
 703             raise error.AutotestRunError(err)
 704         logging.info("Client down, waiting for restart")
 705         if not self.host.wait_up(BOOT_TIME):
 706             # since reboot failed
 707             # hardreset the machine once if possible
 708             # before failing this control file
 709             warning = "%s did not come back up, hard resetting"
 710             warning %= self.host.hostname
 711             logging.warning(warning)
 712             try:
 713                 self.host.hardreset(wait=False)
 714             except (AttributeError, error.AutoservUnsupportedError):
 715                 warning = "Hard reset unsupported on %s"
 716                 warning %= self.host.hostname
 717                 logging.warning(warning)
 718             raise error.AutotestRunError("%s failed to boot after %ds" %
 719                                          (self.host.hostname, BOOT_TIME))
 720         self.host.reboot_followup()
 721
 722
 723     def execute_control(self, timeout=None, client_disconnect_timeout=None):
 724         if not self.background:
 725             collector = log_collector(self.host, self.tag, self.results_dir)
 726             hostname = self.host.hostname
 727             remote_results = collector.client_results_dir
 728             local_results = collector.server_results_dir
 729             self.host.job.add_client_log(hostname, remote_results,
 730                                          local_results)
 731             job_record_context = self.host.job.get_record_context()
 732
 733         section = 0
 734         start_time = time.time()
 735
 736         logger = client_logger(self.host, self.tag, self.results_dir)
 737         try:
 738             while not timeout or time.time() < start_time + timeout:
 739                 if timeout:
 740                     section_timeout = start_time + timeout - time.time()
 741                 else:
 742                     section_timeout = None
 743                 boot_id = self.host.get_boot_id()
 744                 last = self.execute_section(section, section_timeout,
 745                                             logger, client_disconnect_timeout)
 746                 if self.background:
 747                     return
 748                 section += 1
 749                 if self.is_client_job_finished(last):
 750                     logging.info("Client complete")
 751                     return
 752                 elif self.is_client_job_rebooting(last):
 753                     try:
 754                         self._wait_for_reboot(boot_id)
 755                     except error.AutotestRunError, e:
 756                         self.host.job.record("ABORT", None, "reboot", str(e))
 757                         self.host.job.record("END ABORT", None, None, str(e))
 758                         raise
 759                     continue
 760
 761                 # if we reach here, something unexpected happened
 762                 self.log_unexpected_abort(logger)
 763
 764                 # give the client machine a chance to recover from a crash
 765                 self.host.wait_up(CRASH_RECOVERY_TIME)
 766                 msg = ("Aborting - unexpected final status message from "
 767                        "client on %s: %s\n") % (self.host.hostname, last)
 768                 raise error.AutotestRunError(msg)
 769         finally:
 770             logger.close()
 771             if not self.background:
 772                 collector.collect_client_job_results()
 773                 collector.remove_redundant_client_logs()
 774                 state_file = os.path.basename(self.remote_control_file
 775                                               + '.state')
 776                 state_path = os.path.join(self.results_dir, state_file)
 777                 self.host.job.postprocess_client_state(state_path)
 778                 self.host.job.remove_client_log(hostname, remote_results,
 779                                                 local_results)
 780                 job_record_context.restore()
 781
 782         # should only get here if we timed out
 783         assert timeout
 784         raise error.AutotestTimeoutError()
 785
 786
 787 class log_collector(object):
 788     def __init__(self, host, client_tag, results_dir):
 789         self.host = host
 790         if not client_tag:
 791             client_tag = "default"
 792         self.client_results_dir = os.path.join(host.get_autodir(), "results",
 793                                                client_tag)
 794         self.server_results_dir = results_dir
 795
 796
 797     def collect_client_job_results(self):
 798         """ A method that collects all the current results of a running
 799         client job into the results dir. By default does nothing as no
 800         client job is running, but when running a client job you can override
 801         this with something that will actually do something. """
 802
 803         # make an effort to wait for the machine to come up
 804         try:
 805             self.host.wait_up(timeout=30)
 806         except error.AutoservError:
 807             # don't worry about any errors, we'll try and
 808             # get the results anyway
 809             pass
 810
 811         # Copy all dirs in default to results_dir
 812         try:
 813             self.host.get_file(self.client_results_dir + '/',
 814                                self.server_results_dir, preserve_symlinks=True)
 815         except Exception:
 816             # well, don't stop running just because we couldn't get logs
 817             e_msg = "Unexpected error copying test result logs, continuing ..."
 818             logging.error(e_msg)
 819             traceback.print_exc(file=sys.stdout)
 820
 821
 822     def remove_redundant_client_logs(self):
 823         """Remove client.*.log files in favour of client.*.DEBUG files."""
 824         debug_dir = os.path.join(self.server_results_dir, 'debug')
 825         debug_files = [f for f in os.listdir(debug_dir)
 826                        if re.search(r'^client\.\d+\.DEBUG$', f)]
 827         for debug_file in debug_files:
 828             log_file = debug_file.replace('DEBUG', 'log')
 829             log_file = os.path.join(debug_dir, log_file)
 830             if os.path.exists(log_file):
 831                 os.remove(log_file)
 832
 833
 834 # a file-like object for catching stderr from an autotest client and
 835 # extracting status logs from it
 836 class client_logger(object):
 837     """Partial file object to write to both stdout and
 838     the status log file.  We only implement those methods
 839     utils.run() actually calls.
 840     """
 841     status_parser = re.compile(r"^AUTOTEST_STATUS:([^:]*):(.*)$")
 842     test_complete_parser = re.compile(r"^AUTOTEST_TEST_COMPLETE:(.*)$")
 843     fetch_package_parser = re.compile(
 844         r"^AUTOTEST_FETCH_PACKAGE:([^:]*):([^:]*):(.*)$")
 845     extract_indent = re.compile(r"^(\t*).*$")
 846     extract_timestamp = re.compile(r".*\ttimestamp=(\d+)\t.*$")
 847
 848     def __init__(self, host, tag, server_results_dir):
 849         self.host = host
 850         self.job = host.job
 851         self.log_collector = log_collector(host, tag, server_results_dir)
 852         self.leftover = ""
 853         self.last_line = ""
 854         self.logs = {}
 855
 856
 857     def _process_log_dict(self, log_dict):
 858         log_list = log_dict.pop("logs", [])
 859         for key in sorted(log_dict.iterkeys()):
 860             log_list += self._process_log_dict(log_dict.pop(key))
 861         return log_list
 862
 863
 864     def _process_logs(self):
 865         """Go through the accumulated logs in self.log and print them
 866         out to stdout and the status log. Note that this processes
 867         logs in an ordering where:
 868
 869         1) logs to different tags are never interleaved
 870         2) logs to x.y come before logs to x.y.z for all z
 871         3) logs to x.y come before x.z whenever y < z
 872
 873         Note that this will in general not be the same as the
 874         chronological ordering of the logs. However, if a chronological
 875         ordering is desired that one can be reconstructed from the
 876         status log by looking at timestamp lines."""
 877         log_list = self._process_log_dict(self.logs)
 878         for entry in log_list:
 879             self.job.record_entry(entry, log_in_subdir=False)
 880         if log_list:
 881             self.last_line = log_list[-1].render()
 882
 883
 884     def _process_quoted_line(self, tag, line):
 885         """Process a line quoted with an AUTOTEST_STATUS flag. If the
 886         tag is blank then we want to push out all the data we've been
 887         building up in self.logs, and then the newest line. If the
 888         tag is not blank, then push the line into the logs for handling
 889         later."""
 890         entry = base_job.status_log_entry.parse(line)
 891         if entry is None:
 892             return  # the line contains no status lines
 893         if tag == "":
 894             self._process_logs()
 895             self.job.record_entry(entry, log_in_subdir=False)
 896             self.last_line = line
 897         else:
 898             tag_parts = [int(x) for x in tag.split(".")]
 899             log_dict = self.logs
 900             for part in tag_parts:
 901                 log_dict = log_dict.setdefault(part, {})
 902             log_list = log_dict.setdefault("logs", [])
 903             log_list.append(entry)
 904
 905
 906     def _process_info_line(self, line):
 907         """Check if line is an INFO line, and if it is, interpret any control
 908         messages (e.g. enabling/disabling warnings) that it may contain."""
 909         match = re.search(r"^\t*INFO\t----\t----(.*)\t[^\t]*$", line)
 910         if not match:
 911             return   # not an INFO line
 912         for field in match.group(1).split('\t'):
 913             if field.startswith("warnings.enable="):
 914                 func = self.job.warning_manager.enable_warnings
 915             elif field.startswith("warnings.disable="):
 916                 func = self.job.warning_manager.disable_warnings
 917             else:
 918                 continue
 919             warning_type = field.split("=", 1)[1]
 920             func(warning_type)
 921
 922
 923     def _process_line(self, line):
 924         """Write out a line of data to the appropriate stream. Status
 925         lines sent by autotest will be prepended with
 926         "AUTOTEST_STATUS", and all other lines are ssh error
 927         messages."""
 928         status_match = self.status_parser.search(line)
 929         test_complete_match = self.test_complete_parser.search(line)
 930         fetch_package_match = self.fetch_package_parser.search(line)
 931         if status_match:
 932             tag, line = status_match.groups()
 933             self._process_info_line(line)
 934             self._process_quoted_line(tag, line)
 935         elif test_complete_match:
 936             self._process_logs()
 937             fifo_path, = test_complete_match.groups()
 938             try:
 939                 self.log_collector.collect_client_job_results()
 940                 self.host.run("echo A > %s" % fifo_path)
 941             except Exception:
 942                 msg = "Post-test log collection failed, continuing anyway"
 943                 logging.exception(msg)
 944         elif fetch_package_match:
 945             pkg_name, dest_path, fifo_path = fetch_package_match.groups()
 946             serve_packages = global_config.global_config.get_config_value(
 947                 "PACKAGES", "serve_packages_from_autoserv", type=bool)
 948             if serve_packages and pkg_name.endswith(".tar.bz2"):
 949                 try:
 950                     self._send_tarball(pkg_name, dest_path)
 951                 except Exception:
 952                     msg = "Package tarball creation failed, continuing anyway"
 953                     logging.exception(msg)
 954             try:
 955                 self.host.run("echo B > %s" % fifo_path)
 956             except Exception:
 957                 msg = "Package tarball installation failed, continuing anyway"
 958                 logging.exception(msg)
 959         else:
 960             logging.info(line)
 961
 962
 963     def _send_tarball(self, pkg_name, remote_dest):
 964         name, pkg_type = self.job.pkgmgr.parse_tarball_name(pkg_name)
 965         src_dirs = []
 966         if pkg_type == 'test':
 967             for test_dir in ['site_tests', 'tests']:
 968                 src_dir = os.path.join(self.job.clientdir, test_dir, name)
 969                 if os.path.exists(src_dir):
 970                     src_dirs += [src_dir]
 971                     if autoserv_prebuild:
 972                         prebuild.setup(self.job.clientdir, src_dir)
 973                     break
 974         elif pkg_type == 'profiler':
 975             src_dirs += [os.path.join(self.job.clientdir, 'profilers', name)]
 976             if autoserv_prebuild:
 977                 prebuild.setup(self.job.clientdir, src_dir)
 978         elif pkg_type == 'dep':
 979             src_dirs += [os.path.join(self.job.clientdir, 'deps', name)]
 980         elif pkg_type == 'client':
 981             return  # you must already have a client to hit this anyway
 982         else:
 983             return  # no other types are supported
 984
 985         # iterate over src_dirs until we find one that exists, then tar it
 986         for src_dir in src_dirs:
 987             if os.path.exists(src_dir):
 988                 try:
 989                     logging.info('Bundling %s into %s', src_dir, pkg_name)
 990                     temp_dir = autotemp.tempdir(unique_id='autoserv-packager',
 991                                                 dir=self.job.tmpdir)
 992                     tarball_path = self.job.pkgmgr.tar_package(
 993                         pkg_name, src_dir, temp_dir.name, " .")
 994                     self.host.send_file(tarball_path, remote_dest)
 995                 finally:
 996                     temp_dir.clean()
 997                 return
 998
 999
1000     def log_warning(self, msg, warning_type):
1001         """Injects a WARN message into the current status logging stream."""
1002         timestamp = int(time.time())
1003         if self.job.warning_manager.is_valid(timestamp, warning_type):
1004             self.job.record('WARN', None, None, {}, msg)
1005
1006
1007     def write(self, data):
1008         # now start processing the existing buffer and the new data
1009         data = self.leftover + data
1010         lines = data.split('\n')
1011         processed_lines = 0
1012         try:
1013             # process all the buffered data except the last line
1014             # ignore the last line since we may not have all of it yet
1015             for line in lines[:-1]:
1016                 self._process_line(line)
1017                 processed_lines += 1
1018         finally:
1019             # save any unprocessed lines for future processing
1020             self.leftover = '\n'.join(lines[processed_lines:])
1021
1022
1023     def flush(self):
1024         sys.stdout.flush()
1025
1026
1027     def flush_all_buffers(self):
1028         if self.leftover:
1029             self._process_line(self.leftover)
1030             self.leftover = ""
1031         self._process_logs()
1032         self.flush()
1033
1034
1035     def close(self):
1036         self.flush_all_buffers()
1037
1038
1039 SiteAutotest = client_utils.import_site_class(
1040     __file__, "autotest_lib.server.site_autotest", "SiteAutotest",
1041     BaseAutotest)
1042
1043
1044 _SiteRun = client_utils.import_site_class(
1045     __file__, "autotest_lib.server.site_autotest", "_SiteRun", _BaseRun)
1046
1047
1048 class Autotest(SiteAutotest):
1049     pass
1050
1051
1052 class _Run(_SiteRun):
1053     pass
1054
1055
1056 class AutotestHostMixin(object):
1057     """A generic mixin to add a run_test method to classes, which will allow
1058     you to run an autotest client test on a machine directly."""
1059
1060     # for testing purposes
1061     _Autotest = Autotest
1062
1063     def run_test(self, test_name, **dargs):
1064         """Run an autotest client test on the host.
1065
1066         @param test_name: The name of the client test.
1067         @param dargs: Keyword arguments to pass to the test.
1068
1069         @returns: True if the test passes, False otherwise."""
1070         at = self._Autotest()
1071         control_file = ('result = job.run_test(%s)\n'
1072                         'job.set_state("test_result", result)\n')
1073         test_args = [repr(test_name)]
1074         test_args += ['%s=%r' % (k, v) for k, v in dargs.iteritems()]
1075         control_file %= ', '.join(test_args)
1076         at.run(control_file, host=self)
1077         return at.job.get_state('test_result', default=False)