server/profilers.py

   1 import os, shutil, tempfile, logging
   2
   3 import common
   4 from autotest_lib.client.common_lib import utils, error, profiler_manager
   5 from autotest_lib.server import profiler, autotest, standalone_profiler, hosts
   6
   7
   8 PROFILER_TMPDIR = '/tmp/profilers'
   9
  10
  11 def get_profiler_results_dir(autodir):
  12     """
  13     Given the directory of the autotest client used to run a profiler,
  14     return the remote path where profiler results will be stored.
  15     """
  16     return os.path.join(autodir, 'results', 'default', 'profiler_sync',
  17                         'profiling')
  18
  19
  20 def get_profiler_log_path(autodir):
  21     """
  22     Given the directory of a profiler client, find the client log path.
  23     """
  24     return os.path.join(autodir, 'results', 'default', 'debug', 'client.DEBUG')
  25
  26
  27 class profilers(profiler_manager.profiler_manager):
  28     def __init__(self, job):
  29         super(profilers, self).__init__(job)
  30         self.add_log = {}
  31         self.start_delay = 0
  32         # maps hostname to (host object, autotest.Autotest object, Autotest
  33         # install dir), where the host object is the one created specifically
  34         # for profiling
  35         self.installed_hosts = {}
  36         self.current_test = None
  37
  38
  39     def set_start_delay(self, start_delay):
  40         self.start_delay = start_delay
  41
  42
  43     def load_profiler(self, profiler_name, args, dargs):
  44         newprofiler = profiler.profiler_proxy(profiler_name)
  45         newprofiler.initialize(*args, **dargs)
  46         newprofiler.setup(*args, **dargs) # lazy setup is done client-side
  47         return newprofiler
  48
  49
  50     def add(self, profiler, *args, **dargs):
  51         super(profilers, self).add(profiler, *args, **dargs)
  52         self.add_log[profiler] = (args, dargs)
  53
  54
  55     def delete(self, profiler):
  56         super(profilers, self).delete(profiler)
  57         if profiler in self.add_log:
  58             del self.add_log[profiler]
  59
  60
  61     def _install_clients(self):
  62         """
  63         Install autotest on any current job hosts.
  64         """
  65         in_use_hosts = set()
  66         # find hosts in use but not used by us
  67         for host in self.job.hosts:
  68             autodir = host.get_autodir()
  69             if not (autodir and autodir.startswith(PROFILER_TMPDIR)):
  70                 in_use_hosts.add(host.hostname)
  71         logging.debug('Hosts currently in use: %s', in_use_hosts)
  72
  73         # determine what valid host objects we already have installed
  74         profiler_hosts = set()
  75         for host, at, profiler_dir in self.installed_hosts.values():
  76             if host.path_exists(profiler_dir):
  77                 profiler_hosts.add(host.hostname)
  78             else:
  79                 # the profiler was wiped out somehow, drop this install
  80                 logging.warning('The profiler client on %s at %s was deleted',
  81                                 host.hostname, profiler_dir)
  82                 host.close()
  83                 del self.installed_hosts[host.hostname]
  84         logging.debug('Hosts with profiler clients already installed: %s',
  85                       profiler_hosts)
  86
  87         # install autotest on any new hosts in use
  88         for hostname in in_use_hosts - profiler_hosts:
  89             host = hosts.create_host(hostname, auto_monitor=False)
  90             tmp_dir = host.get_tmp_dir(parent=PROFILER_TMPDIR)
  91             at = autotest.Autotest(host)
  92             at.install_no_autoserv(autodir=tmp_dir)
  93             self.installed_hosts[host.hostname] = (host, at, tmp_dir)
  94
  95         # drop any installs from hosts no longer in job.hosts
  96         hostnames_to_drop = profiler_hosts - in_use_hosts
  97         hosts_to_drop = [self.installed_hosts[hostname][0]
  98                          for hostname in hostnames_to_drop]
  99         for host in hosts_to_drop:
 100             host.close()
 101             del self.installed_hosts[host.hostname]
 102
 103
 104     def _get_hosts(self, host=None):
 105         """
 106         Returns a list of (Host, Autotest, install directory) tuples for hosts
 107         currently supported by this profiler. The returned Host object is always
 108         the one created by this profiler, regardless of what's passed in. If
 109         'host' is not None, all entries not matching that host object are
 110         filtered out of the list.
 111         """
 112         if host is None:
 113             return self.installed_hosts.values()
 114         if host.hostname in self.installed_hosts:
 115             return [self.installed_hosts[host.hostname]]
 116         return []
 117
 118
 119     def _get_local_profilers_dir(self, test, hostname):
 120         in_machine_dir = (
 121                 os.path.basename(test.job.resultdir) in test.job.machines)
 122         if len(test.job.machines) > 1 and not in_machine_dir:
 123             local_dir = os.path.join(test.profdir, hostname)
 124             if not os.path.exists(local_dir):
 125                 os.makedirs(local_dir)
 126         else:
 127             local_dir = test.profdir
 128
 129         return local_dir
 130
 131
 132     def _get_failure_logs(self, autodir, test, host):
 133         """
 134         Collect the client logs from a profiler run and put them in a
 135         file named failure-*.log.
 136         """
 137         try:
 138             fd, path = tempfile.mkstemp(suffix='.log', prefix='failure-',
 139                     dir=self._get_local_profilers_dir(test, host.hostname))
 140             os.close(fd)
 141             host.get_file(get_profiler_log_path(autodir), path)
 142             # try to collect any partial profiler logs
 143             self._get_profiler_logs(autodir, test, host)
 144         except (error.AutotestError, error.AutoservError):
 145             logging.exception('Profiler failure log collection failed')
 146             # swallow the exception so that we don't override an existing
 147             # exception being thrown
 148
 149
 150     def _get_all_failure_logs(self, test, hosts):
 151         for host, at, autodir in hosts:
 152             self._get_failure_logs(autodir, test, host)
 153
 154
 155     def _get_profiler_logs(self, autodir, test, host):
 156         results_dir = get_profiler_results_dir(autodir)
 157         local_dir = self._get_local_profilers_dir(test, host.hostname)
 158
 159         self.job.remove_client_log(host.hostname, results_dir, local_dir)
 160
 161         tempdir = tempfile.mkdtemp(dir=self.job.tmpdir)
 162         try:
 163             host.get_file(results_dir + '/', tempdir)
 164         except error.AutoservRunError:
 165             pass # no files to pull back, nothing we can do
 166         utils.merge_trees(tempdir, local_dir)
 167         shutil.rmtree(tempdir, ignore_errors=True)
 168
 169
 170     def _run_clients(self, test, hosts):
 171         """
 172         We initialize the profilers just before start because only then we
 173         know all the hosts involved.
 174         """
 175
 176         hostnames = [host_info[0].hostname for host_info in hosts]
 177         profilers_args = [(p.name, p.args, p.dargs)
 178                           for p in self.list]
 179
 180         for host, at, autodir in hosts:
 181             control_script = standalone_profiler.generate_test(hostnames,
 182                                                                host.hostname,
 183                                                                profilers_args,
 184                                                                180, None)
 185             try:
 186                 at.run(control_script, background=True)
 187             except Exception:
 188                 self._get_failure_logs(autodir, test, host)
 189                 raise
 190
 191             remote_results_dir = get_profiler_results_dir(autodir)
 192             local_results_dir = self._get_local_profilers_dir(test,
 193                                                               host.hostname)
 194             self.job.add_client_log(host.hostname, remote_results_dir,
 195                                     local_results_dir)
 196
 197         try:
 198             # wait for the profilers to be added
 199             standalone_profiler.wait_for_profilers(hostnames)
 200         except Exception:
 201             self._get_all_failure_logs(test, hosts)
 202             raise
 203
 204
 205     def before_start(self, test, host=None):
 206         # create host objects and install the needed clients
 207         # so later in start() we don't spend too much time
 208         self._install_clients()
 209         self._run_clients(test, self._get_hosts(host))
 210
 211
 212     def start(self, test, host=None):
 213         hosts = self._get_hosts(host)
 214
 215         # wait for the profilers to start
 216         hostnames = [host_info[0].hostname for host_info in hosts]
 217         try:
 218             standalone_profiler.start_profilers(hostnames)
 219         except Exception:
 220             self._get_all_failure_logs(test, hosts)
 221             raise
 222
 223         self.current_test = test
 224
 225
 226     def stop(self, test):
 227         assert self.current_test == test
 228
 229         hosts = self._get_hosts()
 230         # wait for the profilers to stop
 231         hostnames = [host_info[0].hostname for host_info in hosts]
 232         try:
 233             standalone_profiler.stop_profilers(hostnames)
 234         except Exception:
 235             self._get_all_failure_logs(test, hosts)
 236             raise
 237
 238
 239     def report(self, test, host=None):
 240         assert self.current_test == test
 241
 242         hosts = self._get_hosts(host)
 243         # when running on specific hosts we cannot wait for the other
 244         # hosts to sync with us
 245         if not host:
 246             hostnames = [host_info[0].hostname for host_info in hosts]
 247             try:
 248                 standalone_profiler.finish_profilers(hostnames)
 249             except Exception:
 250                 self._get_all_failure_logs(test, hosts)
 251                 raise
 252
 253         # pull back all the results
 254         for host, at, autodir in hosts:
 255             self._get_profiler_logs(autodir, test, host)
 256
 257
 258     def handle_reboot(self, host):
 259         if self.current_test:
 260             test = self.current_test
 261             for profiler in self.list:
 262                 if not profiler.supports_reboot:
 263                     msg = 'profiler %s does not support rebooting during tests'
 264                     msg %= profiler.name
 265                     self.job.record('WARN', os.path.basename(test.outputdir),
 266                                     None, msg)
 267
 268             self.report(test, host)
 269             self.before_start(test, host)
 270             self.start(test, host)