tko/models.py

   1 import os
   2
   3 from autotest_lib.client.common_lib import utils
   4 from autotest_lib.tko import utils as tko_utils
   5
   6
   7 class job(object):
   8     def __init__(self, dir, user, label, machine, queued_time, started_time,
   9                  finished_time, machine_owner, machine_group, aborted_by,
  10                  aborted_on, keyval_dict):
  11         self.dir = dir
  12         self.tests = []
  13         self.user = user
  14         self.label = label
  15         self.machine = machine
  16         self.queued_time = queued_time
  17         self.started_time = started_time
  18         self.finished_time = finished_time
  19         self.machine_owner = machine_owner
  20         self.machine_group = machine_group
  21         self.aborted_by = aborted_by
  22         self.aborted_on = aborted_on
  23         self.keyval_dict = keyval_dict
  24
  25
  26     @staticmethod
  27     def read_keyval(dir):
  28         dir = os.path.normpath(dir)
  29         top_dir = tko_utils.find_toplevel_job_dir(dir)
  30         if not top_dir:
  31             top_dir = dir
  32         assert(dir.startswith(top_dir))
  33
  34         # pull in and merge all the keyval files, with higher-level
  35         # overriding values in the lower-level ones
  36         keyval = {}
  37         while True:
  38             try:
  39                 upper_keyval = utils.read_keyval(dir)
  40                 # HACK: exclude hostname from the override - this is a special
  41                 # case where we want lower to override higher
  42                 if "hostname" in upper_keyval and "hostname" in keyval:
  43                     del upper_keyval["hostname"]
  44                 keyval.update(upper_keyval)
  45             except IOError:
  46                 pass  # if the keyval can't be read just move on to the next
  47             if dir == top_dir:
  48                 break
  49             else:
  50                 assert(dir != "/")
  51                 dir = os.path.dirname(dir)
  52         return keyval
  53
  54
  55
  56 class kernel(object):
  57     def __init__(self, base, patches, kernel_hash):
  58         self.base = base
  59         self.patches = patches
  60         self.kernel_hash = kernel_hash
  61
  62
  63     @staticmethod
  64     def compute_hash(base, hashes):
  65         key_string = ','.join([base] + hashes)
  66         return utils.hash('md5', key_string).hexdigest()
  67
  68
  69 class test(object):
  70     def __init__(self, subdir, testname, status, reason, test_kernel,
  71                  machine, started_time, finished_time, iterations,
  72                  attributes, labels):
  73         self.subdir = subdir
  74         self.testname = testname
  75         self.status = status
  76         self.reason = reason
  77         self.kernel = test_kernel
  78         self.machine = machine
  79         self.started_time = started_time
  80         self.finished_time = finished_time
  81         self.iterations = iterations
  82         self.attributes = attributes
  83         self.labels = labels
  84
  85
  86     @staticmethod
  87     def load_iterations(keyval_path):
  88         """Abstract method to load a list of iterations from a keyval
  89         file."""
  90         raise NotImplementedError
  91
  92
  93     @classmethod
  94     def parse_test(cls, job, subdir, testname, status, reason, test_kernel,
  95                    started_time, finished_time, existing_instance=None):
  96         """Given a job and the basic metadata about the test that
  97         can be extracted from the status logs, parse the test
  98         keyval files and use it to construct a complete test
  99         instance."""
 100         tko_utils.dprint("parsing test %s %s" % (subdir, testname))
 101
 102         if subdir:
 103             # grab iterations from the results keyval
 104             iteration_keyval = os.path.join(job.dir, subdir,
 105                                             "results", "keyval")
 106             iterations = cls.load_iterations(iteration_keyval)
 107
 108             # grab test attributes from the subdir keyval
 109             test_keyval = os.path.join(job.dir, subdir, "keyval")
 110             attributes = test.load_attributes(test_keyval)
 111         else:
 112             iterations = []
 113             attributes = {}
 114
 115         # grab test+host attributes from the host keyval
 116         host_keyval = cls.parse_host_keyval(job.dir, job.machine)
 117         attributes.update(dict(("host-%s" % k, v)
 118                                for k, v in host_keyval.iteritems()))
 119
 120         if existing_instance:
 121             def constructor(*args, **dargs):
 122                 existing_instance.__init__(*args, **dargs)
 123                 return existing_instance
 124         else:
 125             constructor = cls
 126         return constructor(subdir, testname, status, reason, test_kernel,
 127                            job.machine, started_time, finished_time,
 128                            iterations, attributes, [])
 129
 130
 131     @classmethod
 132     def parse_partial_test(cls, job, subdir, testname, reason, test_kernel,
 133                            started_time):
 134         """Given a job and the basic metadata available when a test is
 135         started, create a test instance representing the partial result.
 136         Assume that since the test is not complete there are no results files
 137         actually available for parsing."""
 138         tko_utils.dprint("parsing partial test %s %s" % (subdir, testname))
 139
 140         return cls(subdir, testname, "RUNNING", reason, test_kernel,
 141                    job.machine, started_time, None, [], {}, [])
 142
 143
 144     @staticmethod
 145     def load_attributes(keyval_path):
 146         """Load the test attributes into a dictionary from a test
 147         keyval path. Does not assume that the path actually exists."""
 148         if not os.path.exists(keyval_path):
 149             return {}
 150         return utils.read_keyval(keyval_path)
 151
 152
 153     @staticmethod
 154     def parse_host_keyval(job_dir, hostname):
 155         # the "real" job dir may be higher up in the directory tree
 156         job_dir = tko_utils.find_toplevel_job_dir(job_dir)
 157         if not job_dir:
 158             return {} # we can't find a top-level job dir with host keyvals
 159
 160         # the keyval is <job_dir>/host_keyvals/<hostname> if it exists
 161         keyval_path = os.path.join(job_dir, "host_keyvals", hostname)
 162         if os.path.isfile(keyval_path):
 163             return utils.read_keyval(keyval_path)
 164         else:
 165             return {}
 166
 167
 168 class patch(object):
 169     def __init__(self, spec, reference, hash):
 170         self.spec = spec
 171         self.reference = reference
 172         self.hash = hash
 173
 174
 175 class iteration(object):
 176     def __init__(self, index, attr_keyval, perf_keyval):
 177         self.index = index
 178         self.attr_keyval = attr_keyval
 179         self.perf_keyval = perf_keyval
 180
 181
 182
 183     @staticmethod
 184     def parse_line_into_dicts(line, attr_dict, perf_dict):
 185         """Abstract method to parse a keyval line and insert it into
 186         the appropriate dictionary.
 187                 attr_dict: generic iteration attributes
 188                 perf_dict: iteration performance results
 189         """
 190         raise NotImplementedError
 191
 192
 193     @classmethod
 194     def load_from_keyval(cls, keyval_path):
 195         """Load a list of iterations from an iteration keyval file.
 196         Keyval data from separate iterations is separated by blank
 197         lines. Makes use of the parse_line_into_dicts method to
 198         actually parse the individual lines."""
 199         if not os.path.exists(keyval_path):
 200             return []
 201
 202         iterations = []
 203         index = 1
 204         attr, perf = {}, {}
 205         for line in file(keyval_path):
 206             line = line.strip()
 207             if line:
 208                 cls.parse_line_into_dicts(line, attr, perf)
 209             else:
 210                 iterations.append(cls(index, attr, perf))
 211                 index += 1
 212                 attr, perf = {}, {}
 213         if attr or perf:
 214             iterations.append(cls(index, attr, perf))
 215         return iterations