tko/parsers/version_0.py

   1 import re, os
   2
   3 from autotest_lib.client.common_lib import utils as common_utils
   4 from autotest_lib.tko import utils as tko_utils, models, status_lib
   5 from autotest_lib.tko.parsers import base
   6
   7
   8 class NoHostnameError(Exception):
   9     pass
  10
  11
  12 class job(models.job):
  13     def __init__(self, dir):
  14         job_dict = job.load_from_dir(dir)
  15         super(job, self).__init__(dir, **job_dict)
  16
  17
  18     @classmethod
  19     def load_from_dir(cls, dir):
  20         keyval = cls.read_keyval(dir)
  21         tko_utils.dprint(str(keyval))
  22
  23         user = keyval.get("user", None)
  24         label = keyval.get("label", None)
  25         queued_time = tko_utils.get_timestamp(keyval, "job_queued")
  26         started_time = tko_utils.get_timestamp(keyval, "job_started")
  27         finished_time = tko_utils.get_timestamp(keyval, "job_finished")
  28         machine = cls.determine_hostname(keyval, dir)
  29         machine_group = cls.determine_machine_group(machine, dir)
  30         machine_owner = keyval.get("owner", None)
  31
  32         aborted_by = keyval.get("aborted_by", None)
  33         aborted_at = tko_utils.get_timestamp(keyval, "aborted_on")
  34
  35         return {"user": user, "label": label, "machine": machine,
  36                 "queued_time": queued_time, "started_time": started_time,
  37                 "finished_time": finished_time, "machine_owner": machine_owner,
  38                 "machine_group": machine_group, "aborted_by": aborted_by,
  39                 "aborted_on": aborted_at, "keyval_dict": keyval}
  40
  41
  42     @classmethod
  43     def determine_hostname(cls, keyval, job_dir):
  44         host_group_name = keyval.get("host_group_name", None)
  45         machine = keyval.get("hostname", "")
  46         is_multimachine = "," in machine
  47
  48         # determine what hostname to use
  49         if host_group_name:
  50             if is_multimachine or not machine:
  51                 tko_utils.dprint("Using host_group_name %r instead of "
  52                                  "machine name." % host_group_name)
  53                 machine = host_group_name
  54         elif is_multimachine:
  55             try:
  56                 machine = job.find_hostname(job_dir) # find a unique hostname
  57             except NoHostnameError:
  58                 pass  # just use the comma-separated name
  59
  60         tko_utils.dprint("MACHINE NAME: %s" % machine)
  61         return machine
  62
  63
  64     @classmethod
  65     def determine_machine_group(cls, hostname, job_dir):
  66         machine_groups = set()
  67         for individual_hostname in hostname.split(","):
  68             host_keyval = models.test.parse_host_keyval(job_dir,
  69                                                         individual_hostname)
  70             if not host_keyval:
  71                 tko_utils.dprint('Unable to parse host keyval for %s'
  72                                  % individual_hostname)
  73             elif "platform" in host_keyval:
  74                 machine_groups.add(host_keyval["platform"])
  75         machine_group = ",".join(sorted(machine_groups))
  76         tko_utils.dprint("MACHINE GROUP: %s" % machine_group)
  77         return machine_group
  78
  79
  80     @staticmethod
  81     def find_hostname(path):
  82         hostname = os.path.join(path, "sysinfo", "hostname")
  83         try:
  84             machine = open(hostname).readline().rstrip()
  85             return machine
  86         except Exception:
  87             tko_utils.dprint("Could not read a hostname from "
  88                              "sysinfo/hostname")
  89
  90         uname = os.path.join(path, "sysinfo", "uname_-a")
  91         try:
  92             machine = open(uname).readline().split()[1]
  93             return machine
  94         except Exception:
  95             tko_utils.dprint("Could not read a hostname from "
  96                              "sysinfo/uname_-a")
  97
  98         raise NoHostnameError("Unable to find a machine name")
  99
 100
 101 class kernel(models.kernel):
 102     def __init__(self, job, verify_ident=None):
 103         kernel_dict = kernel.load_from_dir(job.dir, verify_ident)
 104         super(kernel, self).__init__(**kernel_dict)
 105
 106
 107     @staticmethod
 108     def load_from_dir(dir, verify_ident=None):
 109         # try and load the booted kernel version
 110         attributes = False
 111         i = 1
 112         build_dir = os.path.join(dir, "build")
 113         while True:
 114             if not os.path.exists(build_dir):
 115                 break
 116             build_log = os.path.join(build_dir, "debug", "build_log")
 117             attributes = kernel.load_from_build_log(build_log)
 118             if attributes:
 119                 break
 120             i += 1
 121             build_dir = os.path.join(dir, "build.%d" % (i))
 122
 123         if not attributes:
 124             if verify_ident:
 125                 base = verify_ident
 126             else:
 127                 base = kernel.load_from_sysinfo(dir)
 128             patches = []
 129             hashes = []
 130         else:
 131             base, patches, hashes = attributes
 132         tko_utils.dprint("kernel.__init__() found kernel version %s"
 133                          % base)
 134
 135         # compute the kernel hash
 136         if base == "UNKNOWN":
 137             kernel_hash = "UNKNOWN"
 138         else:
 139             kernel_hash = kernel.compute_hash(base, hashes)
 140
 141         return {"base": base, "patches": patches,
 142                 "kernel_hash": kernel_hash}
 143
 144
 145     @staticmethod
 146     def load_from_sysinfo(path):
 147         for subdir in ("reboot1", ""):
 148             uname_path = os.path.join(path, "sysinfo", subdir,
 149                                       "uname_-a")
 150             if not os.path.exists(uname_path):
 151                 continue
 152             uname = open(uname_path).readline().split()
 153             return re.sub("-autotest$", "", uname[2])
 154         return "UNKNOWN"
 155
 156
 157     @staticmethod
 158     def load_from_build_log(path):
 159         if not os.path.exists(path):
 160             return None
 161
 162         base, patches, hashes = "UNKNOWN", [], []
 163         for line in file(path):
 164             head, rest = line.split(": ", 1)
 165             rest = rest.split()
 166             if head == "BASE":
 167                 base = rest[0]
 168             elif head == "PATCH":
 169                 patches.append(patch(*rest))
 170                 hashes.append(rest[2])
 171         return base, patches, hashes
 172
 173
 174 class test(models.test):
 175     def __init__(self, subdir, testname, status, reason, test_kernel,
 176                  machine, started_time, finished_time, iterations,
 177                  attributes, labels):
 178         # for backwards compatibility with the original parser
 179         # implementation, if there is no test version we need a NULL
 180         # value to be used; also, if there is a version it should
 181         # be terminated by a newline
 182         if "version" in attributes:
 183             attributes["version"] = str(attributes["version"])
 184         else:
 185             attributes["version"] = None
 186
 187         super(test, self).__init__(subdir, testname, status, reason,
 188                                    test_kernel, machine, started_time,
 189                                    finished_time, iterations,
 190                                    attributes, labels)
 191
 192
 193     @staticmethod
 194     def load_iterations(keyval_path):
 195         return iteration.load_from_keyval(keyval_path)
 196
 197
 198 class patch(models.patch):
 199     def __init__(self, spec, reference, hash):
 200         tko_utils.dprint("PATCH::%s %s %s" % (spec, reference, hash))
 201         super(patch, self).__init__(spec, reference, hash)
 202         self.spec = spec
 203         self.reference = reference
 204         self.hash = hash
 205
 206
 207 class iteration(models.iteration):
 208     @staticmethod
 209     def parse_line_into_dicts(line, attr_dict, perf_dict):
 210         key, value = line.split("=", 1)
 211         perf_dict[key] = value
 212
 213
 214 class status_line(object):
 215     def __init__(self, indent, status, subdir, testname, reason,
 216                  optional_fields):
 217         # pull out the type & status of the line
 218         if status == "START":
 219             self.type = "START"
 220             self.status = None
 221         elif status.startswith("END "):
 222             self.type = "END"
 223             self.status = status[4:]
 224         else:
 225             self.type = "STATUS"
 226             self.status = status
 227         assert (self.status is None or
 228                 self.status in status_lib.statuses)
 229
 230         # save all the other parameters
 231         self.indent = indent
 232         self.subdir = self.parse_name(subdir)
 233         self.testname = self.parse_name(testname)
 234         self.reason = reason
 235         self.optional_fields = optional_fields
 236
 237
 238     @staticmethod
 239     def parse_name(name):
 240         if name == "----":
 241             return None
 242         return name
 243
 244
 245     @staticmethod
 246     def is_status_line(line):
 247         return re.search(r"^\t*(\S[^\t]*\t){3}", line) is not None
 248
 249
 250     @classmethod
 251     def parse_line(cls, line):
 252         if not status_line.is_status_line(line):
 253             return None
 254         match = re.search(r"^(\t*)(.*)$", line, flags=re.DOTALL)
 255         if not match:
 256             # A more useful error message than:
 257             #  AttributeError: 'NoneType' object has no attribute 'groups'
 258             # to help us debug WTF happens on occasion here.
 259             raise RuntimeError("line %r could not be parsed." % line)
 260         indent, line = match.groups()
 261         indent = len(indent)
 262
 263         # split the line into the fixed and optional fields
 264         parts = line.rstrip("\n").split("\t")
 265
 266         part_index = 3
 267         status, subdir, testname = parts[0:part_index]
 268
 269         # all optional parts should be of the form "key=value". once we've found
 270         # a non-matching part, treat it and the rest of the parts as the reason.
 271         optional_fields = {}
 272         while part_index < len(parts):
 273             kv = parts[part_index].split('=', 1)
 274             if len(kv) < 2:
 275                 break
 276
 277             optional_fields[kv[0]] = kv[1]
 278             part_index += 1
 279
 280         reason = "\t".join(parts[part_index:])
 281
 282         # build up a new status_line and return it
 283         return cls(indent, status, subdir, testname, reason,
 284                    optional_fields)
 285
 286
 287 class parser(base.parser):
 288     @staticmethod
 289     def make_job(dir):
 290         return job(dir)
 291
 292
 293     def state_iterator(self, buffer):
 294         new_tests = []
 295         boot_count = 0
 296         group_subdir = None
 297         sought_level = 0
 298         stack = status_lib.status_stack()
 299         current_kernel = kernel(self.job)
 300         boot_in_progress = False
 301         alert_pending = None
 302         started_time = None
 303
 304         while not self.finished or buffer.size():
 305             # stop processing once the buffer is empty
 306             if buffer.size() == 0:
 307                 yield new_tests
 308                 new_tests = []
 309                 continue
 310
 311             # parse the next line
 312             line = buffer.get()
 313             tko_utils.dprint('\nSTATUS: ' + line.strip())
 314             line = status_line.parse_line(line)
 315             if line is None:
 316                 tko_utils.dprint('non-status line, ignoring')
 317                 continue # ignore non-status lines
 318
 319             # have we hit the job start line?
 320             if (line.type == "START" and not line.subdir and
 321                 not line.testname):
 322                 sought_level = 1
 323                 tko_utils.dprint("found job level start "
 324                                  "marker, looking for level "
 325                                  "1 groups now")
 326                 continue
 327
 328             # have we hit the job end line?
 329             if (line.type == "END" and not line.subdir and
 330                 not line.testname):
 331                 tko_utils.dprint("found job level end "
 332                                  "marker, looking for level "
 333                                  "0 lines now")
 334                 sought_level = 0
 335
 336             # START line, just push another layer on to the stack
 337             # and grab the start time if this is at the job level
 338             # we're currently seeking
 339             if line.type == "START":
 340                 group_subdir = None
 341                 stack.start()
 342                 if line.indent == sought_level:
 343                     started_time = \
 344                                  tko_utils.get_timestamp(
 345                         line.optional_fields, "timestamp")
 346                 tko_utils.dprint("start line, ignoring")
 347                 continue
 348             # otherwise, update the status on the stack
 349             else:
 350                 tko_utils.dprint("GROPE_STATUS: %s" %
 351                                  [stack.current_status(),
 352                                   line.status, line.subdir,
 353                                   line.testname, line.reason])
 354                 stack.update(line.status)
 355
 356             if line.status == "ALERT":
 357                 tko_utils.dprint("job level alert, recording")
 358                 alert_pending = line.reason
 359                 continue
 360
 361             # ignore Autotest.install => GOOD lines
 362             if (line.testname == "Autotest.install" and
 363                 line.status == "GOOD"):
 364                 tko_utils.dprint("Successful Autotest "
 365                                  "install, ignoring")
 366                 continue
 367
 368             # ignore END lines for a reboot group
 369             if (line.testname == "reboot" and line.type == "END"):
 370                 tko_utils.dprint("reboot group, ignoring")
 371                 continue
 372
 373             # convert job-level ABORTs into a 'CLIENT_JOB' test, and
 374             # ignore other job-level events
 375             if line.testname is None:
 376                 if (line.status == "ABORT" and
 377                     line.type != "END"):
 378                     line.testname = "CLIENT_JOB"
 379                 else:
 380                     tko_utils.dprint("job level event, "
 381                                     "ignoring")
 382                     continue
 383
 384             # use the group subdir for END lines
 385             if line.type == "END":
 386                 line.subdir = group_subdir
 387
 388             # are we inside a block group?
 389             if (line.indent != sought_level and
 390                 line.status != "ABORT" and
 391                 not line.testname.startswith('reboot.')):
 392                 if line.subdir:
 393                     tko_utils.dprint("set group_subdir: "
 394                                      + line.subdir)
 395                     group_subdir = line.subdir
 396                 tko_utils.dprint("ignoring incorrect indent "
 397                                  "level %d != %d," %
 398                                  (line.indent, sought_level))
 399                 continue
 400
 401             # use the subdir as the testname, except for
 402             # boot.* and kernel.* tests
 403             if (line.testname is None or
 404                 not re.search(r"^(boot(\.\d+)?$|kernel\.)",
 405                               line.testname)):
 406                 if line.subdir and '.' in line.subdir:
 407                     line.testname = line.subdir
 408
 409             # has a reboot started?
 410             if line.testname == "reboot.start":
 411                 started_time = tko_utils.get_timestamp(
 412                     line.optional_fields, "timestamp")
 413                 tko_utils.dprint("reboot start event, "
 414                                  "ignoring")
 415                 boot_in_progress = True
 416                 continue
 417
 418             # has a reboot finished?
 419             if line.testname == "reboot.verify":
 420                 line.testname = "boot.%d" % boot_count
 421                 tko_utils.dprint("reboot verified")
 422                 boot_in_progress = False
 423                 verify_ident = line.reason.strip()
 424                 current_kernel = kernel(self.job, verify_ident)
 425                 boot_count += 1
 426
 427             if alert_pending:
 428                 line.status = "ALERT"
 429                 line.reason = alert_pending
 430                 alert_pending = None
 431
 432             # create the actual test object
 433             finished_time = tko_utils.get_timestamp(
 434                 line.optional_fields, "timestamp")
 435             final_status = stack.end()
 436             tko_utils.dprint("Adding: "
 437                              "%s\nSubdir:%s\nTestname:%s\n%s" %
 438                              (final_status, line.subdir,
 439                               line.testname, line.reason))
 440             new_test = test.parse_test(self.job, line.subdir,
 441                                        line.testname,
 442                                        final_status, line.reason,
 443                                        current_kernel,
 444                                        started_time,
 445                                        finished_time)
 446             started_time = None
 447             new_tests.append(new_test)
 448
 449         # the job is finished, but we never came back from reboot
 450         if boot_in_progress:
 451             testname = "boot.%d" % boot_count
 452             reason = "machine did not return from reboot"
 453             tko_utils.dprint(("Adding: ABORT\nSubdir:----\n"
 454                               "Testname:%s\n%s")
 455                              % (testname, reason))
 456             new_test = test.parse_test(self.job, None, testname,
 457                                        "ABORT", reason,
 458                                        current_kernel, None, None)
 459             new_tests.append(new_test)
 460         yield new_tests