3 from autotest_lib
.client
.common_lib
import utils
as common_utils
4 from autotest_lib
.tko
import utils
as tko_utils
, models
, status_lib
5 from autotest_lib
.tko
.parsers
import base
8 class NoHostnameError(Exception):
12 class job(models
.job
):
13 def __init__(self
, dir):
14 job_dict
= job
.load_from_dir(dir)
15 super(job
, self
).__init
__(dir, **job_dict
)
19 def load_from_dir(cls
, dir):
20 keyval
= cls
.read_keyval(dir)
21 tko_utils
.dprint(str(keyval
))
23 user
= keyval
.get("user", None)
24 label
= keyval
.get("label", None)
25 queued_time
= tko_utils
.get_timestamp(keyval
, "job_queued")
26 started_time
= tko_utils
.get_timestamp(keyval
, "job_started")
27 finished_time
= tko_utils
.get_timestamp(keyval
, "job_finished")
28 machine
= cls
.determine_hostname(keyval
, dir)
29 machine_group
= cls
.determine_machine_group(machine
, dir)
30 machine_owner
= keyval
.get("owner", None)
32 aborted_by
= keyval
.get("aborted_by", None)
33 aborted_at
= tko_utils
.get_timestamp(keyval
, "aborted_on")
35 return {"user": user
, "label": label
, "machine": machine
,
36 "queued_time": queued_time
, "started_time": started_time
,
37 "finished_time": finished_time
, "machine_owner": machine_owner
,
38 "machine_group": machine_group
, "aborted_by": aborted_by
,
39 "aborted_on": aborted_at
, "keyval_dict": keyval
}
43 def determine_hostname(cls
, keyval
, job_dir
):
44 host_group_name
= keyval
.get("host_group_name", None)
45 machine
= keyval
.get("hostname", "")
46 is_multimachine
= "," in machine
48 # determine what hostname to use
50 if is_multimachine
or not machine
:
51 tko_utils
.dprint("Using host_group_name %r instead of "
52 "machine name." % host_group_name
)
53 machine
= host_group_name
56 machine
= job
.find_hostname(job_dir
) # find a unique hostname
57 except NoHostnameError
:
58 pass # just use the comma-separated name
60 tko_utils
.dprint("MACHINE NAME: %s" % machine
)
65 def determine_machine_group(cls
, hostname
, job_dir
):
66 machine_groups
= set()
67 for individual_hostname
in hostname
.split(","):
68 host_keyval
= models
.test
.parse_host_keyval(job_dir
,
71 tko_utils
.dprint('Unable to parse host keyval for %s'
72 % individual_hostname
)
73 elif "platform" in host_keyval
:
74 machine_groups
.add(host_keyval
["platform"])
75 machine_group
= ",".join(sorted(machine_groups
))
76 tko_utils
.dprint("MACHINE GROUP: %s" % machine_group
)
81 def find_hostname(path
):
82 hostname
= os
.path
.join(path
, "sysinfo", "hostname")
84 machine
= open(hostname
).readline().rstrip()
87 tko_utils
.dprint("Could not read a hostname from "
90 uname
= os
.path
.join(path
, "sysinfo", "uname_-a")
92 machine
= open(uname
).readline().split()[1]
95 tko_utils
.dprint("Could not read a hostname from "
98 raise NoHostnameError("Unable to find a machine name")
101 class kernel(models
.kernel
):
102 def __init__(self
, job
, verify_ident
=None):
103 kernel_dict
= kernel
.load_from_dir(job
.dir, verify_ident
)
104 super(kernel
, self
).__init
__(**kernel_dict
)
108 def load_from_dir(dir, verify_ident
=None):
109 # try and load the booted kernel version
112 build_dir
= os
.path
.join(dir, "build")
114 if not os
.path
.exists(build_dir
):
116 build_log
= os
.path
.join(build_dir
, "debug", "build_log")
117 attributes
= kernel
.load_from_build_log(build_log
)
121 build_dir
= os
.path
.join(dir, "build.%d" % (i
))
127 base
= kernel
.load_from_sysinfo(dir)
131 base
, patches
, hashes
= attributes
132 tko_utils
.dprint("kernel.__init__() found kernel version %s"
135 # compute the kernel hash
136 if base
== "UNKNOWN":
137 kernel_hash
= "UNKNOWN"
139 kernel_hash
= kernel
.compute_hash(base
, hashes
)
141 return {"base": base
, "patches": patches
,
142 "kernel_hash": kernel_hash
}
146 def load_from_sysinfo(path
):
147 for subdir
in ("reboot1", ""):
148 uname_path
= os
.path
.join(path
, "sysinfo", subdir
,
150 if not os
.path
.exists(uname_path
):
152 uname
= open(uname_path
).readline().split()
153 return re
.sub("-autotest$", "", uname
[2])
158 def load_from_build_log(path
):
159 if not os
.path
.exists(path
):
162 base
, patches
, hashes
= "UNKNOWN", [], []
163 for line
in file(path
):
164 head
, rest
= line
.split(": ", 1)
168 elif head
== "PATCH":
169 patches
.append(patch(*rest
))
170 hashes
.append(rest
[2])
171 return base
, patches
, hashes
174 class test(models
.test
):
175 def __init__(self
, subdir
, testname
, status
, reason
, test_kernel
,
176 machine
, started_time
, finished_time
, iterations
,
178 # for backwards compatibility with the original parser
179 # implementation, if there is no test version we need a NULL
180 # value to be used; also, if there is a version it should
181 # be terminated by a newline
182 if "version" in attributes
:
183 attributes
["version"] = str(attributes
["version"])
185 attributes
["version"] = None
187 super(test
, self
).__init
__(subdir
, testname
, status
, reason
,
188 test_kernel
, machine
, started_time
,
189 finished_time
, iterations
,
194 def load_iterations(keyval_path
):
195 return iteration
.load_from_keyval(keyval_path
)
198 class patch(models
.patch
):
199 def __init__(self
, spec
, reference
, hash):
200 tko_utils
.dprint("PATCH::%s %s %s" % (spec
, reference
, hash))
201 super(patch
, self
).__init
__(spec
, reference
, hash)
203 self
.reference
= reference
207 class iteration(models
.iteration
):
209 def parse_line_into_dicts(line
, attr_dict
, perf_dict
):
210 key
, value
= line
.split("=", 1)
211 perf_dict
[key
] = value
214 class status_line(object):
215 def __init__(self
, indent
, status
, subdir
, testname
, reason
,
217 # pull out the type & status of the line
218 if status
== "START":
221 elif status
.startswith("END "):
223 self
.status
= status
[4:]
227 assert (self
.status
is None or
228 self
.status
in status_lib
.statuses
)
230 # save all the other parameters
232 self
.subdir
= self
.parse_name(subdir
)
233 self
.testname
= self
.parse_name(testname
)
235 self
.optional_fields
= optional_fields
239 def parse_name(name
):
246 def is_status_line(line
):
247 return re
.search(r
"^\t*(\S[^\t]*\t){3}", line
) is not None
251 def parse_line(cls
, line
):
252 if not status_line
.is_status_line(line
):
254 match
= re
.search(r
"^(\t*)(.*)$", line
, flags
=re
.DOTALL
)
256 # A more useful error message than:
257 # AttributeError: 'NoneType' object has no attribute 'groups'
258 # to help us debug WTF happens on occasion here.
259 raise RuntimeError("line %r could not be parsed." % line
)
260 indent
, line
= match
.groups()
263 # split the line into the fixed and optional fields
264 parts
= line
.rstrip("\n").split("\t")
267 status
, subdir
, testname
= parts
[0:part_index
]
269 # all optional parts should be of the form "key=value". once we've found
270 # a non-matching part, treat it and the rest of the parts as the reason.
272 while part_index
< len(parts
):
273 kv
= parts
[part_index
].split('=', 1)
277 optional_fields
[kv
[0]] = kv
[1]
280 reason
= "\t".join(parts
[part_index
:])
282 # build up a new status_line and return it
283 return cls(indent
, status
, subdir
, testname
, reason
,
287 class parser(base
.parser
):
293 def state_iterator(self
, buffer):
298 stack
= status_lib
.status_stack()
299 current_kernel
= kernel(self
.job
)
300 boot_in_progress
= False
304 while not self
.finished
or buffer.size():
305 # stop processing once the buffer is empty
306 if buffer.size() == 0:
311 # parse the next line
313 tko_utils
.dprint('\nSTATUS: ' + line
.strip())
314 line
= status_line
.parse_line(line
)
316 tko_utils
.dprint('non-status line, ignoring')
317 continue # ignore non-status lines
319 # have we hit the job start line?
320 if (line
.type == "START" and not line
.subdir
and
323 tko_utils
.dprint("found job level start "
324 "marker, looking for level "
328 # have we hit the job end line?
329 if (line
.type == "END" and not line
.subdir
and
331 tko_utils
.dprint("found job level end "
332 "marker, looking for level "
336 # START line, just push another layer on to the stack
337 # and grab the start time if this is at the job level
338 # we're currently seeking
339 if line
.type == "START":
342 if line
.indent
== sought_level
:
344 tko_utils
.get_timestamp(
345 line
.optional_fields
, "timestamp")
346 tko_utils
.dprint("start line, ignoring")
348 # otherwise, update the status on the stack
350 tko_utils
.dprint("GROPE_STATUS: %s" %
351 [stack
.current_status(),
352 line
.status
, line
.subdir
,
353 line
.testname
, line
.reason
])
354 stack
.update(line
.status
)
356 if line
.status
== "ALERT":
357 tko_utils
.dprint("job level alert, recording")
358 alert_pending
= line
.reason
361 # ignore Autotest.install => GOOD lines
362 if (line
.testname
== "Autotest.install" and
363 line
.status
== "GOOD"):
364 tko_utils
.dprint("Successful Autotest "
368 # ignore END lines for a reboot group
369 if (line
.testname
== "reboot" and line
.type == "END"):
370 tko_utils
.dprint("reboot group, ignoring")
373 # convert job-level ABORTs into a 'CLIENT_JOB' test, and
374 # ignore other job-level events
375 if line
.testname
is None:
376 if (line
.status
== "ABORT" and
378 line
.testname
= "CLIENT_JOB"
380 tko_utils
.dprint("job level event, "
384 # use the group subdir for END lines
385 if line
.type == "END":
386 line
.subdir
= group_subdir
388 # are we inside a block group?
389 if (line
.indent
!= sought_level
and
390 line
.status
!= "ABORT" and
391 not line
.testname
.startswith('reboot.')):
393 tko_utils
.dprint("set group_subdir: "
395 group_subdir
= line
.subdir
396 tko_utils
.dprint("ignoring incorrect indent "
398 (line
.indent
, sought_level
))
401 # use the subdir as the testname, except for
402 # boot.* and kernel.* tests
403 if (line
.testname
is None or
404 not re
.search(r
"^(boot(\.\d+)?$|kernel\.)",
406 if line
.subdir
and '.' in line
.subdir
:
407 line
.testname
= line
.subdir
409 # has a reboot started?
410 if line
.testname
== "reboot.start":
411 started_time
= tko_utils
.get_timestamp(
412 line
.optional_fields
, "timestamp")
413 tko_utils
.dprint("reboot start event, "
415 boot_in_progress
= True
418 # has a reboot finished?
419 if line
.testname
== "reboot.verify":
420 line
.testname
= "boot.%d" % boot_count
421 tko_utils
.dprint("reboot verified")
422 boot_in_progress
= False
423 verify_ident
= line
.reason
.strip()
424 current_kernel
= kernel(self
.job
, verify_ident
)
428 line
.status
= "ALERT"
429 line
.reason
= alert_pending
432 # create the actual test object
433 finished_time
= tko_utils
.get_timestamp(
434 line
.optional_fields
, "timestamp")
435 final_status
= stack
.end()
436 tko_utils
.dprint("Adding: "
437 "%s\nSubdir:%s\nTestname:%s\n%s" %
438 (final_status
, line
.subdir
,
439 line
.testname
, line
.reason
))
440 new_test
= test
.parse_test(self
.job
, line
.subdir
,
442 final_status
, line
.reason
,
447 new_tests
.append(new_test
)
449 # the job is finished, but we never came back from reboot
451 testname
= "boot.%d" % boot_count
452 reason
= "machine did not return from reboot"
453 tko_utils
.dprint(("Adding: ABORT\nSubdir:----\n"
455 % (testname
, reason
))
456 new_test
= test
.parse_test(self
.job
, None, testname
,
458 current_kernel
, None, None)
459 new_tests
.append(new_test
)