tko/frontend.py

   1 import os, re, db, sys, datetime
   2 import common
   3 from autotest_lib.client.common_lib import kernel_versions
   4
   5 MAX_RECORDS = 50000L
   6 MAX_CELLS = 500000L
   7
   8 tko = os.path.dirname(os.path.realpath(os.path.abspath(__file__)))
   9 root_url_file = os.path.join(tko, '.root_url')
  10 if os.path.exists(root_url_file):
  11     html_root = open(root_url_file, 'r').readline().rstrip()
  12 else:
  13     html_root = '/results/'
  14
  15
  16 class status_cell:
  17     # One cell in the matrix of status data.
  18     def __init__(self):
  19         # Count is a dictionary: status -> count of tests with status
  20         self.status_count = {}
  21         self.reasons_list = []
  22         self.job_tag = None
  23         self.job_tag_count = 0
  24
  25
  26     def add(self, status, count, job_tags, reasons = None):
  27         assert count > 0
  28
  29         self.job_tag = job_tags
  30         self.job_tag_count += count
  31         if self.job_tag_count > 1:
  32             self.job_tag = None
  33
  34         self.status_count[status] = count
  35         ### status == 6 means 'GOOD'
  36         if status != 6:
  37             ## None implies sorting problems and extra CRs in a cell
  38             if reasons:
  39                 self.reasons_list.append(reasons)
  40
  41
  42 class status_data:
  43     def __init__(self, sql_rows, x_field, y_field, query_reasons = False):
  44         data = {}
  45         y_values = set()
  46
  47         # Walk through the query, filing all results by x, y info
  48         for row in sql_rows:
  49             if query_reasons:
  50                 (x,y, status, count, job_tags, reasons) = row
  51             else:
  52                 (x,y, status, count, job_tags) = row
  53                 reasons = None
  54             if not data.has_key(x):
  55                 data[x] = {}
  56             if not data[x].has_key(y):
  57                 y_values.add(y)
  58                 data[x][y] = status_cell()
  59             data[x][y].add(status, count, job_tags, reasons)
  60
  61         # 2-d hash of data - [x-value][y-value]
  62         self.data = data
  63         # List of possible columns (x-values)
  64         self.x_values = smart_sort(data.keys(), x_field)
  65         # List of rows columns (y-values)
  66         self.y_values = smart_sort(list(y_values), y_field)
  67         nCells = len(self.y_values)*len(self.x_values)
  68         if nCells > MAX_CELLS:
  69             msg = 'Exceeded allowed number of cells in a table'
  70             raise db.MySQLTooManyRows(msg)
  71
  72
  73 def get_matrix_data(db_obj, x_axis, y_axis, where = None,
  74                     query_reasons = False):
  75     # Searches on the test_view table - x_axis and y_axis must both be
  76     # column names in that table.
  77     x_field = test_view_field_dict[x_axis]
  78     y_field = test_view_field_dict[y_axis]
  79     query_fields_list = [x_field, y_field, 'status','COUNT(status)']
  80     query_fields_list.append("LEFT(GROUP_CONCAT(job_tag),100)")
  81     if query_reasons:
  82         query_fields_list.append(
  83                 "LEFT(GROUP_CONCAT(DISTINCT reason SEPARATOR '|'),500)"
  84                 )
  85     fields = ','.join(query_fields_list)
  86
  87     group_by = '%s, %s, status' % (x_field, y_field)
  88     rows = db_obj.select(fields, 'tko_test_view',
  89                     where=where, group_by=group_by, max_rows = MAX_RECORDS)
  90     return status_data(rows, x_field, y_field, query_reasons)
  91
  92
  93 # Dictionary used simply for fast lookups from short reference names for users
  94 # to fieldnames in test_view
  95 test_view_field_dict = {
  96         'kernel'        : 'kernel_printable',
  97         'hostname'      : 'machine_hostname',
  98         'test'          : 'test',
  99         'label'         : 'job_label',
 100         'machine_group' : 'machine_group',
 101         'reason'        : 'reason',
 102         'tag'           : 'job_tag',
 103         'user'          : 'job_username',
 104         'status'        : 'status_word',
 105         'time'          : 'test_finished_time',
 106         'start_time'    : 'test_started_time',
 107         'time_daily'    : 'DATE(test_finished_time)'
 108 }
 109
 110
 111 def smart_sort(list, field):
 112     if field == 'kernel_printable':
 113         def kernel_encode(kernel):
 114             return kernel_versions.version_encode(kernel)
 115         list.sort(key = kernel_encode, reverse = True)
 116         return list
 117     ## old records may contain time=None
 118     ## make None comparable with timestamp datetime or date
 119     elif field == 'test_finished_time':
 120         def convert_None_to_datetime(date_time):
 121             if not date_time:
 122                 return datetime.datetime(1970, 1, 1, 0, 0, 0)
 123             else:
 124                 return date_time
 125         list = map(convert_None_to_datetime, list)
 126     elif field == 'DATE(test_finished_time)':
 127         def convert_None_to_date(date):
 128             if not date:
 129                 return datetime.date(1970, 1, 1)
 130             else:
 131                 return date
 132         list = map(convert_None_to_date, list)
 133     list.sort()
 134     return list
 135
 136
 137 class group:
 138     @classmethod
 139     def select(klass, db):
 140         """Return all possible machine groups"""
 141         rows = db.select('distinct machine_group', 'tko_machines',
 142                                         'machine_group is not null')
 143         groupnames = sorted([row[0] for row in rows])
 144         return [klass(db, groupname) for groupname in groupnames]
 145
 146
 147     def __init__(self, db, name):
 148         self.name = name
 149         self.db = db
 150
 151
 152     def machines(self):
 153         return machine.select(self.db, { 'machine_group' : self.name })
 154
 155
 156     def tests(self, where = {}):
 157         values = [self.name]
 158         sql = 't inner join tko_machines m on m.machine_idx=t.machine_idx'
 159         sql += ' where m.machine_group=%s'
 160         for key in where.keys():
 161             sql += ' and %s=%%s' % key
 162             values.append(where[key])
 163         return test.select_sql(self.db, sql, values)
 164
 165
 166 class machine:
 167     @classmethod
 168     def select(klass, db, where = {}):
 169         fields = ['machine_idx', 'hostname', 'machine_group', 'owner']
 170         machines = []
 171         for row in db.select(','.join(fields), 'tko_machines', where):
 172             machines.append(klass(db, *row))
 173         return machines
 174
 175
 176     def __init__(self, db, idx, hostname, group, owner):
 177         self.db = db
 178         self.idx = idx
 179         self.hostname = hostname
 180         self.group = group
 181         self.owner = owner
 182
 183
 184 class kernel:
 185     @classmethod
 186     def select(klass, db, where = {}):
 187         fields = ['kernel_idx', 'kernel_hash', 'base', 'printable']
 188         rows = db.select(','.join(fields), 'tko_kernels', where)
 189         return [klass(db, *row) for row in rows]
 190
 191
 192     def __init__(self, db, idx, hash, base, printable):
 193         self.db = db
 194         self.idx = idx
 195         self.hash = hash
 196         self.base = base
 197         self.printable = printable
 198         self.patches = []    # THIS SHOULD PULL IN PATCHES!
 199
 200
 201 class test:
 202     @classmethod
 203     def select(klass, db, where={}, distinct=False):
 204         fields = ['test_idx', 'job_idx', 'test', 'subdir',
 205                   'kernel_idx', 'status', 'reason', 'machine_idx']
 206         tests = []
 207         for row in db.select(','.join(fields), 'tko_tests', where,
 208                              distinct):
 209             tests.append(klass(db, *row))
 210         return tests
 211
 212
 213     @classmethod
 214     def select_sql(klass, db, sql, values):
 215         fields = ['test_idx', 'job_idx', 'test', 'subdir',
 216                   'kernel_idx', 'status', 'reason', 'machine_idx']
 217         fields = ['t.'+field for field in fields]
 218         rows = db.select_sql(','.join(fields), 'tko_tests', sql, values)
 219         return [klass(db, *row) for row in rows]
 220
 221
 222     def __init__(self, db, test_idx, job_idx, testname, subdir, kernel_idx,
 223                  status_num, reason, machine_idx):
 224         self.idx = test_idx
 225         self.job = job(db, job_idx)
 226         self.testname = testname
 227         self.subdir = subdir
 228         self.kernel_idx = kernel_idx
 229         self.__kernel = None
 230         self.__iterations = None
 231         self.machine_idx = machine_idx
 232         self.__machine = None
 233         self.status_num = status_num
 234         self.status_word = db.status_word[status_num]
 235         self.reason = reason
 236         self.db = db
 237         if self.subdir:
 238             self.url = html_root + self.job.tag + '/' + self.subdir
 239         else:
 240             self.url = None
 241
 242
 243     def iterations(self):
 244         """
 245         Caching function for iterations
 246         """
 247         if not self.__iterations:
 248             self.__iterations = {}
 249             # A dictionary - dict{key} = [value1, value2, ....]
 250             where = {'test_idx' : self.idx}
 251             for i in iteration.select(self.db, where):
 252                 if self.__iterations.has_key(i.key):
 253                     self.__iterations[i.key].append(i.value)
 254                 else:
 255                     self.__iterations[i.key] = [i.value]
 256         return self.__iterations
 257
 258
 259     def kernel(self):
 260         """
 261         Caching function for kernels
 262         """
 263         if not self.__kernel:
 264             where = {'kernel_idx' : self.kernel_idx}
 265             self.__kernel = kernel.select(self.db, where)[0]
 266         return self.__kernel
 267
 268
 269     def machine(self):
 270         """
 271         Caching function for kernels
 272         """
 273         if not self.__machine:
 274             where = {'machine_idx' : self.machine_idx}
 275             self.__machine = machine.select(self.db, where)[0]
 276         return self.__machine
 277
 278
 279 class job:
 280     def __init__(self, db, job_idx):
 281         where = {'job_idx' : job_idx}
 282         rows = db.select('tag, machine_idx', 'tko_jobs', where)
 283         if rows:
 284             self.tag, self.machine_idx = rows[0]
 285             self.job_idx = job_idx
 286
 287
 288 class iteration:
 289     @classmethod
 290     def select(klass, db, where):
 291         fields = ['iteration', 'attribute', 'value']
 292         iterations = []
 293         rows = db.select(','.join(fields), 'tko_iteration_result', where)
 294         for row in rows:
 295             iterations.append(klass(*row))
 296         return iterations
 297
 298
 299     def __init__(self, iteration, key, value):
 300         self.iteration = iteration
 301         self.key = key
 302         self.value = value
 303
 304 # class patch:
 305 #       def __init__(self):
 306 #               self.spec = None