tko/compose_query.cgi

   1 #!/usr/bin/python
   2
   3 """
   4 Selects all rows and columns that satisfy the condition specified
   5 and draws the matrix. There is a seperate SQL query made for every (x,y)
   6 in the matrix.
   7 """
   8
   9 print "Content-type: text/html\n"
  10
  11 import sys, os, urllib, cgi, cgitb, re, datetime, time
  12
  13 total_wall_time_start = time.time()
  14
  15 import common
  16 from autotest_lib.tko import display, frontend, db, query_lib
  17 from autotest_lib.client.common_lib import kernel_versions
  18
  19 html_header = """\
  20 <form action="/tko/compose_query.cgi" method="get">
  21 <table border="0">
  22 <tr>
  23   <td>Column: </td>
  24   <td>Row: </td>
  25   <td>Condition: </td>
  26   <td align="center">
  27   <a href="http://autotest.kernel.org/wiki/AutotestTKOCondition">Help</a>
  28   </td>
  29 </tr>
  30 <tr>
  31   <td>
  32   <SELECT NAME="columns">
  33   %s
  34  </SELECT>
  35   </td>
  36   <td>
  37   <SELECT NAME="rows">
  38   %s
  39   </SELECT>
  40   </td>
  41   <td>
  42     <input type="text" name="condition" size="30" value="%s">
  43     <input type="hidden" name="title" value="%s">
  44   </td>
  45   <td align="center"><input type="submit" value="Submit">
  46   </td>
  47 </tr>
  48 </table>
  49 </form>
  50 <form action="/tko/save_query.cgi" method="get">
  51 <table border="0">
  52 <tr>
  53  <td>Name your query:&nbsp;&nbsp;</td>
  54   <td>
  55     <input type="text" name="label" size="15" value="">
  56   </td>
  57   <td align="center">&nbsp;<input type="submit" value="Save Query">
  58   </td>
  59  <td>&nbsp;&nbsp;<a href="/tko/query_history.cgi">View saved queries</a></td>
  60   <td>
  61     <input type="hidden" name="columns" value="%s">
  62     <input type="hidden" name="rows" value="%s">
  63     <input type="hidden" name="condition" value="%s">
  64   </td>
  65 </tr>
  66 </table>
  67 </form>
  68 """
  69
  70
  71 next_field = {
  72     'machine_group': 'hostname',
  73     'hostname': 'tag',
  74     'tag': 'tag',
  75
  76     'kernel': 'test',
  77     'test': 'label',
  78     'label': 'tag',
  79
  80     'reason': 'tag',
  81     'user': 'tag',
  82     'status': 'tag',
  83
  84     'time': 'tag',
  85     'time_daily': 'time',
  86 }
  87
  88
  89 def parse_field(form, form_field, field_default):
  90     if not form_field in form:
  91         return field_default
  92     field_input = form[form_field].value.lower()
  93     if field_input and field_input in frontend.test_view_field_dict:
  94         return field_input
  95     return field_default
  96
  97
  98 def parse_condition(form, form_field, field_default):
  99     if not form_field in form:
 100         return field_default
 101     return form[form_field].value
 102
 103
 104 form = cgi.FieldStorage()
 105
 106 title_field = parse_condition(form, 'title', '')
 107 row = parse_field(form, 'rows', 'kernel')
 108 column = parse_field(form, 'columns', 'machine_group')
 109 condition_field = parse_condition(form, 'condition', '')
 110
 111 if 'brief' in form.keys() and form['brief'].value <> '0':
 112     display.set_brief_mode()
 113
 114 ## caller can specify rows and columns that shall be included into the report
 115 ## regardless of whether actual test data is available yet
 116 force_row_field = parse_condition(form,'force_row','')
 117 force_column_field = parse_condition(form,'force_column','')
 118
 119
 120 def split_forced_fields(force_field):
 121     if force_field:
 122         return force_field.split()
 123     else:
 124         return []
 125
 126 force_row =  split_forced_fields(force_row_field)
 127 force_column =  split_forced_fields(force_column_field)
 128
 129 cgitb.enable()
 130 db_obj = db.db()
 131
 132
 133 def construct_link(x, y):
 134     next_row = row
 135     next_column = column
 136     condition_list = []
 137     if condition_field != '':
 138         condition_list.append(condition_field)
 139     if y:
 140         next_row = next_field[row]
 141         condition_list.append("%s='%s'" % (row, y))
 142     if x:
 143         next_column = next_field[column]
 144         condition_list.append("%s='%s'" % (column, x))
 145     next_condition = '&'.join(condition_list)
 146     link = '/tko/compose_query.cgi?' + urllib.urlencode({'columns': next_column,
 147                'rows': next_row, 'condition': next_condition,
 148                'title': title_field})
 149     return link
 150
 151
 152 def construct_logs_link(x, y, job_tag):
 153     job_path = frontend.html_root + job_tag + '/'
 154     test = ''
 155     if (row == 'test' and
 156         not y.split('.')[0] in ('boot', 'build', 'install')):
 157         test = y
 158     if (column == 'test' and
 159         not x.split('.')[0] in ('boot', 'build', 'install')):
 160         test = x
 161     return '/tko/retrieve_logs.cgi?' + urllib.urlencode({'job' : job_path,
 162          'test' : test})
 163
 164
 165 def create_select_options(selected_val):
 166     ret = ""
 167     for option in sorted(frontend.test_view_field_dict.keys()):
 168         if selected_val == option:
 169             selected = " SELECTED"
 170         else:
 171             selected = ""
 172
 173         ret += '<OPTION VALUE="%s"%s>%s</OPTION>\n' % \
 174                         (option, selected, option)
 175     return ret
 176
 177
 178 def map_kernel_base(kernel_name):
 179     ## insert <br> after each / in kernel name
 180     ## but spare consequtive //
 181     kernel_name = kernel_name.replace('/','/<br>')
 182     kernel_name = kernel_name.replace('/<br>/<br>','//')
 183     return kernel_name
 184
 185
 186 def header_tuneup(field_name, header):
 187         ## header tune up depends on particular field name and may include:
 188         ## - breaking header into several strings if it is long url
 189         ## - creating date from datetime stamp
 190         ## - possibly, expect more various refinements for different fields
 191         if field_name == 'kernel':
 192                 return  map_kernel_base(header)
 193         else:
 194                 return header
 195
 196
 197 # Kernel name mappings -- the kernels table 'printable' field is
 198 # effectively a sortable identifier for the kernel It encodes the base
 199 # release which is used for overall sorting, plus where patches are
 200 # applied it adds an increasing pNNN patch combination identifier
 201 # (actually the kernel_idx for the entry).  This allows sorting
 202 # as normal by the base kernel version and then sub-sorting by the
 203 # "first time we saw" a patch combination which should keep them in
 204 # approximatly date order.  This patch identifier is not suitable
 205 # for display, so we have to map it to a suitable html fragment for
 206 # display.  This contains the kernel base version plus the truncated
 207 # names of all the patches,
 208 #
 209 #     2.6.24-mm1 p112
 210 #     +add-new-string-functions-
 211 #     +x86-amd-thermal-interrupt
 212 #
 213 # This mapping is produced when the first mapping is request, with
 214 # a single query over the patches table; the result is then cached.
 215 #
 216 # Note: that we only count a base version as patched if it contains
 217 # patches which are not already "expressed" in the base version.
 218 # This includes both -gitN and -mmN kernels.
 219 map_kernel_map = None
 220
 221
 222 def map_kernel_init():
 223     fields = ['base', 'k.kernel_idx', 'name', 'url']
 224     map = {}
 225     for (base, idx, name, url) in db_obj.select(','.join(fields),
 226             'tko_kernels k, tko_patches p', 'k.kernel_idx=p.kernel_idx'):
 227         match = re.match(r'.*(-mm[0-9]+|-git[0-9]+)\.(bz2|gz)$', url)
 228         if match:
 229             continue
 230
 231         key = base + ' p%d' % (idx)
 232         if not map.has_key(key):
 233             map[key] = map_kernel_base(base) + ' p%d' % (idx)
 234         map[key] += ('<br>+<span title="' + name + '">' +
 235                  name[0:25] + '</span>')
 236
 237     return map
 238
 239
 240 def map_kernel(name):
 241     global map_kernel_map
 242     if map_kernel_map is None:
 243         map_kernel_map = map_kernel_init()
 244
 245     if map_kernel_map.has_key(name):
 246         return map_kernel_map[name]
 247
 248     return map_kernel_base(name.split(' ')[0])
 249
 250
 251 field_map = {
 252     'kernel':map_kernel
 253 }
 254
 255 sql_wall_time = 0
 256
 257 def gen_matrix():
 258     where = None
 259     if condition_field.strip() != '':
 260         try:
 261             where = query_lib.parse_scrub_and_gen_condition(
 262                 condition_field, frontend.test_view_field_dict)
 263             print "<!-- where clause: %s -->" % (where,)
 264         except:
 265             msg = "Unspecified error when parsing condition"
 266             return [[display.box(msg)]]
 267
 268     wall_time_start = time.time()
 269     try:
 270         ## Unfortunately, we can not request reasons of failure always
 271         ## because it may result in an inflated size of data transfer
 272         ## (at the moment we fetch 500 bytes of reason descriptions into
 273         ## each cell )
 274         ## If 'status' in [row,column] then either width or height
 275         ## of the table <=7, hence table is not really 2D, and
 276         ## query_reason is relatively save.
 277         ## At the same time view when either rows or columns grouped
 278         ## by status is when users need reasons of failures the most.
 279
 280         ## TO DO: implement [Show/Hide reasons] button or link in
 281         ## all views and make thorough performance testing
 282         test_data = frontend.get_matrix_data(db_obj, column, row, where,
 283                 query_reasons = ('status' in [row,column])
 284                 )
 285         global sql_wall_time
 286         sql_wall_time = time.time() - wall_time_start
 287
 288     except db.MySQLTooManyRows, error:
 289         return [[display.box(str(error))]]
 290
 291     for f_row in force_row:
 292         if not f_row in test_data.y_values:
 293             test_data.y_values.append(f_row)
 294     for f_column in force_column:
 295         if not f_column in test_data.x_values:
 296             test_data.x_values.append(f_column)
 297
 298     if not test_data.y_values:
 299         msg = "There are no results for this query (yet?)."
 300         return [[display.box(msg)]]
 301
 302     dict_url = {'columns': row,
 303                'rows': column, 'condition': condition_field,
 304                'title': title_field}
 305     link = '/tko/compose_query.cgi?' + urllib.urlencode(dict_url)
 306     header_row = [display.box("<center>(Flip Axis)</center>", link=link)]
 307
 308     for x in test_data.x_values:
 309         dx = x
 310         if field_map.has_key(column):
 311             dx = field_map[column](x)
 312         x_header = header_tuneup(column, dx)
 313         link = construct_link(x, None)
 314         header_row.append(display.box(x_header,header=True,link=link))
 315
 316     matrix = [header_row]
 317     # For each row, we are looping horizontally over the columns.
 318     for y in test_data.y_values:
 319         dy = y
 320         if field_map.has_key(row):
 321             dy = field_map[row](y)
 322         y_header = header_tuneup(row, dy)
 323         link = construct_link(None, y)
 324         cur_row = [display.box(y_header, header=True, link=link)]
 325         for x in test_data.x_values:
 326             ## next 2 lines: temporary, until non timestamped
 327             ## records are in the database
 328             if x==datetime.datetime(1970,1,1): x = None
 329             if y==datetime.datetime(1970,1,1): y = None
 330             try:
 331                 box_data = test_data.data[x][y]
 332             except:
 333                 cur_row.append(display.box(None, None,
 334                            row_label=y, column_label=x))
 335                 continue
 336             job_tag = test_data.data[x][y].job_tag
 337             if job_tag:
 338                 link = construct_logs_link(x, y, job_tag)
 339             else:
 340                 link = construct_link(x, y)
 341
 342             apnd = display.status_precounted_box(db_obj, box_data,
 343                                  link, y, x)
 344             cur_row.append(apnd)
 345         matrix.append(cur_row)
 346     return matrix
 347
 348
 349 def main():
 350     if display.is_brief_mode():
 351         ## create main grid table only as provided by gen_matrix()
 352         display.print_table(gen_matrix())
 353     else:
 354         # create the actual page
 355         print '<html><head><title>'
 356         print 'Filtered Autotest Results'
 357         print '</title></head><body>'
 358         display.print_main_header()
 359         print html_header % (create_select_options(column),
 360                      create_select_options(row),
 361                      condition_field, title_field,
 362                      ## history form
 363                      column,row,condition_field)
 364         if title_field:
 365             print '<h1> %s </h1>' % (title_field)
 366         print display.color_keys_row()
 367         display.print_table(gen_matrix())
 368         print display.color_keys_row()
 369         total_wall_time = time.time() - total_wall_time_start
 370
 371         perf_info = '<p style="font-size:x-small;">'
 372         perf_info += 'sql access wall time = %s secs,' % sql_wall_time
 373         perf_info += 'total wall time = %s secs</p>' % total_wall_time
 374         print perf_info
 375         print '</body></html>'
 376
 377
 378 main()