Fix C++ PCH issues.
[clang.git] / tools / scan-view / ScanView.py
blob837adae0f76356db9c1a01b7d46cfe6a1a1f6029
1 import BaseHTTPServer
2 import SimpleHTTPServer
3 import os
4 import sys
5 import urllib, urlparse
6 import posixpath
7 import StringIO
8 import re
9 import shutil
10 import threading
11 import time
12 import socket
13 import itertools
15 import Reporter
16 import ConfigParser
18 ###
19 # Various patterns matched or replaced by server.
21 kReportFileRE = re.compile('(.*/)?report-(.*)\\.html')
23 kBugKeyValueRE = re.compile('<!-- BUG([^ ]*) (.*) -->')
25 # <!-- REPORTPROBLEM file="crashes/clang_crash_ndSGF9.mi" stderr="crashes/clang_crash_ndSGF9.mi.stderr.txt" info="crashes/clang_crash_ndSGF9.mi.info" -->
27 kReportCrashEntryRE = re.compile('<!-- REPORTPROBLEM (.*?)-->')
28 kReportCrashEntryKeyValueRE = re.compile(' ?([^=]+)="(.*?)"')
30 kReportReplacements = []
32 # Add custom javascript.
33 kReportReplacements.append((re.compile('<!-- SUMMARYENDHEAD -->'), """\
34 <script language="javascript" type="text/javascript">
35 function load(url) {
36 if (window.XMLHttpRequest) {
37 req = new XMLHttpRequest();
38 } else if (window.ActiveXObject) {
39 req = new ActiveXObject("Microsoft.XMLHTTP");
41 if (req != undefined) {
42 req.open("GET", url, true);
43 req.send("");
46 </script>"""))
48 # Insert additional columns.
49 kReportReplacements.append((re.compile('<!-- REPORTBUGCOL -->'),
50 '<td></td><td></td>'))
52 # Insert report bug and open file links.
53 kReportReplacements.append((re.compile('<!-- REPORTBUG id="report-(.*)\\.html" -->'),
54 ('<td class="Button"><a href="report/\\1">Report Bug</a></td>' +
55 '<td class="Button"><a href="javascript:load(\'open/\\1\')">Open File</a></td>')))
57 kReportReplacements.append((re.compile('<!-- REPORTHEADER -->'),
58 '<h3><a href="/">Summary</a> > Report %(report)s</h3>'))
60 kReportReplacements.append((re.compile('<!-- REPORTSUMMARYEXTRA -->'),
61 '<td class="Button"><a href="report/%(report)s">Report Bug</a></td>'))
63 # Insert report crashes link.
65 # Disabled for the time being until we decide exactly when this should
66 # be enabled. Also the radar reporter needs to be fixed to report
67 # multiple files.
69 #kReportReplacements.append((re.compile('<!-- REPORTCRASHES -->'),
70 # '<br>These files will automatically be attached to ' +
71 # 'reports filed here: <a href="report_crashes">Report Crashes</a>.'))
73 ###
74 # Other simple parameters
76 kResources = posixpath.join(posixpath.dirname(__file__), 'Resources')
77 kConfigPath = os.path.expanduser('~/.scanview.cfg')
79 ###
81 __version__ = "0.1"
83 __all__ = ["create_server"]
85 class ReporterThread(threading.Thread):
86 def __init__(self, report, reporter, parameters, server):
87 threading.Thread.__init__(self)
88 self.report = report
89 self.server = server
90 self.reporter = reporter
91 self.parameters = parameters
92 self.success = False
93 self.status = None
95 def run(self):
96 result = None
97 try:
98 if self.server.options.debug:
99 print >>sys.stderr, "%s: SERVER: submitting bug."%(sys.argv[0],)
100 self.status = self.reporter.fileReport(self.report, self.parameters)
101 self.success = True
102 time.sleep(3)
103 if self.server.options.debug:
104 print >>sys.stderr, "%s: SERVER: submission complete."%(sys.argv[0],)
105 except Reporter.ReportFailure,e:
106 self.status = e.value
107 except Exception,e:
108 s = StringIO.StringIO()
109 import traceback
110 print >>s,'<b>Unhandled Exception</b><br><pre>'
111 traceback.print_exc(e,file=s)
112 print >>s,'</pre>'
113 self.status = s.getvalue()
115 class ScanViewServer(BaseHTTPServer.HTTPServer):
116 def __init__(self, address, handler, root, reporters, options):
117 BaseHTTPServer.HTTPServer.__init__(self, address, handler)
118 self.root = root
119 self.reporters = reporters
120 self.options = options
121 self.halted = False
122 self.config = None
123 self.load_config()
125 def load_config(self):
126 self.config = ConfigParser.RawConfigParser()
128 # Add defaults
129 self.config.add_section('ScanView')
130 for r in self.reporters:
131 self.config.add_section(r.getName())
132 for p in r.getParameters():
133 if p.saveConfigValue():
134 self.config.set(r.getName(), p.getName(), '')
136 # Ignore parse errors
137 try:
138 self.config.read([kConfigPath])
139 except:
140 pass
142 # Save on exit
143 import atexit
144 atexit.register(lambda: self.save_config())
146 def save_config(self):
147 # Ignore errors (only called on exit).
148 try:
149 f = open(kConfigPath,'w')
150 self.config.write(f)
151 f.close()
152 except:
153 pass
155 def halt(self):
156 self.halted = True
157 if self.options.debug:
158 print >>sys.stderr, "%s: SERVER: halting." % (sys.argv[0],)
160 def serve_forever(self):
161 while not self.halted:
162 if self.options.debug > 1:
163 print >>sys.stderr, "%s: SERVER: waiting..." % (sys.argv[0],)
164 try:
165 self.handle_request()
166 except OSError,e:
167 print 'OSError',e.errno
169 def finish_request(self, request, client_address):
170 if self.options.autoReload:
171 import ScanView
172 self.RequestHandlerClass = reload(ScanView).ScanViewRequestHandler
173 BaseHTTPServer.HTTPServer.finish_request(self, request, client_address)
175 def handle_error(self, request, client_address):
176 # Ignore socket errors
177 info = sys.exc_info()
178 if info and isinstance(info[1], socket.error):
179 if self.options.debug > 1:
180 print >>sys.stderr, "%s: SERVER: ignored socket error." % (sys.argv[0],)
181 return
182 BaseHTTPServer.HTTPServer.handle_error(self, request, client_address)
184 # Borrowed from Quixote, with simplifications.
185 def parse_query(qs, fields=None):
186 if fields is None:
187 fields = {}
188 for chunk in filter(None, qs.split('&')):
189 if '=' not in chunk:
190 name = chunk
191 value = ''
192 else:
193 name, value = chunk.split('=', 1)
194 name = urllib.unquote(name.replace('+', ' '))
195 value = urllib.unquote(value.replace('+', ' '))
196 item = fields.get(name)
197 if item is None:
198 fields[name] = [value]
199 else:
200 item.append(value)
201 return fields
203 class ScanViewRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
204 server_version = "ScanViewServer/" + __version__
205 dynamic_mtime = time.time()
207 def do_HEAD(self):
208 try:
209 SimpleHTTPServer.SimpleHTTPRequestHandler.do_HEAD(self)
210 except Exception,e:
211 self.handle_exception(e)
213 def do_GET(self):
214 try:
215 SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self)
216 except Exception,e:
217 self.handle_exception(e)
219 def do_POST(self):
220 """Serve a POST request."""
221 try:
222 length = self.headers.getheader('content-length') or "0"
223 try:
224 length = int(length)
225 except:
226 length = 0
227 content = self.rfile.read(length)
228 fields = parse_query(content)
229 f = self.send_head(fields)
230 if f:
231 self.copyfile(f, self.wfile)
232 f.close()
233 except Exception,e:
234 self.handle_exception(e)
236 def log_message(self, format, *args):
237 if self.server.options.debug:
238 sys.stderr.write("%s: SERVER: %s - - [%s] %s\n" %
239 (sys.argv[0],
240 self.address_string(),
241 self.log_date_time_string(),
242 format%args))
244 def load_report(self, report):
245 path = os.path.join(self.server.root, 'report-%s.html'%report)
246 data = open(path).read()
247 keys = {}
248 for item in kBugKeyValueRE.finditer(data):
249 k,v = item.groups()
250 keys[k] = v
251 return keys
253 def load_crashes(self):
254 path = posixpath.join(self.server.root, 'index.html')
255 data = open(path).read()
256 problems = []
257 for item in kReportCrashEntryRE.finditer(data):
258 fieldData = item.group(1)
259 fields = dict([i.groups() for i in
260 kReportCrashEntryKeyValueRE.finditer(fieldData)])
261 problems.append(fields)
262 return problems
264 def handle_exception(self, exc):
265 import traceback
266 s = StringIO.StringIO()
267 print >>s, "INTERNAL ERROR\n"
268 traceback.print_exc(exc, s)
269 f = self.send_string(s.getvalue(), 'text/plain')
270 if f:
271 self.copyfile(f, self.wfile)
272 f.close()
274 def get_scalar_field(self, name):
275 if name in self.fields:
276 return self.fields[name][0]
277 else:
278 return None
280 def submit_bug(self, c):
281 title = self.get_scalar_field('title')
282 description = self.get_scalar_field('description')
283 report = self.get_scalar_field('report')
284 reporterIndex = self.get_scalar_field('reporter')
285 files = []
286 for fileID in self.fields.get('files',[]):
287 try:
288 i = int(fileID)
289 except:
290 i = None
291 if i is None or i<0 or i>=len(c.files):
292 return (False, 'Invalid file ID')
293 files.append(c.files[i])
295 if not title:
296 return (False, "Missing title.")
297 if not description:
298 return (False, "Missing description.")
299 try:
300 reporterIndex = int(reporterIndex)
301 except:
302 return (False, "Invalid report method.")
304 # Get the reporter and parameters.
305 reporter = self.server.reporters[reporterIndex]
306 parameters = {}
307 for o in reporter.getParameters():
308 name = '%s_%s'%(reporter.getName(),o.getName())
309 if name not in self.fields:
310 return (False,
311 'Missing field "%s" for %s report method.'%(name,
312 reporter.getName()))
313 parameters[o.getName()] = self.get_scalar_field(name)
315 # Update config defaults.
316 if report != 'None':
317 self.server.config.set('ScanView', 'reporter', reporterIndex)
318 for o in reporter.getParameters():
319 if o.saveConfigValue():
320 name = o.getName()
321 self.server.config.set(reporter.getName(), name, parameters[name])
323 # Create the report.
324 bug = Reporter.BugReport(title, description, files)
326 # Kick off a reporting thread.
327 t = ReporterThread(bug, reporter, parameters, self.server)
328 t.start()
330 # Wait for thread to die...
331 while t.isAlive():
332 time.sleep(.25)
333 submitStatus = t.status
335 return (t.success, t.status)
337 def send_report_submit(self):
338 report = self.get_scalar_field('report')
339 c = self.get_report_context(report)
340 if c.reportSource is None:
341 reportingFor = "Report Crashes > "
342 fileBug = """\
343 <a href="/report_crashes">File Bug</a> > """%locals()
344 else:
345 reportingFor = '<a href="/%s">Report %s</a> > ' % (c.reportSource,
346 report)
347 fileBug = '<a href="/report/%s">File Bug</a> > ' % report
348 title = self.get_scalar_field('title')
349 description = self.get_scalar_field('description')
351 res,message = self.submit_bug(c)
353 if res:
354 statusClass = 'SubmitOk'
355 statusName = 'Succeeded'
356 else:
357 statusClass = 'SubmitFail'
358 statusName = 'Failed'
360 result = """
361 <head>
362 <title>Bug Submission</title>
363 <link rel="stylesheet" type="text/css" href="/scanview.css" />
364 </head>
365 <body>
366 <h3>
367 <a href="/">Summary</a> >
368 %(reportingFor)s
369 %(fileBug)s
370 Submit</h3>
371 <form name="form" action="">
372 <table class="form">
373 <tr><td>
374 <table class="form_group">
375 <tr>
376 <td class="form_clabel">Title:</td>
377 <td class="form_value">
378 <input type="text" name="title" size="50" value="%(title)s" disabled>
379 </td>
380 </tr>
381 <tr>
382 <td class="form_label">Description:</td>
383 <td class="form_value">
384 <textarea rows="10" cols="80" name="description" disabled>
385 %(description)s
386 </textarea>
387 </td>
388 </table>
389 </td></tr>
390 </table>
391 </form>
392 <h1 class="%(statusClass)s">Submission %(statusName)s</h1>
393 %(message)s
395 <hr>
396 <a href="/">Return to Summary</a>
397 </body>
398 </html>"""%locals()
399 return self.send_string(result)
401 def send_open_report(self, report):
402 try:
403 keys = self.load_report(report)
404 except IOError:
405 return self.send_error(400, 'Invalid report.')
407 file = keys.get('FILE')
408 if not file or not posixpath.exists(file):
409 return self.send_error(400, 'File does not exist: "%s"' % file)
411 import startfile
412 if self.server.options.debug:
413 print >>sys.stderr, '%s: SERVER: opening "%s"'%(sys.argv[0],
414 file)
416 status = startfile.open(file)
417 if status:
418 res = 'Opened: "%s"' % file
419 else:
420 res = 'Open failed: "%s"' % file
422 return self.send_string(res, 'text/plain')
424 def get_report_context(self, report):
425 class Context:
426 pass
427 if report is None or report == 'None':
428 data = self.load_crashes()
429 # Don't allow empty reports.
430 if not data:
431 raise ValueError, 'No crashes detected!'
432 c = Context()
433 c.title = 'clang static analyzer failures'
435 stderrSummary = ""
436 for item in data:
437 if 'stderr' in item:
438 path = posixpath.join(self.server.root, item['stderr'])
439 if os.path.exists(path):
440 lns = itertools.islice(open(path), 0, 10)
441 stderrSummary += '%s\n--\n%s' % (item.get('src',
442 '<unknown>'),
443 ''.join(lns))
445 c.description = """\
446 The clang static analyzer failed on these inputs:
449 STDERR Summary
450 --------------
452 """ % ('\n'.join([item.get('src','<unknown>') for item in data]),
453 stderrSummary)
454 c.reportSource = None
455 c.navMarkup = "Report Crashes > "
456 c.files = []
457 for item in data:
458 c.files.append(item.get('src',''))
459 c.files.append(posixpath.join(self.server.root,
460 item.get('file','')))
461 c.files.append(posixpath.join(self.server.root,
462 item.get('clangfile','')))
463 c.files.append(posixpath.join(self.server.root,
464 item.get('stderr','')))
465 c.files.append(posixpath.join(self.server.root,
466 item.get('info','')))
467 # Just in case something failed, ignore files which don't
468 # exist.
469 c.files = [f for f in c.files
470 if os.path.exists(f) and os.path.isfile(f)]
471 else:
472 # Check that this is a valid report.
473 path = posixpath.join(self.server.root, 'report-%s.html' % report)
474 if not posixpath.exists(path):
475 raise ValueError, 'Invalid report ID'
476 keys = self.load_report(report)
477 c = Context()
478 c.title = keys.get('DESC','clang error (unrecognized')
479 c.description = """\
480 Bug reported by the clang static analyzer.
482 Description: %s
483 File: %s
484 Line: %s
485 """%(c.title, keys.get('FILE','<unknown>'), keys.get('LINE', '<unknown>'))
486 c.reportSource = 'report-%s.html' % report
487 c.navMarkup = """<a href="/%s">Report %s</a> > """ % (c.reportSource,
488 report)
490 c.files = [path]
491 return c
493 def send_report(self, report, configOverrides=None):
494 def getConfigOption(section, field):
495 if (configOverrides is not None and
496 section in configOverrides and
497 field in configOverrides[section]):
498 return configOverrides[section][field]
499 return self.server.config.get(section, field)
501 # report is None is used for crashes
502 try:
503 c = self.get_report_context(report)
504 except ValueError, e:
505 return self.send_error(400, e.message)
507 title = c.title
508 description= c.description
509 reportingFor = c.navMarkup
510 if c.reportSource is None:
511 extraIFrame = ""
512 else:
513 extraIFrame = """\
514 <iframe src="/%s" width="100%%" height="40%%"
515 scrolling="auto" frameborder="1">
516 <a href="/%s">View Bug Report</a>
517 </iframe>""" % (c.reportSource, c.reportSource)
519 reporterSelections = []
520 reporterOptions = []
522 try:
523 active = int(getConfigOption('ScanView','reporter'))
524 except:
525 active = 0
526 for i,r in enumerate(self.server.reporters):
527 selected = (i == active)
528 if selected:
529 selectedStr = ' selected'
530 else:
531 selectedStr = ''
532 reporterSelections.append('<option value="%d"%s>%s</option>'%(i,selectedStr,r.getName()))
533 options = '\n'.join([ o.getHTML(r,title,getConfigOption) for o in r.getParameters()])
534 display = ('none','')[selected]
535 reporterOptions.append("""\
536 <tr id="%sReporterOptions" style="display:%s">
537 <td class="form_label">%s Options</td>
538 <td class="form_value">
539 <table class="form_inner_group">
541 </table>
542 </td>
543 </tr>
544 """%(r.getName(),display,r.getName(),options))
545 reporterSelections = '\n'.join(reporterSelections)
546 reporterOptionsDivs = '\n'.join(reporterOptions)
547 reportersArray = '[%s]'%(','.join([`r.getName()` for r in self.server.reporters]))
549 if c.files:
550 fieldSize = min(5, len(c.files))
551 attachFileOptions = '\n'.join(["""\
552 <option value="%d" selected>%s</option>""" % (i,v) for i,v in enumerate(c.files)])
553 attachFileRow = """\
554 <tr>
555 <td class="form_label">Attach:</td>
556 <td class="form_value">
557 <select style="width:100%%" name="files" multiple size=%d>
559 </select>
560 </td>
561 </tr>
562 """ % (min(5, len(c.files)), attachFileOptions)
563 else:
564 attachFileRow = ""
566 result = """<html>
567 <head>
568 <title>File Bug</title>
569 <link rel="stylesheet" type="text/css" href="/scanview.css" />
570 </head>
571 <script language="javascript" type="text/javascript">
572 var reporters = %(reportersArray)s;
573 function updateReporterOptions() {
574 index = document.getElementById('reporter').selectedIndex;
575 for (var i=0; i < reporters.length; ++i) {
576 o = document.getElementById(reporters[i] + "ReporterOptions");
577 if (i == index) {
578 o.style.display = "";
579 } else {
580 o.style.display = "none";
584 </script>
585 <body onLoad="updateReporterOptions()">
586 <h3>
587 <a href="/">Summary</a> >
588 %(reportingFor)s
589 File Bug</h3>
590 <form name="form" action="/report_submit" method="post">
591 <input type="hidden" name="report" value="%(report)s">
593 <table class="form">
594 <tr><td>
595 <table class="form_group">
596 <tr>
597 <td class="form_clabel">Title:</td>
598 <td class="form_value">
599 <input type="text" name="title" size="50" value="%(title)s">
600 </td>
601 </tr>
602 <tr>
603 <td class="form_label">Description:</td>
604 <td class="form_value">
605 <textarea rows="10" cols="80" name="description">
606 %(description)s
607 </textarea>
608 </td>
609 </tr>
611 %(attachFileRow)s
613 </table>
614 <br>
615 <table class="form_group">
616 <tr>
617 <td class="form_clabel">Method:</td>
618 <td class="form_value">
619 <select id="reporter" name="reporter" onChange="updateReporterOptions()">
620 %(reporterSelections)s
621 </select>
622 </td>
623 </tr>
624 %(reporterOptionsDivs)s
625 </table>
626 <br>
627 </td></tr>
628 <tr><td class="form_submit">
629 <input align="right" type="submit" name="Submit" value="Submit">
630 </td></tr>
631 </table>
632 </form>
634 %(extraIFrame)s
636 </body>
637 </html>"""%locals()
639 return self.send_string(result)
641 def send_head(self, fields=None):
642 if (self.server.options.onlyServeLocal and
643 self.client_address[0] != '127.0.0.1'):
644 return self.send_error('401', 'Unauthorized host.')
646 if fields is None:
647 fields = {}
648 self.fields = fields
650 o = urlparse.urlparse(self.path)
651 self.fields = parse_query(o.query, fields)
652 path = posixpath.normpath(urllib.unquote(o.path))
654 # Split the components and strip the root prefix.
655 components = path.split('/')[1:]
657 # Special case some top-level entries.
658 if components:
659 name = components[0]
660 if len(components)==2:
661 if name=='report':
662 return self.send_report(components[1])
663 elif name=='open':
664 return self.send_open_report(components[1])
665 elif len(components)==1:
666 if name=='quit':
667 self.server.halt()
668 return self.send_string('Goodbye.', 'text/plain')
669 elif name=='report_submit':
670 return self.send_report_submit()
671 elif name=='report_crashes':
672 overrides = { 'ScanView' : {},
673 'Radar' : {},
674 'Email' : {} }
675 for i,r in enumerate(self.server.reporters):
676 if r.getName() == 'Radar':
677 overrides['ScanView']['reporter'] = i
678 break
679 overrides['Radar']['Component'] = 'llvm - checker'
680 overrides['Radar']['Component Version'] = 'X'
681 return self.send_report(None, overrides)
682 elif name=='favicon.ico':
683 return self.send_path(posixpath.join(kResources,'bugcatcher.ico'))
685 # Match directory entries.
686 if components[-1] == '':
687 components[-1] = 'index.html'
689 suffix = '/'.join(components)
691 # The summary may reference source files on disk using rooted
692 # paths. Make sure these resolve correctly for now.
693 # FIXME: This isn't a very good idea... we should probably
694 # mark rooted paths somehow.
695 if os.path.exists(posixpath.join('/', suffix)):
696 path = posixpath.join('/', suffix)
697 else:
698 path = posixpath.join(self.server.root, suffix)
700 if self.server.options.debug > 1:
701 print >>sys.stderr, '%s: SERVER: sending path "%s"'%(sys.argv[0],
702 path)
703 return self.send_path(path)
705 def send_404(self):
706 self.send_error(404, "File not found")
707 return None
709 def send_path(self, path):
710 ctype = self.guess_type(path)
711 if ctype.startswith('text/'):
712 # Patch file instead
713 return self.send_patched_file(path, ctype)
714 else:
715 mode = 'rb'
716 try:
717 f = open(path, mode)
718 except IOError:
719 return self.send_404()
720 return self.send_file(f, ctype)
722 def send_file(self, f, ctype):
723 # Patch files to add links, but skip binary files.
724 self.send_response(200)
725 self.send_header("Content-type", ctype)
726 fs = os.fstat(f.fileno())
727 self.send_header("Content-Length", str(fs[6]))
728 self.send_header("Last-Modified", self.date_time_string(fs.st_mtime))
729 self.end_headers()
730 return f
732 def send_string(self, s, ctype='text/html', headers=True, mtime=None):
733 if headers:
734 self.send_response(200)
735 self.send_header("Content-type", ctype)
736 self.send_header("Content-Length", str(len(s)))
737 if mtime is None:
738 mtime = self.dynamic_mtime
739 self.send_header("Last-Modified", self.date_time_string(mtime))
740 self.end_headers()
741 return StringIO.StringIO(s)
743 def send_patched_file(self, path, ctype):
744 # Allow a very limited set of variables. This is pretty gross.
745 variables = {}
746 variables['report'] = ''
747 m = kReportFileRE.match(path)
748 if m:
749 variables['report'] = m.group(2)
751 try:
752 f = open(path,'r')
753 except IOError:
754 return self.send_404()
755 fs = os.fstat(f.fileno())
756 data = f.read()
757 for a,b in kReportReplacements:
758 data = a.sub(b % variables, data)
759 return self.send_string(data, ctype, mtime=fs.st_mtime)
762 def create_server(address, options, root):
763 import Reporter
765 reporters = Reporter.getReporters()
767 return ScanViewServer(address, ScanViewRequestHandler,
768 root,
769 reporters,
770 options)