4 CmpRuns - A simple tool for comparing two static analyzer runs to determine
5 which reports have been added, removed, or changed.
7 This is designed to support automated testing using the static analyzer, from
9 1. To monitor changes in the static analyzer's reports on real code bases, for
12 2. For use by end users who want to integrate regular static analyzer testing
13 into a buildbot like environment.
22 def __init__(self
, elts
=()):
24 for key
,value
in elts
:
27 def __getitem__(self
, item
):
28 return self
.data
[item
]
29 def __setitem__(self
, key
, value
):
31 self
.data
[key
].append(value
)
33 self
.data
[key
] = [value
]
35 return self
.data
.items()
37 return self
.data
.values()
39 return self
.data
.keys()
42 def get(self
, key
, default
=None):
43 return self
.data
.get(key
, default
)
48 def __init__(self
, run
, files
):
52 class AnalysisDiagnostic
:
53 def __init__(self
, data
, report
, htmlReport
):
56 self
.htmlReport
= htmlReport
58 def getReadableName(self
):
59 loc
= self
.data
['location']
60 filename
= self
.report
.run
.getSourceName(self
.report
.files
[loc
['file']])
64 # FIXME: Get a report number based on this key, to 'distinguish'
65 # reports, or something.
67 return '%s:%d:%d' % (filename
, line
, column
)
69 def getReportData(self
):
70 if self
.htmlReport
is None:
71 return "This diagnostic does not have any report data."
73 return open(os
.path
.join(self
.report
.run
.path
,
74 self
.htmlReport
), "rb").read()
77 def __init__(self
, path
, opts
):
83 def getSourceName(self
, path
):
84 if path
.startswith(self
.opts
.root
):
85 return path
[len(self
.opts
.root
):]
88 def loadResults(path
, opts
):
89 run
= AnalysisRun(path
, opts
)
91 for f
in os
.listdir(path
):
92 if (not f
.startswith('report') or
93 not f
.endswith('plist')):
96 p
= os
.path
.join(path
, f
)
97 data
= plistlib
.readPlist(p
)
99 # Ignore empty reports.
100 if not data
['files']:
103 # Extract the HTML reports, if they exists.
104 if 'HTMLDiagnostics_files' in data
['diagnostics'][0]:
106 for d
in data
['diagnostics']:
107 # FIXME: Why is this named files, when does it have multiple
109 assert len(d
['HTMLDiagnostics_files']) == 1
110 htmlFiles
.append(d
.pop('HTMLDiagnostics_files')[0])
112 htmlFiles
= [None] * len(data
['diagnostics'])
114 report
= AnalysisReport(run
, data
.pop('files'))
115 diagnostics
= [AnalysisDiagnostic(d
, report
, h
)
116 for d
,h
in zip(data
.pop('diagnostics'),
121 run
.reports
.append(report
)
122 run
.diagnostics
.extend(diagnostics
)
126 def compareResults(A
, B
):
128 compareResults - Generate a relation from diagnostics in run A to
129 diagnostics in run B.
131 The result is the relation as a list of triples (a, b, confidence) where
132 each element {a,b} is None or an element from the respective run, and
133 confidence is a measure of the match quality (where 0 indicates equality,
134 and None is used if either element is None).
139 # Quickly eliminate equal elements.
142 eltsA
= list(A
.diagnostics
)
143 eltsB
= list(B
.diagnostics
)
144 eltsA
.sort(key
= lambda d
: d
.data
)
145 eltsB
.sort(key
= lambda d
: d
.data
)
146 while eltsA
and eltsB
:
150 res
.append((a
, b
, 0))
151 elif a
.data
> b
.data
:
160 # FIXME: Add fuzzy matching. One simple and possible effective idea would be
161 # to bin the diagnostics, print them in a normalized form (based solely on
162 # the structure of the diagnostic), compute the diff, then use that as the
163 # basis for matching. This has the nice property that we don't depend in any
164 # way on the diagnostic format.
167 res
.append((a
, None, None))
169 res
.append((None, b
, None))
174 from optparse
import OptionParser
175 parser
= OptionParser("usage: %prog [options] [dir A] [dir B]")
176 parser
.add_option("", "--root", dest
="root",
177 help="Prefix to ignore on source files",
178 action
="store", type=str, default
="")
179 parser
.add_option("", "--verbose-log", dest
="verboseLog",
180 help="Write additional information to LOG [default=None]",
181 action
="store", type=str, default
=None,
183 (opts
, args
) = parser
.parse_args()
186 parser
.error("invalid number of arguments")
190 # Load the run results.
191 resultsA
= loadResults(dirA
, opts
)
192 resultsB
= loadResults(dirB
, opts
)
194 # Open the verbose log, if given.
196 auxLog
= open(opts
.verboseLog
, "wb")
200 diff
= compareResults(resultsA
, resultsB
)
204 print "ADDED: %r" % b
.getReadableName()
206 print >>auxLog
, ("('ADDED', %r, %r)" % (b
.getReadableName(),
209 print "REMOVED: %r" % a
.getReadableName()
211 print >>auxLog
, ("('REMOVED', %r, %r)" % (a
.getReadableName(),
214 print "CHANGED: %r to %r" % (a
.getReadableName(),
217 print >>auxLog
, ("('CHANGED', %r, %r, %r, %r)"
218 % (a
.getReadableName(),
225 print "TOTAL REPORTS: %r" % len(resultsB
.diagnostics
)
227 print >>auxLog
, "('TOTAL', %r)" % len(resultsB
.diagnostics
)
229 if __name__
== '__main__':