5 # This code is part of the LWN git data miner.
7 # Copyright 2007 LWN.net
8 # Copyright 2007 Jonathan Corbet <corbet@lwn.net>
10 # This file may be distributed under the terms of the GNU General
11 # Public License, version 2.
14 import database
, ConfigFile
15 import getopt
, datetime
16 import os
, re
, sys
, rfc822
, string
19 # Some people, when confronted with a problem, think "I know, I'll use regular
20 # expressions." Now they have two problems.
23 Pcommit
= re
.compile (r
'^commit ([0-9a-f]+)$')
24 Pauthor
= re
.compile (r
'^Author: ([^<]+)\s<([^>]+)>$')
25 Psob
= re
.compile (r
'Signed-off-by:\s+([^<]+)\s+<([^>]+)>')
26 Pmerge
= re
.compile (r
'^Merge:.*$')
27 Padd
= re
.compile (r
'^\+[^\+].*$')
28 Prem
= re
.compile (r
'^-[^-].*$')
29 Pdate
= re
.compile (r
'^(Commit)?Date:\s+(.*)$')
30 Pfilea
= re
.compile (r
'^---\s+(.*)$')
31 Pfileb
= re
.compile (r
'^\+\+\+\s+(.*)$')
49 CFName
= 'gitdm.config'
53 # -a Andrew Morton's signoffs shadow Linus's
54 # -c cfile Specify a configuration file
55 # -d Output individual developer stats
56 # -D Output date statistics
57 # -h hfile HTML output to hfile
58 # -l count Maximum length for output lists
59 # -o file File for text output
60 # -r pattern Restrict to files matching pattern
61 # -s Ignore author SOB lines
62 # -u Map unknown employers to '(Unknown)'
63 # -z Dump out the hacker database at completion
66 global Outfile
, ListCount
, MapUnknown
, HTMLfile
, DevReports
67 global DateStats
, AuthorSOBs
, FileFilter
, AkpmOverLt
, DumpDB
70 opts
, rest
= getopt
.getopt (sys
.argv
[1:], 'adDh:l:o:r:suz')
81 HTMLfile
= open (opt
[1], 'w')
83 ListCount
= int (opt
[1])
85 Outfile
= open (opt
[1], 'w')
87 print 'Filter on "%s"' % (opt
[1])
88 FileFilter
= re
.compile (opt
[1])
98 def LookupStoreHacker (name
, email
):
99 email
= database
.RemapEmail (email
)
100 h
= database
.LookupEmail (email
)
101 if h
: # already there
103 elist
= database
.LookupEmployer (email
, MapUnknown
)
104 h
= database
.LookupName (name
)
106 h
.addemail (email
, elist
)
108 return database
.StoreHacker(name
, elist
, email
)
116 def AddDateLines(date
, lines
):
118 print 'Skip big patch (%d)' % lines
120 dt
= (date
.year
, date
.month
, date
.day
)
122 DateMap
[date
] += lines
124 DateMap
[date
] = lines
126 def PrintDateStats():
127 dates
= DateMap
.keys ()
130 datef
= open ('datelc', 'w')
132 total
+= DateMap
[date
]
133 datef
.write ('%d/%02d/%02d %6d %7d\n' % (date
[0], date
[1], date
[2],
134 DateMap
[date
], total
))
137 # The core hack for grabbing the information about a changeset.
140 global NextLine
, TotalAdded
, TotalRemoved
, TotalChanged
143 m
= Pcommit
.match (NextLine
)
146 NextLine
= sys
.stdin
.readline ()
151 p
.commit
= m
.group (1)
152 p
.merge
= p
.added
= p
.removed
= 0
153 p
.author
= LookupStoreHacker('Unknown hacker', 'unknown@hacker.net')
154 p
.email
= 'unknown@hacker.net'
156 NextLine
= sys
.stdin
.readline ()
157 ignore
= (FileFilter
is not None)
161 # If this line starts a new commit, drop out.
163 m
= Pcommit
.match (Line
)
166 NextLine
= sys
.stdin
.readline ()
168 # Maybe it's an author line?
170 m
= Pauthor
.match (Line
)
172 p
.email
= database
.RemapEmail (m
.group (2))
173 p
.author
= LookupStoreHacker(m
.group (1), p
.email
)
176 # Could be a signed-off-by:
178 m
= Psob
.search (Line
)
180 email
= database
.RemapEmail (m
.group (2))
181 sobber
= LookupStoreHacker(m
.group (1), email
)
182 if sobber
!= p
.author
or AuthorSOBs
:
183 p
.sobs
.append ((email
, LookupStoreHacker(m
.group (1), m
.group (2))))
186 # If this one is a merge, make note of the fact.
188 m
= Pmerge
.match (Line
)
193 # See if it's the date.
195 m
= Pdate
.match (Line
)
197 dt
= rfc822
.parsedate(m
.group (2))
198 p
.date
= datetime
.date (dt
[0], dt
[1], dt
[2])
201 # If we have a file filter, check for file lines.
204 ignore
= ApplyFileFilter (Line
, ignore
)
206 # OK, maybe it's part of the diff itself.
209 if Padd
.match (Line
):
212 if Prem
.match (Line
):
215 # Record some global information - but only if this patch had
216 # stuff which wasn't ignored. This work should be done
219 if ((p
.added
+ p
.removed
) > 0 or not FileFilter
) and not p
.merge
:
220 TotalAdded
+= p
.added
221 TotalRemoved
+= p
.removed
222 TotalChanged
+= max (p
.added
, p
.removed
)
223 AddDateLines (p
.date
, max (p
.added
, p
.removed
))
224 empl
= p
.author
.emailemployer (p
.email
, p
.date
)
228 for sobemail
, sobber
in p
.sobs
:
229 empl
= sobber
.emailemployer (sobemail
, p
.date
)
234 def ApplyFileFilter (line
, ignore
):
236 # If this is the first file line (--- a/), set ignore one way
239 m
= Pfilea
.match (line
)
242 if FileFilter
.search (file):
246 # For the second line, we can turn ignore off, but not on
248 m
= Pfileb
.match (line
)
251 if FileFilter
.search (file):
256 # If this patch is signed off by both Andrew Morton and Linus Torvalds,
257 # remove the (redundant) Linus signoff.
260 if Linus
in p
.sobs
and Akpm
in p
.sobs
:
261 p
.sobs
.remove (Linus
)
264 # HTML output support stuff.
268 HClasses
= ['Even', 'Odd']
271 <table cellspacing=3>
272 <tr><th colspan=3>%s</th></tr>
276 def BeginReport (title
):
279 Outfile
.write ('\n%s\n' % title
)
281 HTMLfile
.write (THead
% title
)
284 TRow
= ''' <tr class="%s">
285 <td>%s</td><td align="right">%d</td><td align="right">%.1f%%</td></tr>
288 def ReportLine (text
, count
, pct
):
292 Outfile
.write ('%-25s %4d (%.1f%%)\n' % (text
, count
, pct
))
294 HTMLfile
.write (TRow
% (HClasses
[HTMLclass
], text
, count
, pct
))
299 HTMLfile
.write ('</table>\n\n')
302 # Comparison and report generation functions.
304 def ComparePCount (h1
, h2
):
305 return len (h2
.patches
) - len (h1
.patches
)
307 def ReportByPCount (hlist
):
308 hlist
.sort (ComparePCount
)
310 BeginReport ('Developers with the most changesets')
312 pcount
= len (h
.patches
)
313 changed
= max(h
.added
, h
.removed
)
314 delta
= h
.added
- h
.removed
316 ReportLine (h
.name
, pcount
, (pcount
*100.0)/CSCount
)
318 if count
>= ListCount
:
322 def CompareLChanged (h1
, h2
):
323 return max(h2
.added
, h2
.removed
) - max(h1
.added
, h1
.removed
)
325 def ReportByLChanged (hlist
):
326 hlist
.sort (CompareLChanged
)
328 BeginReport ('Developers with the most changed lines')
330 pcount
= len (h
.patches
)
331 changed
= max(h
.added
, h
.removed
)
332 delta
= h
.added
- h
.removed
333 if (h
.added
+ h
.removed
) > 0:
334 ReportLine (h
.name
, changed
, (changed
*100.0)/TotalChanged
)
336 if count
>= ListCount
:
340 def CompareLRemoved (h1
, h2
):
341 return (h2
.removed
- h2
.added
) - (h1
.removed
- h1
.added
)
343 def ReportByLRemoved (hlist
):
344 hlist
.sort (CompareLRemoved
)
346 BeginReport ('Developers with the most lines removed')
348 pcount
= len (h
.patches
)
349 changed
= max(h
.added
, h
.removed
)
350 delta
= h
.added
- h
.removed
352 ReportLine (h
.name
, -delta
, (-delta
*100.0)/TotalRemoved
)
354 if count
>= ListCount
:
358 def CompareEPCount (e1
, e2
):
359 return e2
.count
- e1
.count
361 def ReportByPCEmpl (elist
):
362 elist
.sort (CompareEPCount
)
364 BeginReport ('Top changeset contributors by employer')
367 ReportLine (e
.name
, e
.count
, (e
.count
*100.0)/CSCount
)
369 if count
>= ListCount
:
375 def CompareELChanged (e1
, e2
):
376 return e2
.changed
- e1
.changed
378 def ReportByELChanged (elist
):
379 elist
.sort (CompareELChanged
)
381 BeginReport ('Top lines changed by employer')
384 ReportLine (e
.name
, e
.changed
, (e
.changed
*100.0)/TotalChanged
)
386 if count
>= ListCount
:
392 def CompareSOBs (h1
, h2
):
393 return len (h2
.signoffs
) - len (h1
.signoffs
)
395 def ReportBySOBs (hlist
):
396 hlist
.sort (CompareSOBs
)
399 totalsobs
+= len (h
.signoffs
)
401 BeginReport ('Developers with the most signoffs (total %d)' % totalsobs
)
403 scount
= len (h
.signoffs
)
405 ReportLine (h
.name
, scount
, (scount
*100.0)/totalsobs
)
407 if count
>= ListCount
:
411 def CompareESOBs (e1
, e2
):
412 return e2
.sobs
- e1
.sobs
414 def ReportByESOBs (elist
):
415 elist
.sort (CompareESOBs
)
420 BeginReport ('Employers with the most signoffs (total %d)' % totalsobs
)
423 ReportLine (e
.name
, e
.sobs
, (e
.sobs
*100.0)/totalsobs
)
425 if count
>= ListCount
:
430 # Here starts the real program. Read the config files.
432 ConfigFile
.ConfigFile (CFName
)
435 # Let's pre-seed the database with a couple of hackers
436 # we want to remember.
438 Linus
= ('torvalds@linux-foundation.org',
439 LookupStoreHacker ('Linus Torvalds', 'torvalds@linux-foundation.org'))
440 Akpm
= ('akpm@linux-foundation.org',
441 LookupStoreHacker ('Andrew Morton', 'akpm@linux-foundation.org'))
443 NextLine
= sys
.stdin
.readline ()
444 TotalChanged
= TotalAdded
= TotalRemoved
= 0
450 print >> sys
.stderr
, 'Grabbing changesets...\r',
452 printcount
= CSCount
= 0
454 if (printcount
% 50) == 0:
455 print >> sys
.stderr
, 'Grabbing changesets...%d\r' % printcount
,
460 if p
.added
> 100000 or p
.removed
> 100000:
461 print 'Skipping massive add'
463 if FileFilter
and p
.added
== 0 and p
.removed
== 0:
466 p
.author
.addpatch (p
)
467 for sobemail
, sob
in p
.sobs
:
470 print >> sys
.stderr
, 'Grabbing changesets...done'
477 hlist
= database
.AllHackers ()
478 elist
= database
.AllEmployers ()
479 Outfile
.write ('Processed %d csets from %d developers\n' % (CSCount
,
481 Outfile
.write ('%d employers found\n' % len (elist
))
482 Outfile
.write ('A total of %d lines added, %d removed (delta %d)\n' %
483 (TotalAdded
, TotalRemoved
, TotalAdded
- TotalRemoved
))
484 if TotalChanged
== 0:
485 TotalChanged
= 1 # HACK to avoid div by zero
491 ReportByPCount (hlist
)
492 ReportByLChanged (hlist
)
493 ReportByLRemoved (hlist
)
495 ReportByPCEmpl (elist
)
496 ReportByELChanged (elist
)
497 ReportByESOBs (elist
)