database.py

   1 #
   2 # The "database".
   3 #
   4 # This code is part of the LWN git data miner.
   5 #
   6 # Copyright 2007-11 Eklektix, Inc.
   7 # Copyright 2007-11 Jonathan Corbet <corbet@lwn.net>
   8 #
   9 # This file may be distributed under the terms of the GNU General
  10 # Public License, version 2.
  11 #
  12 import sys, datetime
  13
  14
  15 class Hacker:
  16     def __init__ (self, name, id, elist, email):
  17         self.name = name
  18         self.id = id
  19         self.employer = [ elist ]
  20         self.email = [ email ]
  21         self.changed = self.added = self.removed = 0
  22         self.patches = [ ]
  23         self.signoffs = [ ]
  24         self.reviews = [ ]
  25         self.tested = [ ]
  26         self.reports = [ ]
  27         self.testcred = self.repcred = 0
  28         self.versions = [ ]
  29
  30     def addemail (self, email, elist):
  31         self.email.append (email)
  32         self.employer.append (elist)
  33         HackersByEmail[email] = self
  34
  35     def emailemployer (self, email, date):
  36         for i in range (0, len (self.email)):
  37             if self.email[i] == email:
  38                 for edate, empl in self.employer[i]:
  39                     if edate > date:
  40                         return empl
  41         print 'OOPS.  ', self.name, self.employer, self.email, email, date
  42         return None # Should not happen
  43
  44     def addpatch (self, patch):
  45         self.added += patch.added
  46         self.removed += patch.removed
  47         self.changed += max(patch.added, patch.removed)
  48         self.patches.append (patch)
  49
  50     #
  51     # Note that the author is represented in this release.
  52     #
  53     def addversion (self, release):
  54         if release not in self.versions:
  55             self.versions.append (release)
  56     #
  57     # There's got to be a better way.
  58     #
  59     def addsob (self, patch):
  60         self.signoffs.append (patch)
  61     def addreview (self, patch):
  62         self.reviews.append (patch)
  63     def addtested (self, patch):
  64         self.tested.append (patch)
  65     def addreport (self, patch):
  66         self.reports.append (patch)
  67
  68     def reportcredit (self, patch):
  69         self.repcred += 1
  70     def testcredit (self, patch):
  71         self.testcred += 1
  72
  73 HackersByName = { }
  74 HackersByEmail = { }
  75 HackersByID = { }
  76 MaxID = 0
  77
  78 def StoreHacker (name, elist, email):
  79     global MaxID
  80
  81     id = MaxID
  82     MaxID += 1
  83     h = Hacker (name, id, elist, email)
  84     HackersByName[name] = h
  85     HackersByEmail[email] = h
  86     HackersByID[id] = h
  87     return h
  88
  89 def LookupEmail (addr):
  90     try:
  91         return HackersByEmail[addr]
  92     except KeyError:
  93         return None
  94
  95 def LookupName (name):
  96     try:
  97         return HackersByName[name]
  98     except KeyError:
  99         return None
 100
 101 def LookupID (id):
 102     try:
 103         return HackersByID[id]
 104     except KeyError:
 105         return None
 106
 107 def LookupStoreHacker(name, email, mapunknown = True):
 108     email = RemapEmail(email)
 109     h = LookupEmail(email)
 110     if h: # already there
 111         return h
 112     elist = LookupEmployer(email, mapunknown)
 113     h = LookupName(name)
 114     if h: # new email
 115         h.addemail(email, elist)
 116         return h
 117     return StoreHacker(name, elist, email)
 118
 119
 120 def AllHackers ():
 121     return HackersByID.values ()
 122
 123 def DumpDB ():
 124     out = open ('database.dump', 'w')
 125     names = HackersByName.keys ()
 126     names.sort ()
 127     for name in names:
 128         h = HackersByName[name]
 129         out.write ('%4d %s %d p (+%d -%d) sob: %d\n' % (h.id, h.name,
 130                                                         len (h.patches),
 131                                                         h.added, h.removed,
 132                                                         len (h.signoffs)))
 133         for i in range (0, len (h.email)):
 134             out.write ('\t%s -> \n' % (h.email[i]))
 135             for date, empl in h.employer[i]:
 136                 out.write ('\t\t %d-%d-%d %s\n' % (date.year, date.month, date.day,
 137                                                  empl.name))
 138         if h.versions:
 139             out.write ('\tVersions: %s\n' % ','.join (h.versions))
 140
 141 #
 142 # Hack: The first visible tag comes a ways into the stream; when we see it,
 143 # push it backward through the changes we've already seen.
 144 #
 145 def ApplyFirstTag (tag):
 146     for n in HackersByName.keys ():
 147         if HackersByName[n].versions:
 148             HackersByName[n].versions = [tag]
 149
 150 #
 151 # Employer info.
 152 #
 153 class Employer:
 154     def __init__ (self, name):
 155         self.name = name
 156         self.added = self.removed = self.count = self.changed = 0
 157         self.sobs = 0
 158         self.hackers = [ ]
 159
 160     def AddCSet (self, patch):
 161         self.added += patch.added
 162         self.removed += patch.removed
 163         self.changed += max(patch.added, patch.removed)
 164         self.count += 1
 165         if patch.author not in self.hackers:
 166             self.hackers.append (patch.author)
 167
 168     def AddSOB (self):
 169         self.sobs += 1
 170
 171 Employers = { }
 172
 173 def GetEmployer (name):
 174     try:
 175         return Employers[name]
 176     except KeyError:
 177         e = Employer (name)
 178         Employers[name] = e
 179         return e
 180
 181 def AllEmployers ():
 182     return Employers.values ()
 183
 184 #
 185 # Certain obnoxious developers, who will remain nameless (because we
 186 # would never want to run afoul of Thomas) want their work split among
 187 # multiple companies.  Let's try to cope with that.  Let's also hope
 188 # this doesn't spread.
 189 #
 190 class VirtualEmployer (Employer):
 191     def __init__ (self, name):
 192         Employer.__init__ (self, name)
 193         self.splits = [ ]
 194
 195     def addsplit (self, name, fraction):
 196         self.splits.append ((name, fraction))
 197
 198     #
 199     # Go through and (destructively) apply our credits to the
 200     # real employer.  Only one level of weirdness is supported.
 201     #
 202     def applysplits (self):
 203         for name, fraction in self.splits:
 204             real = GetEmployer (name)
 205             real.added += int (self.added*fraction)
 206             real.removed += int (self.removed*fraction)
 207             real.changed += int (self.changed*fraction)
 208             real.count += int (self.count*fraction)
 209         self.__init__ (name) # Reset counts just in case
 210
 211     def store (self):
 212         if Employers.has_key (self.name):
 213             print Employers[self.name]
 214             sys.stderr.write ('WARNING: Virtual empl %s overwrites another\n'
 215                               % (self.name))
 216         if len (self.splits) == 0:
 217             sys.stderr.write ('WARNING: Virtual empl %s has no splits\n'
 218                               % (self.name))
 219             # Should check that they add up too, but I'm lazy
 220         Employers[self.name] = self
 221
 222 class FileType:
 223     def __init__ (self, patterns={}, order=[]):
 224         self.patterns = patterns
 225         self.order = order
 226
 227     def guess_file_type (self, filename, patterns=None, order=None):
 228         patterns = patterns or self.patterns
 229         order = order or self.order
 230
 231         for file_type in order:
 232             if patterns.has_key (file_type):
 233                 for patt in patterns[file_type]:
 234                     if patt.search (filename):
 235                         return file_type
 236
 237         return 'unknown'
 238
 239 #
 240 # By default we recognize nothing.
 241 #
 242 FileTypes = FileType ({}, [])
 243
 244 #
 245 # Mix all the virtual employers into their real destinations.
 246 #
 247 def MixVirtuals ():
 248     for empl in AllEmployers ():
 249         if isinstance (empl, VirtualEmployer):
 250             empl.applysplits ()
 251
 252 #
 253 # The email map.
 254 #
 255 EmailAliases = { }
 256
 257 def AddEmailAlias (variant, canonical):
 258     if EmailAliases.has_key (variant):
 259         sys.stderr.write ('Duplicate email alias for %s\n' % (variant))
 260     EmailAliases[variant] = canonical
 261
 262 def RemapEmail (email):
 263     email = email.lower ()
 264     try:
 265         return EmailAliases[email]
 266     except KeyError:
 267         return email
 268
 269 #
 270 # Email-to-employer mapping.
 271 #
 272 EmailToEmployer = { }
 273 nextyear = datetime.date.today () + datetime.timedelta (days = 365)
 274
 275 def AddEmailEmployerMapping (email, employer, end = nextyear):
 276     if end is None:
 277         end = nextyear
 278     email = email.lower ()
 279     empl = GetEmployer (employer)
 280     try:
 281         l = EmailToEmployer[email]
 282         for i in range (0, len(l)):
 283             date, xempl = l[i]
 284             if date == end:  # probably both nextyear
 285                 print 'WARNING: duplicate email/empl for %s' % (email)
 286             if date > end:
 287                 l.insert (i, (end, empl))
 288                 return
 289         l.append ((end, empl))
 290     except KeyError:
 291         EmailToEmployer[email] = [(end, empl)]
 292
 293 def MapToEmployer (email, unknown = 0):
 294     # Somebody sometimes does s/@/ at /; let's fix it.
 295     email = email.lower ().replace (' at ', '@')
 296     try:
 297         return EmailToEmployer[email]
 298     except KeyError:
 299         pass
 300     namedom = email.split ('@')
 301     if len (namedom) < 2:
 302         print 'Oops...funky email %s' % email
 303         return [(nextyear, GetEmployer ('Funky'))]
 304     s = namedom[1].split ('.')
 305     for dots in range (len (s) - 2, -1, -1):
 306         addr = '.'.join (s[dots:])
 307         try:
 308             return EmailToEmployer[addr]
 309         except KeyError:
 310             pass
 311     #
 312     # We don't know who they work for.
 313     #
 314     if unknown:
 315         return [(nextyear, GetEmployer ('(Unknown)'))]
 316     return [(nextyear, GetEmployer (email))]
 317
 318
 319 def LookupEmployer (email, mapunknown = 0):
 320     elist = MapToEmployer (email, mapunknown)
 321     return elist # GetEmployer (ename)
 322