3 # git log --pretty="%H %P" | this program
4 # See option descriptions at bottom
6 # This little program cranks through a series of patches, trying to determine
7 # which trees each flowed through on its way to the mainline. It does a
8 # 'git describe' on each, so don't expect it to be fast for large numbers
11 # One warning: it is easily confused by local branches, tags, etc. For
12 # best results, run it on a mainline tree with no added frobs. Using
13 # "git clone --reference" is a relatively easy way to come up with such
14 # a tree without redownloading the whole mess.
16 import sys, subprocess, argparse, pickle
20 Mergepat = patterns.patterns['ExtMerge']
21 IntMerge = patterns.patterns['IntMerge']
22 IntMerge2 = patterns.patterns['IntMerge2']
26 def __init__(self, id, tree = None, signed = False):
30 self.tree = tree or '?'
37 def normalize_tree(self, tree):
38 colonslash = tree.find('://')
40 tree = tree[colonslash+3:]
41 if tree.find('git.kernel.org') >= 0:
42 stree = tree.split('/')
43 return '$KORG/%s/%s' % (stree[-2], stree[-1])
47 command = ['git', 'log', '-1', '--show-signature', self.id]
48 p = subprocess.run(command, cwd = Repo, capture_output = True,
51 # Sometimes we don't match a pattern; that means that the
52 # committer radically modified the merge message. A certain
53 # Eric makes them look like ordinary commits... Others use
54 # it to justify backmerges of the mainline. Either way, the
55 # best response is to treat it like an internal merge.
58 for line in p.stdout.split('\n'):
60 # Note if there's a GPG signature
62 if line.startswith('gpg:'):
66 # Maybe it's a merge of an external tree.
68 m = Mergepat.search(line)
70 self.tree = self.normalize_tree(m.group(3))
74 # Or maybe it's an internal merge.
76 m = IntMerge.search(line) or IntMerge2.search(line)
81 def add_commit(self, id):
82 self.commits.append(id)
84 def add_merge(self, merge):
85 self.merges.append(merge)
88 # Read the list of commits from the input stream and find which
89 # merge brought in each.
91 def ingest_commits(src):
93 expected = 'nothing yet'
94 for line in src.readlines():
95 sline = line[:-1].split()
97 is_merge = (len(sline) > 2)
98 if (commit == expected) and not is_merge:
101 mc = Mergelist[find_merge(sline[0])] # Needs try
103 mc.add_merge(Merge(commit))
105 mc.add_commit(commit)
107 if (count % 50) == 0:
108 sys.stderr.write('\r%5d ' % (count))
114 # Figure out which merge brought in a commit.
118 def find_merge(commit):
119 command = ['git', 'describe', '--contains', commit]
120 p = subprocess.run(command, cwd = Repo, capture_output = True,
122 desc = p.stdout.strip()
124 # The description line has the form:
128 # the portion up to the last ^ describes the merge we are after;
129 # in the absence of an ^, assume it's on the main branch.
131 uparrow = desc.rfind('^')
135 # OK, now get the real commit ID of the merge. Maybe we have
139 return MergeIDs[desc[:uparrow]]
143 # Nope, we have to dig it out the hard way.
145 command = ['git', 'log', '--pretty=%H', '-1', desc[:uparrow]]
146 p = subprocess.run(command, cwd = Repo, capture_output = True,
148 merge = p.stdout.strip()
150 # If we get back the same commit, we're looking at one of Linus's
151 # version number tags.
155 MergeIDs[desc[:uparrow]] = merge
159 # Internal merges aren't interesting from our point of view. So go through,
160 # find them all, and move any commits from such into the parent.
162 def zorch_internals(merge):
164 for m in merge.merges:
167 merge.commits += m.commits
168 new_merges += m.merges
171 merge.merges = new_merges
174 # Figure out how many commits flowed at each stage.
176 def count_commits(merge):
177 merge.ccount = len(merge.commits) + 1 # +1 to count the merge itself
178 for m in merge.merges:
179 merge.ccount += count_commits(m)
183 # ...and how many flowed between each pair of trees
188 def tree_stats(merge):
189 SignedTrees.add('mainline')
191 tcount = Treecounts[merge.tree]
193 tcount = Treecounts[merge.tree] = { }
194 for m in merge.merges:
196 SignedTrees.add(m.tree)
197 mcount = tcount.get(m.tree, 0)
198 tcount[m.tree] = mcount + m.ccount
202 # Maybe we only want so many top-level trees
204 def trim_trees(limit):
205 srcs = Treecounts['mainline']
206 srcnames = srcs.keys()
207 srcnames = sorted(srcnames, key = lambda i: srcs[i], reverse = True)
208 nextra = len(srcnames) - limit
210 for extra in srcnames[limit:]:
211 zapped += srcs[extra]
213 srcs['%d other trees' % (nextra)] = zapped
215 # Take our map of the commit structure and boil it down to how many commits
216 # moved from one tree to the next.
219 def dumptree(start, indent = ''):
223 print('%s%s%s: %d/%d %s' % (indent, int, start.id[:10],
224 len(start.merges), len(start.commits),
226 for merge in start.merges:
227 dumptree(merge, indent + ' ')
229 def dumpflow(tree, indent = '', seen = []):
231 srcs = Treecounts[tree]
234 srctrees = sorted(srcs.keys(), key = lambda i: srcs[i], reverse = True)
237 print('Skip', src, srcs[src], seen)
239 if src in SignedTrees:
240 print('%s%4d ** %s' % (indent, srcs[src], src))
242 print('%s%4d %s' % (indent, srcs[src], src))
243 dumpflow(src, indent = indent + ' ', seen = seen + [tree])
245 def totalsignedstats(tree, signed = [ ], unsigned = [ ]):
247 srcs = Treecounts[tree]
250 for src in sorted(srcs.keys(), key = lambda i: srcs[i], reverse = True):
251 if (src in signed) or (src in unsigned):
253 if src in SignedTrees:
257 return 'stopped here'
260 srcs = Treecounts[tree]
261 spulls = upulls = scommits = ucommits = 0
262 for src in srcs.keys():
263 if src in SignedTrees:
265 scommits += srcs[src]
268 ucommits += srcs[src]
269 print('%d repos total, %d signed, %d unsigned' % (spulls + upulls,
271 print(' %d commits from signed, %d from unsigned' % (scommits, ucommits))
273 print(len(SignedTrees), len(Treecounts.keys()))
279 graph = graphviz.Digraph('mainline', filename = file, format = 'svg')
280 graph.body.extend(['label="Patch flow into the mainline"',
284 graph.body.extend([' layout = twopi; graph [ranksep=10]',
286 graph.attr('node', fontsize="20", color="blue", penwidth='4',
288 graph.node('mainline')
289 graph.attr('node', fontsize="14", color="black", shape='polygon',
292 GV_out_node_signed(graph, 'mainline')
294 GV_out_node(graph, 'mainline')
297 def GV_fixname(name):
298 return name.replace(':', '/') # or Graphviz chokes
301 if count >= RedThresh:
303 if count >= YellowThresh:
308 # Output nodes with traffic coloring
310 def GV_out_node(graph, node, seen = []):
312 srcs = Treecounts[node]
313 except KeyError: # "applied by linus"
315 srctrees = sorted(srcs.keys(), key = lambda i: srcs[i], reverse = True)
318 graph.edge(GV_fixname(src), GV_fixname(node),
319 taillabel='%d' % srcs[src], labelfontsize="14",
320 color = GV_color(srcs[src]), penwidth='2')
321 GV_out_node(graph, src, seen + [node])
324 # Output nodes showing signature status
326 def GV_out_node_signed(graph, node, seen = []):
328 srcs = Treecounts[node]
329 except KeyError: # "applied by linus"
331 srctrees = sorted(srcs.keys(), key = lambda i: srcs[i], reverse = True)
334 if src in SignedTrees:
337 graph.attr('node', color=color)
338 graph.edge(GV_fixname(src), GV_fixname(node),
339 taillabel='%d' % srcs[src], labelfontsize="14",
340 color = color, penwidth='2')
341 GV_out_node_signed(graph, src, seen + [node])
343 # argument parsing stuff.
346 p = argparse.ArgumentParser()
347 p.add_argument('-d', '--dump', help = 'Dump merge list to file',
348 required = False, default = '')
349 p.add_argument('-g', '--gvoutput', help = 'Graphviz output',
350 required = False, default = '')
351 p.add_argument('-l', '--load', help = 'Load merge list from file',
352 required = False, default = '')
353 p.add_argument('-o', '--output', help = 'Output file',
354 required = False, default = '-')
355 p.add_argument('-r', '--repo', help = 'Repository location',
356 required = False, default = '/home/corbet/kernel')
357 p.add_argument('-t', '--trim', help = 'Trim top level to this many trees',
358 required = False, default = 0, type = int)
359 p.add_argument('-R', '--red', help = 'Red color threshold',
360 required = False, default = 800, type = int)
361 p.add_argument('-Y', '--yellow', help = 'Yellow color threshold',
362 required = False, default = 200, type = int)
363 p.add_argument('-s', '--signed', help = 'Display signed trees',
364 action='store_true', default = False)
365 p.add_argument('-T', '--twopi', help = 'Do a twopi plot',
366 action = 'store_true', default = False)
371 args = p.parse_args()
374 YellowThresh = args.yellow
375 DoSigned = args.signed
380 dumpfile = open(args.load, 'rb')
381 Mergelist = pickle.loads(dumpfile.read())
383 Mainline = Mergelist['mainline']
385 Mainline = Merge('mainline', tree = 'mainline', signed = True)
386 ingest_commits(sys.stdin)
388 dumpfile = open(args.dump, 'wb')
389 dumpfile.write(pickle.dumps(Mergelist))
392 # Now generate the flow graph.
395 zorch_internals(Mainline)
397 Treecounts['mainline'] = { 'Applied by Linus': len(Mainline.commits) }
398 print('total commits', count_commits(Mainline))
401 trim_trees(args.trim)
405 GV_out(args.gvoutput)