From dc870efd045c1409a04db388a05f027609a6a924 Mon Sep 17 00:00:00 2001 From: Fredrik Kuivinen Date: Sun, 4 Dec 2005 01:11:01 +0100 Subject: [PATCH] Make the patch generation lazy. This gives huge speed improvements on large repositories. Some git command renaming crept into this commit too. Signed-off-by: Fredrik Kuivinen --- ctcore.py | 31 ++++++++++++++++++++- git.py | 96 ++++++++++++++++++++++++++++++++++++++++++++++++--------------- hg.py | 26 +++++++++++------ main.py | 75 ++++++++++++++++++++++++++----------------------- 4 files changed, 160 insertions(+), 68 deletions(-) diff --git a/ctcore.py b/ctcore.py index eb80d63..a947b1f 100644 --- a/ctcore.py +++ b/ctcore.py @@ -13,7 +13,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import sys, locale, codecs +import sys, locale, codecs, sets applicationName = 'Commit Tool' shortName = 'ct' @@ -45,6 +45,35 @@ else: class File: def __init__(self): self.listViewItem = None + self.text = None + self.textW = None + self.patch = None + + def getPatch(self): + if not self.patch: + self.patch = self.getPatchImpl() + + return self.patch + +class FileSet: + def __init__(self, addCallback, removeCallback): + self.addCallback = addCallback + self.removeCallback = removeCallback + self.files = sets.Set() + + def add(self, file): + self.files.add(file) + self.addCallback(file) + + def remove(self, file): + self.files.discard(file) + self.removeCallback(file) + + def __iter__(self): + return self.files.__iter__() + + def __len__(self): + return len(self.files) class ProgramError(Exception): def __init__(self, progStr, error): diff --git a/git.py b/git.py index b2ffb12..bc8c794 100644 --- a/git.py +++ b/git.py @@ -13,8 +13,8 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import sys, os, re - +import sys, os, re, itertools +from sets import Set from ctcore import * def initialize(): @@ -34,7 +34,51 @@ def initialize(): files = runProgram(['git-diff-files', '--name-only', '-z']).split('\0') files.pop() runXargsStyle(['git-update-index', '--remove', '--'], files) - + +class GitFile(File): + def __init__(self, unknown=False): + File.__init__(self) + self.unknown = unknown + + def code(self): + '''Only defined for non-unknown files''' + return (self.dstName, self.dstSHA, self.dstMode) + + def getPatchImpl(self): + if self.unknown: + runProgram(['git-update-index', '--add', '--', self.srcName]) + patch = runProgram(['git-diff-index', '-p', '--cached', 'HEAD', '--', self.srcName]) + runProgram(['git-update-index', '--force-remove', '--', self.srcName]) + return patch + elif self.change == 'C' or self.change == 'R': + return getPatch(self.srcName, self.dstName) + else: + return getPatch(self.srcName) + +class GitFileSet(FileSet): + def __init__(self, addCallback, removeCallback): + FileSet.__init__(self, addCallback, removeCallback) + self.codeDict = {} + + def add(self, file): + if not file.unknown: + self.codeDict[file.code()] = file + FileSet.add(self, file) + + def remove(self, file): + if not file.unknown: + del self.codeDict[file.code()] + FileSet.remove(self, file) + + def getByCode(self, file): + if file.unknown: + return None + else: + return self.codeDict.get(file.code()) + +def fileSetFactory(addCallback, removeCallback): + return GitFileSet(addCallback, removeCallback) + parseDiffRE = re.compile(':([0-9]+) ([0-9]+) ([0-9a-f]{40}) ([0-9a-f]{40}) ([MCRNADUT])([0-9]*)') def parseDiff(prog): inp = runProgram(prog) @@ -51,7 +95,7 @@ def parseDiff(prog): print "Unknown output from " + str(prog) + "!: " + rec + "\n" continue - f = File() + f = GitFile() f.srcMode = m.group(1) f.dstMode = m.group(2) f.srcSHA = m.group(3) @@ -63,12 +107,6 @@ def parseDiff(prog): f.score = m.group(6) f.srcName = f.dstName = it.next() - if f.change == 'C' or f.change == 'R': - f.dstName = it.next() - f.patch = getPatch(f.srcName, f.dstName) - else: - f.patch = getPatch(f.srcName) - ret.append(f) except StopIteration: pass @@ -102,27 +140,39 @@ def getUnknownFiles(): fileObjects = [] - runXargsStyle(['git-update-index', '--add', '--'], files) for fileName in files: - f = File() + f = GitFile(unknown=True) f.srcName = f.dstName = fileName f.change = '?' - f.patch = runProgram(['git-diff-index', '-p', '--cached', 'HEAD', '--', fileName]) + fileObjects.append(f) f.text = 'New file: ' + fileName - runXargsStyle(['git-update-index', '--force-remove', '--'], files) return fileObjects # HEAD is src in the returned File objects. That is, srcName is the # name in HEAD and dstName is the name in the cache. -def getFiles(): +def updateFiles(fileSet): files = parseDiff('git-diff-files -z') for f in files: doUpdateCache(f.srcName) + markForDeletion = Set() + for f in fileSet: + markForDeletion.add(f) + + if settings().showUnknown: + unknowns = getUnknownFiles() + else: + unknowns = [] + files = parseDiff('git-diff-index -z -M --cached HEAD') - for f in files: + for f in itertools.chain(files, unknowns): + fs = fileSet.getByCode(f) + if fs: + markForDeletion.discard(fs) + continue + c = f.change if c == 'C': f.text = 'Copy from ' + f.srcName + ' to ' + f.dstName @@ -137,17 +187,17 @@ def getFiles(): else: f.text = f.srcName - if settings().showUnknown: - files.extend(getUnknownFiles()) + fileSet.add(f) - return files + for f in markForDeletion: + fileSet.remove(f) def getPatch(file, otherFile = None): if otherFile: f = [file, otherFile] else: f = [file] - return runProgram(['git-diff-index', '-p', '-M', '--cached', 'HEAD'] + f) + return runProgram(['git-diff-index', '-p', '-M', '--cached', 'HEAD', '--'] + f) def doUpdateCache(filename): runProgram(['git-update-index', '--remove', '--add', '--replace', '--', filename]) @@ -205,17 +255,17 @@ def discardFile(file): runProgram(['git-read-tree', 'HEAD']) c = file.change if c == 'M' or c == 'T': - runProgram(['git-checkout-cache', '-f', '-q', '--', file.dstName]) + runProgram(['git-checkout-index', '-f', '-q', '--', file.dstName]) elif c == 'A' or c == 'C': # The file won't be tracked by git now. We could unlink it # from the working directory, but that seems a little bit # too dangerous. pass elif c == 'D': - runProgram(['git-checkout-cache', '-f', '-q', '--', file.dstName]) + runProgram(['git-checkout-index', '-f', '-q', '--', file.dstName]) elif c == 'R': # Same comment applies here as to the 'A' or 'C' case. - runProgram(['git-checkout-cache', '-f', '-q', '--', file.srcName]) + runProgram(['git-checkout-index', '-f', '-q', '--', file.srcName]) def ignoreFile(file): ignoreExpr = re.sub(r'([][*?!\\])', r'\\\1', file.dstName) diff --git a/hg.py b/hg.py index ce8f85e..7192ee5 100644 --- a/hg.py +++ b/hg.py @@ -22,11 +22,20 @@ def doCommit(keepFiles, filesToCommit, msg): for f in filesToCommit: commitFileNames.append(f.dstName) runProgram(['hg', 'commit', '-A', '-l', '-'] + commitFileNames, msg) - + +class HgFile(File): + def __init__(self, unknown=False): + File.__init__(self) + + def getPatchImpl(self): + return getPatch(self) + +def fileSetFactory(addCallback, removeCallback): + return FileSet(addCallback, removeCallback) parseDiffRE = re.compile('([AMR?]) (.*)') -def __getPatch(file, otherFile = None): +def getPatch(file, otherFile = None): if file.change == 'M' or \ file.change == 'R' or \ file.change == 'A': @@ -52,7 +61,7 @@ def __parseStatus(): print "Unknown output from hg status!: " + rec + "\n" continue - f = File() + f = HgFile() f.change = m.group(1) if f.change == '?' and not settings().showUnknown: @@ -60,17 +69,16 @@ def __parseStatus(): f.srcName = f.dstName = m.group(2) - f.patch = __getPatch(f) - ret.append(f) except StopIteration: pass return ret -# HEAD is src in the returned File objects. That is, srcName is the -# name in HEAD and dstName is the name in the cache. -def getFiles(): +def updateFiles(fileSet): + for f in list([x for x in fileSet]): + fileSet.remove(f) + files = __parseStatus() for f in files: c = f.change @@ -83,7 +91,7 @@ def getFiles(): else: f.text = f.srcName - return files + fileSet.add(f) def initialize(): def basicsFailed(msg): diff --git a/main.py b/main.py index 717d663..5563bd1 100755 --- a/main.py +++ b/main.py @@ -164,8 +164,8 @@ class MainWidget(qt.QMainWindow): self.patchColors = {'std': 'black', 'new': '#009600', 'remove': '#C80000', 'head': '#C800C8'} - self.files = [] - + self.files = scm.fileSetFactory(lambda f: self.addFile(f), + lambda f: self.removeFile(f)) f = File() f.text = "Commit message" f.textW = self.newTextEdit() @@ -296,7 +296,14 @@ EDITOR.''') self.refreshFiles() def currentChange(self, item): - self.text.raiseWidget(item.file.textW) + f = item.file + if not f.textW: + f.textW = self.newTextEdit() + f.textW.setReadOnly(True) + f.textW.setTextFormat(Qt.RichText) + f.textW.setText(formatPatchRichText(f.getPatch(), self.patchColors)) + + self.text.raiseWidget(f.textW) self.currentContextItem = item if item.commitMsg: self.updateCommitCursor() @@ -348,67 +355,65 @@ EDITOR.''') ret = FileState() cur = self.filesW.currentItem() if cur and cur != self.cmitItem: - ret.current = self.filesW.currentItem().file.dstName + ret.current = self.filesW.currentItem().file.text else: ret.current = None ret.selected = sets.Set() for x in self.filesW: if x.isSelected(): - ret.selected.add(x.file.dstName) + ret.selected.add(x.file.text) return ret def restoreFileState(self, state): for f in self.files: - f.listViewItem.setSelected(f.dstName in state.selected) + f.listViewItem.setSelected(f.text in state.selected) for x in self.filesW: - if x.file.dstName == state.current: + if x.file.text == state.current: self.filesW.setCurrentItem(x) def newTextEdit(self): ret = qt.QTextEdit() self.text.addWidget(ret) - return ret - - def setFiles(self, files): - state = self.getFileState() - self.filesW.clear() - self.createCmitItem() - for f in self.files: - self.text.removeWidget(f.textW) - f.listViewItem = None + return ret + + def addFile(self, file): + f = file + f.listViewItem = MyListItem(self.filesW, f) + + # The patch for this file is generated lazily in currentChange + + # Only display files that match the filter. + f.listViewItem.setVisible(self.filterMatch(f)) + + self.filesW.insertItem(f.listViewItem) - self.files = [] - for f in files: - f.textW = self.newTextEdit() - f.textW.setReadOnly(True) - f.textW.setTextFormat(Qt.RichText) - f.textW.setText(formatPatchRichText(f.patch, self.patchColors)) - self.files.append(f) - f.listViewItem = MyListItem(self.filesW, f) - # Only display files that match the filter. - f.listViewItem.setVisible(self.filterMatch(f)) - self.filesW.insertItem(f.listViewItem) + def removeFile(self, file): + f = file + self.text.removeWidget(f.textW) + self.filesW.takeItem(f.listViewItem) + f.listViewItem = None + def refreshFiles(self): + state = self.getFileState() + + self.setUpdatesEnabled(False) + scm.updateFiles(self.files) self.filesW.setCurrentItem(self.cmitItem) # For some reason the currentChanged signal isn't emitted # here. We call currentChange ourselves instead. self.currentChange(self.cmitItem) - self.restoreFileState(state) + self.setUpdatesEnabled(True) + self.update() - def refreshFiles(self, ignored=None): - files = scm.getFiles() - if settings().quitOnNoChanges and len(files) == 0: + if settings().quitOnNoChanges and len(self.files) == 0: self.close() - else: - self.setFiles(files) - - return len(files) > 0 + return len(self.files) > 0 def filterMatch(self, file): return file.dstName.find(unicode(self.filter.text())) != -1 -- 2.11.4.GIT