git-ftp-sync

   1 #!/usr/bin/env python
   2
   3 # Script for mirroring (a part of) git repository to a FTP site. Only
   4 # changed files are copied to FTP. Works with bare repositories.
   5 #
   6 # Author: Michal Sojka <sojkam1@fel.cvut.cz>
   7 # License: GPL
   8
   9 # Typical usage in hooks/post-receive (or pre-receive if you want push
  10 # command to fail in case of error during mirroring):
  11 #     export PASSWORD=secret
  12 #     /usr/local/bin/git-ftp-sync -H ftp.example.com -u username -r www
  13 # or
  14 #     /usr/local/bin/git-ftp-sync -d /var/www/projectweb -r www
  15
  16 # The first comamnd line mirrors the content og www directory in
  17 # repository to FTP site at ftp.example.com as user "username" with
  18 # password "secret".
  19
  20
  21 import sys
  22 from ftplib import FTP
  23 import ftplib
  24 from optparse import OptionParser
  25 from subprocess import Popen, PIPE, call
  26 import re
  27 import os
  28 from os.path import dirname, normpath
  29
  30 try:
  31     defpass = os.environ["PASSWORD"]
  32     del os.environ["PASSWORD"]
  33 except KeyError:
  34     defpass = ""
  35
  36 opt_parser = OptionParser(usage="usage: %prog [options] repository revision")
  37 opt_parser.add_option("-H", "--host", dest="host",
  38                   help="FTP server address. If not specified update files localy without FTP.")
  39 opt_parser.add_option("-u", "--user", dest="user",
  40                   help="FTP login name")
  41 opt_parser.add_option("-p", "--pass", dest="password", default=defpass,
  42                   help="FTP password (defaults to environment variable PASSWORD)")
  43 opt_parser.add_option("-r", "--repodir", dest="repodir", default="",
  44                   help="Synchronize only this directory (and its subdirectories) from within a repository")
  45 opt_parser.add_option("-d", "--dir", dest="dir", default="",
  46                   help="An existing directory (on FTP site or localhost), where to store synchronized files.")
  47
  48 class RepoChange:
  49     """Represents one line in git diff --name-status"""
  50     def __init__(self, type, path, oldrev, newrev, options):
  51         self.type = type.strip()        # one of ADMRCUX
  52         self.repo_path = path
  53         if path.startswith(options.repodir): # www/something -> something
  54             path = path[len(options.repodir):]
  55         if options.dir:                 # something -> prefix/something
  56             path = options.dir+"/"+path
  57         self.dest_path = normpath(path)
  58         self.oldrev = oldrev
  59         self.newrev = newrev
  60
  61     def isDir(self):
  62         return self.repo_path[-1] == "/"
  63
  64     def inDir(self, dir):
  65         """Dir: empty string or / closed directory name(s), without starting . or /"""
  66         return self.repo_path.startswith(dir)
  67
  68 class Syncer:
  69     """Abstract class performing synchronization"""
  70     def sync(self, change, dest_root):
  71         if change.type[0] == 'A' and change.isDir():
  72             # Never happens in git
  73             self.mkd(change.dest_path)
  74         elif change.type[0] in ["A", "C", "M"]:
  75             # Check whether the target directory exists
  76             retcode = call("git ls-tree %s %s|grep -q ." % (change.oldrev, dirname(change.repo_path)), shell=True)
  77             if (retcode != 0 and normpath(dirname(change.dest_path)) != normpath(dest_root)):
  78                 self.mkd(dirname(change.dest_path))
  79             # Upload the file
  80             print "%s: UPLOAD"%self.__class__.__name__, change.dest_path
  81             pipe = Popen("git cat-file blob %s:%s" % (change.newrev, change.repo_path),
  82                          stdout=PIPE, shell=True)
  83             self._storbinary(pipe.stdout, change.dest_path)
  84         elif change.type[0] == "D":
  85             if change.isDir():
  86                 # Never happens in git
  87                 self.rmd(change.dest_path)
  88             else:
  89                 print "%s: DEL   "%self.__class__.__name__, change.dest_path
  90                 self._delete(change.dest_path)
  91         else:
  92             perror("Unknown change: %s %s" % (change.type, change.dest_path))
  93             sys.exit(1)
  94
  95     def mkd(self, path):
  96         print "%s: MKD   "%self.__class__.__name__, path
  97         self._mkd(path)
  98
  99     def rmd(self, path):
 100         print "%s: RMD   "%self.__class__.__name__, path
 101         self._rmd(path);
 102
 103     def delete_empty_directories(self, changes, revision, dest_root):
 104         dirs = {}
 105         for change in changes:
 106             if change.type[0] in ["D", "R"]:
 107                 dirs[dirname(change.repo_path)] = dirname(change.dest_path)
 108         for d in dirs.keys():
 109             retcode = call("git ls-tree %s %s|grep -q ." % (revision, d), shell=True)
 110             if (retcode != 0 and normpath(dirs[d]) != normpath(dest_root)):
 111                 self.rmd(dirs[d])
 112
 113     def close(self):
 114         pass
 115
 116 class FTPSync(Syncer):
 117     def __init__(self, ftp):
 118         self.ftp = ftp
 119
 120     def close(self):
 121         self.ftp.close()
 122
 123     def _mkd(self, path):
 124         try:
 125             self.ftp.mkd(path)
 126         except ftplib.error_perm, detail:
 127             perror("FTP warning: %s %s" % (detail, path))
 128
 129     def _storbinary(self, string, path):
 130         #print >> sys.stderr, self.dest_path
 131         self.ftp.storbinary("STOR %s"%path, string)
 132
 133     def _rmd(self, path):
 134         try:
 135             # FIXME: this should be recursive deletion
 136             self.ftp.rmd(path)
 137         except ftplib.error_perm, detail:
 138             perror("FTP warning: %s %s" % (detail, path))
 139
 140     def _delete(self, path):
 141         try:
 142             #print >> sys.stderr, path
 143             self.ftp.delete(path)
 144         except ftplib.error_perm, detail:
 145             perror("FTP warning: %s %s" % (detail, path))
 146
 147
 148 class LocalSync(Syncer):
 149     def _mkd(self, path):
 150         try:
 151             os.mkdir(path)
 152         except OSError, detail:
 153             print "warning:", detail
 154
 155     def _storbinary(self, file, path):
 156         f = open(path, 'wb')
 157         s = file.read(10000)
 158         while s:
 159             f.write(s)
 160             s = file.read(10000)
 161         f.close()
 162
 163     def _rmd(self, path):
 164         """ Delete everything reachable from the directory named in 'self.dest_path',
 165         assuming there are no symbolic links."""
 166         for root, dirs, files in os.walk(path, topdown=False):
 167             for name in files:
 168                 os.remove(os.path.join(root, name))
 169             for name in dirs:
 170                 os.rmdir(os.path.join(root, name))
 171         os.rmdir(path)
 172
 173     def _delete(self, path):
 174         os.remove(path)
 175
 176 def selftest():
 177     commands = """
 178     set -x -e
 179     commit_push() { git commit -q -m "$1"; git push $REPO master; }
 180     mkdir repo.git
 181     cd repo.git
 182     git --bare init
 183     REPO=$PWD
 184     cd ..
 185     mkdir www
 186     mkdir working
 187     cd working
 188     git init
 189     mkdir www
 190
 191     # Commit something to www directory in repo without the activated hook
 192     echo 'Bye' > www/first.html
 193     git add www/first.html
 194     commit_push 'Added first.html'
 195
 196     # Activate the hook and commit a non-mirrored file
 197     echo "%s" > $REPO/hooks/post-receive
 198     chmod +x $REPO/hooks/post-receive
 199     echo 'abc' > non-mirrored-file
 200     git add non-mirrored-file
 201     commit_push 'Added non-mirrored-file'
 202
 203     # Check that the first commit appears in www even if the hook was
 204     # activated later (only for local sync)
 205     grep -q PASSWORD $REPO/hooks/post-receive || test -f ../www/first.html
 206
 207     cd www
 208     echo 'Hello' > index.html
 209     git add index.html
 210     commit_push 'Added index.html'
 211     mkdir subdir
 212     echo 'asdf' > subdir/subfile.html
 213     git add subdir/subfile.html
 214     commit_push 'Added subdir'
 215
 216     git mv subdir subdir2
 217     commit_push 'Renamed directory'
 218
 219     git mv subdir2/subfile.html subdir2/renamed.html
 220     commit_push 'Renamed file'
 221
 222     git rm -r subdir2
 223     commit_push 'Removed subdir2'
 224     git mv index.html index2.html
 225     commit_push 'Renamed file index->index2'
 226     cd ..
 227     git rm www/index2.html
 228     commit_push 'Removed index2'
 229     git rm www/first.html
 230     commit_push 'Removed first.html'
 231     cd $REPO/..
 232     rm -rf working repo.git www
 233     """
 234     selfpath = sys.argv[0]
 235
 236     print "Checking local sync"
 237     hook = "#!/bin/sh\n%s -d %s/www/ -r www" % (selfpath, os.getcwd())
 238     ret = os.system(commands % hook)
 239     if ret: return
 240
 241     print "\n\nChecking FTP sync (server: localhost, user: test, password: env. variable PASSWORD)"
 242     password = defpass
 243     hook = "#!/bin/sh\nexport PASSWORD=%(password)s\n%(selfpath)s -r www -H localhost -u test" % locals()
 244     ret = os.system(commands % hook)
 245     if ret: return
 246
 247     print
 248     print "Selftest succeeded!"
 249
 250 def perror(str):
 251     print >>sys.stderr, "git-ftp-sync: %s"%str
 252
 253
 254 def update_ref(hash):
 255     remote = "ftp"
 256     branch = "master"
 257     if (not os.path.exists("refs/remotes/"+remote)):
 258         os.makedirs("refs/remotes/"+remote)
 259     file("refs/remotes/"+remote+"/"+branch, "w").write(newrev+"\n")
 260
 261 def add_to_change_list(changes, git_command):
 262     # Read changes
 263     gitdiff = Popen(git_command,
 264                     stdout=PIPE, stderr=PIPE, close_fds=True, shell=True)
 265     change_re = re.compile("(\S+)\s+(.*)$");
 266     for line in gitdiff.stdout:
 267         #print line,
 268         m = change_re.match(line)
 269         if (m):
 270             change = RepoChange(m.group(1), m.group(2), oldrev, newrev, options)
 271             if change.inDir(options.repodir):
 272                 changes.append(change)
 273
 274 # Parse command line options
 275 (options, args) = opt_parser.parse_args()
 276
 277 if len(args) > 0 and args[0] == "selftest":
 278     selftest()
 279     sys.exit(0)
 280
 281 options.repodir = normpath(options.repodir).lstrip(".")
 282 if options.repodir: options.repodir+="/"
 283
 284 changes = list()
 285
 286 # Read the changes
 287 for line in sys.stdin:
 288     (oldrev, newrev, refname) = line.split()
 289     if refname == "refs/heads/master":
 290         try:
 291             oldrev=file("refs/remotes/ftp/master").readline().strip();
 292             git_command = "/usr/bin/git diff --name-status %s %s"%(oldrev, newrev)
 293         except IOError:
 294             # We are run for the first time, so (A)dd all files in the repo.
 295             git_command = r"git ls-tree -r --name-only %s | sed -e 's/\(.*\)/A \1/'"%(newrev);
 296
 297         add_to_change_list(changes, git_command)
 298
 299 if not changes:
 300     perror("No changes to sync")
 301     sys.exit(0)
 302
 303 # Apply changes
 304 try:
 305     if options.host:
 306         syncer = FTPSync(FTP(options.host, options.user, options.password))
 307     else:
 308         syncer = LocalSync()
 309
 310     for change in changes:
 311         syncer.sync(change, options.dir)
 312     syncer.delete_empty_directories(changes, newrev, options.dir)
 313
 314     syncer.close()
 315
 316 except ftplib.all_errors, detail:
 317     perror("FTP synchronization error: %s" % detail);
 318     perror("I will try it next time again");
 319     sys.exit(1)
 320
 321 # If succeessfull, update remote ref
 322 update_ref(newrev)