git-ftp-sync

   1 #!/usr/bin/env python
   2
   3 # Script for mirroring (a part of) git repository to an FTP site. Only
   4 # changed files are copied to FTP. Works with bare repositories.
   5 #
   6 # Author: Michal Sojka <sojkam1@fel.cvut.cz>
   7 # License: GPL
   8
   9 # Typical usage in hooks/post-receive (or pre-receive if you want push
  10 # command to fail in case of error during mirroring):
  11 #     export PASSWORD=secret
  12 #     /usr/local/bin/git-ftp-sync -H ftp.example.com -u username -r www
  13 # or
  14 #     /usr/local/bin/git-ftp-sync -d /var/www/projectweb -r www
  15
  16 # The first comamnd line mirrors the content of www directory in
  17 # repository to FTP site at ftp.example.com as user "username" with
  18 # password "secret".
  19
  20
  21 import sys
  22 from ftplib import FTP
  23 import ftplib
  24 from optparse import OptionParser
  25 from subprocess import Popen, PIPE, call
  26 import re
  27 import os
  28 from os.path import dirname, normpath
  29 from urlparse import urlparse
  30
  31 try:
  32     defpass = os.environ["PASSWORD"]
  33     del os.environ["PASSWORD"]
  34 except KeyError:
  35     defpass = ""
  36
  37 opt_parser = OptionParser(usage="usage: %prog [options] repository revision")
  38 opt_parser.add_option("-U", "--url", dest="url", default="",
  39                   help="Destination URL (available protocols: sftp, ftp, file). Depricates -H, -u and -d.")
  40 opt_parser.add_option("-p", "--pass", dest="password", default=defpass,
  41                   help="FTP password (defaults to environment variable PASSWORD)")
  42 opt_parser.add_option("-r", "--repodir", dest="repodir", default="",
  43                   help="Synchronize only this directory (and its subdirectories) from within a repository")
  44 opt_parser.add_option("-H", "--host", dest="host",
  45                   help="FTP server address. If not specified update files localy without FTP.")
  46 opt_parser.add_option("-u", "--user", dest="user",
  47                   help="FTP login name")
  48 opt_parser.add_option("-d", "--dir", dest="dir", default="",
  49                   help="An existing directory (on FTP site or localhost), where to store synchronized files.")
  50
  51 class RepoChange:
  52     """Represents one line in git diff --name-status"""
  53     def __init__(self, type, path, oldrev, newrev, options, destroot=None):
  54         self.type = type.strip()        # one of ADMRCUX
  55         self.repo_path = path
  56         if path.startswith(options.repodir): # www/something -> something
  57             path = path[len(options.repodir):]
  58         if destroot:                 # something -> prefix/something
  59             path = destroot+"/"+path
  60         self.dest_path = normpath(path)
  61         self.oldrev = oldrev
  62         self.newrev = newrev
  63
  64     def isDir(self):
  65         return self.repo_path[-1] == "/"
  66
  67     def inDir(self, dir):
  68         """Dir: empty string or / closed directory name(s), without starting . or /"""
  69         return self.repo_path.startswith(dir)
  70
  71 class Syncer:
  72     """Abstract class performing synchronization"""
  73     def sync(self, change, dest_root):
  74         if change.type[0] == 'A' and change.isDir():
  75             # Never happens in git
  76             self.mkd(change.dest_path)
  77         elif change.type[0] in ["A", "C", "M"]:
  78             # Check whether the target directory exists
  79             if (normpath(dirname(change.dest_path)) != normpath(dest_root)):
  80                 retcode = 1
  81                 if (change.oldrev):         # If there is previous revision, check for it
  82                     retcode = call("git ls-tree %s %s|grep -q ." % (change.oldrev, dirname(change.repo_path)), shell=True)
  83                 if (retcode != 0):
  84                     dirs = normpath(dirname(change.dest_path)).split("/")
  85                     #print "YYYYYYYYYYYYY",dirname(change.dest_path)
  86                     destdir = ""
  87                     for i in dirs:
  88                         if (i==""): i="/"
  89                         destdir = os.path.join(destdir, i);
  90                         #print "XXXXXXXXXXXXX",destdir
  91                         #self.mkd(dirname(change.dest_path))
  92                         self.mkd(destdir)
  93
  94             # Upload the file
  95             print "%s: UPLOAD"%self.__class__.__name__, change.dest_path
  96             pipe = Popen("git cat-file blob %s:%s" % (change.newrev, change.repo_path),
  97                          stdout=PIPE, shell=True)
  98             self._storbinary(pipe.stdout, change.dest_path)
  99         elif change.type[0] == "D":
 100             if change.isDir():
 101                 # Never happens in git
 102                 self.rmd(change.dest_path)
 103             else:
 104                 print "%s: DEL   "%self.__class__.__name__, change.dest_path
 105                 self._delete(change.dest_path)
 106         else:
 107             perror("Unknown change: %s %s" % (change.type, change.dest_path))
 108             sys.exit(1)
 109
 110     def mkd(self, path):
 111         print "%s: MKD   "%self.__class__.__name__, path
 112         self._mkd(path)
 113
 114     def rmd(self, path):
 115         print "%s: RMD   "%self.__class__.__name__, path
 116         self._rmd(path);
 117
 118     def delete_empty_directories(self, changes, revision, dest_root):
 119         dirs = {}
 120         for change in changes:
 121             if change.type[0] in ["D", "R"]:
 122                 dirs[dirname(change.repo_path)] = dirname(change.dest_path)
 123         for d in dirs.keys():
 124             retcode = call("git ls-tree %s %s|grep -q ." % (revision, d), shell=True)
 125             if (retcode != 0 and normpath(dirs[d]) != normpath(dest_root)):
 126                 self.rmd(dirs[d])
 127
 128     def close(self):
 129         pass
 130
 131 class FTPSync(Syncer):
 132     def __init__(self, ftp):
 133         self.ftp = ftp
 134
 135     def close(self):
 136         self.ftp.close()
 137
 138     def _mkd(self, path):
 139         try:
 140             self.ftp.mkd(path)
 141         except ftplib.error_perm, detail:
 142             perror("FTP warning: %s %s" % (detail, path))
 143
 144     def _storbinary(self, string, path):
 145         #print >> sys.stderr, self.dest_path
 146         self.ftp.storbinary("STOR %s"%path, string)
 147
 148     def _rmd(self, path):
 149         try:
 150             # FIXME: this should be recursive deletion
 151             self.ftp.rmd(path)
 152         except ftplib.error_perm, detail:
 153             perror("FTP warning: %s %s" % (detail, path))
 154
 155     def _delete(self, path):
 156         try:
 157             #print >> sys.stderr, path
 158             self.ftp.delete(path)
 159         except ftplib.error_perm, detail:
 160             perror("FTP warning: %s %s" % (detail, path))
 161
 162 class SFTPSync(Syncer):
 163     def __init__(self, url):
 164         import paramiko
 165         # get host key, if we know one
 166         hostkeytype = None
 167         hostkey = None
 168         try:
 169             host_keys = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
 170         except IOError:
 171             try:
 172                 # try ~/ssh/ too, because windows can't have a folder named ~/.ssh/
 173                 host_keys = paramiko.util.load_host_keys(os.path.expanduser('~/ssh/known_hosts'))
 174             except IOError:
 175                 print '*** Unable to open host keys file'
 176                 host_keys = {}
 177
 178         if host_keys.has_key(url.hostname):
 179             hostkeytype = host_keys[url.hostname].keys()[0]
 180             hostkey = host_keys[url.hostname][hostkeytype]
 181             print 'Using host key of type %s' % hostkeytype
 182
 183
 184         # now, connect and use paramiko Transport to negotiate SSH2 across the connection
 185         port = 22
 186         if url.port: port = url.port
 187         self.t = paramiko.Transport((url.hostname, port))
 188         password = url.password
 189         if not password: password = options.password
 190         self.t.connect(username=url.username, password=password, hostkey=hostkey)
 191         self.sftp = paramiko.SFTPClient.from_transport(self.t)
 192
 193     def close(self):
 194         self.sftp.close()
 195         self.t.close()
 196
 197     def _mkd(self, path):
 198         try:
 199             self.sftp.mkdir(path)
 200         except IOError, detail:
 201             print "sftp warning:", detail
 202
 203     def _storbinary(self, file, path):
 204         remote = self.sftp.open(path, 'w')
 205         s = file.read(10000)
 206         while s:
 207             remote.write(s)
 208             s = file.read(10000)
 209         remote.close()
 210
 211
 212     def _rmd(self, path):
 213         """ Delete everything reachable from the directory named in 'self.dest_path',
 214         assuming there are no symbolic links."""
 215         self.sftp.rmdir(path)
 216         # FIXME: this should be recursive deletion
 217
 218     def _delete(self, path):
 219         self.sftp.remove(path)
 220
 221
 222 class LocalSync(Syncer):
 223     def _mkd(self, path):
 224         try:
 225             os.mkdir(path)
 226         except OSError, detail:
 227             print "warning:", detail
 228
 229     def _storbinary(self, file, path):
 230         f = open(path, 'wb')
 231         s = file.read(10000)
 232         while s:
 233             f.write(s)
 234             s = file.read(10000)
 235         f.close()
 236
 237     def _rmd(self, path):
 238         """ Delete everything reachable from the directory named in 'self.dest_path',
 239         assuming there are no symbolic links."""
 240         for root, dirs, files in os.walk(path, topdown=False):
 241             for name in files:
 242                 os.remove(os.path.join(root, name))
 243             for name in dirs:
 244                 os.rmdir(os.path.join(root, name))
 245         os.rmdir(path)
 246
 247     def _delete(self, path):
 248         os.remove(path)
 249
 250 def selftest():
 251     commands = """
 252     set -x -e
 253     commit_push() { git commit -q -m "$1"; git push $REPO master; }
 254     mkdir repo.git
 255     cd repo.git
 256     git --bare init
 257     REPO=$PWD
 258     cd ..
 259     mkdir www
 260     mkdir working
 261     cd working
 262     git init
 263     mkdir www
 264
 265     # Commit something to www directory in repo without the activated hook
 266     mkdir www/dir
 267     mkdir www/dir/dir2
 268     echo 'Bye' > www/dir/dir2/first.html
 269     git add www/dir/dir2/first.html
 270     commit_push 'Added first.html'
 271
 272     # Activate the hook and commit a non-mirrored file
 273     echo "%s" > $REPO/hooks/post-receive
 274     chmod +x $REPO/hooks/post-receive
 275     echo 'abc' > non-mirrored-file
 276     git add non-mirrored-file
 277     commit_push 'Added non-mirrored-file'
 278
 279     # Check that the first commit appears in www even if the hook was
 280     # activated later (only for local sync)
 281     grep -q PASSWORD $REPO/hooks/post-receive || test -f ../www/dir/dir2/first.html
 282
 283     cd www
 284     echo 'Hello' > index.html
 285     git add index.html
 286     commit_push 'Added index.html'
 287     mkdir subdir
 288     echo 'asdf' > subdir/subfile.html
 289     git add subdir/subfile.html
 290     commit_push 'Added subdir'
 291
 292     git mv subdir subdir2
 293     commit_push 'Renamed directory'
 294
 295     git mv subdir2/subfile.html subdir2/renamed.html
 296     commit_push 'Renamed file'
 297
 298     git rm -r subdir2
 299     commit_push 'Removed subdir2'
 300     git mv index.html index2.html
 301     commit_push 'Renamed file index->index2'
 302     cd ..
 303     git rm www/index2.html
 304     commit_push 'Removed index2'
 305     git rm www/dir/dir2/first.html
 306     commit_push 'Removed first.html'
 307     cd $REPO/..
 308     rm -rf working repo.git www
 309     """
 310     selfpath = sys.argv[0]
 311
 312     print "Checking local sync"
 313     hook = "#!/bin/sh\n%s -d %s/www/ -r www" % (selfpath, os.getcwd())
 314     ret = os.system(commands % hook)
 315     if ret: return
 316
 317     print "\n\nChecking FTP sync (server: localhost, user: test, password: env. variable PASSWORD)"
 318     password = defpass
 319     hook = "#!/bin/sh\nexport PASSWORD=%(password)s\n%(selfpath)s -r www -H localhost -u test" % locals()
 320     ret = os.system(commands % hook)
 321     if ret: return
 322
 323     print "\n\nChecking SFTP sync (server: localhost, user: test, password: env. variable PASSWORD)"
 324     password = defpass
 325     hook = """#!/bin/sh\nexport PASSWORD=%(password)s\n%(selfpath)s -r www --url sftp://test@localhost""" % locals()
 326     ret = os.system(commands % hook)
 327     if ret: return
 328
 329     print
 330     print "Selftest succeeded!"
 331
 332 def perror(str):
 333     print >>sys.stderr, "git-ftp-sync: %s"%str
 334
 335
 336 def update_ref(hash):
 337     cmd = "git update-ref refs/remotes/ftp/master %s"%(hash)
 338     #print "Runnging", cmd
 339     os.system(cmd)
 340
 341 def add_to_change_list(changes, git_command, oldrev, newrev, destroot):
 342     # Read changes
 343     ##print "Running: ", git_command
 344     gitdiff = Popen(git_command,
 345                     stdout=PIPE, stderr=PIPE, close_fds=True, shell=True)
 346     change_re = re.compile("(\S+)\s+(.*)$");
 347     for line in gitdiff.stdout:
 348         ##print line,
 349         m = change_re.match(line)
 350         if (m):
 351             change = RepoChange(m.group(1), m.group(2), oldrev, newrev, options, destroot)
 352             if change.inDir(options.repodir):
 353                 changes.append(change)
 354
 355 # Parse command line options
 356 (options, args) = opt_parser.parse_args()
 357
 358 if len(args) > 0 and args[0] == "selftest":
 359     selftest()
 360     sys.exit(0)
 361
 362 options.repodir = normpath(options.repodir).lstrip(".")
 363 if options.repodir: options.repodir+="/"
 364
 365 if options.host:
 366     url = urlparse("ftp://"+options.user+"@"+options.host+"/"+options.dir)
 367 elif options.dir and not options.url:
 368     url = urlparse("file:///"+options.dir)
 369 else:
 370     url = urlparse(options.url)
 371
 372 changes = list()
 373
 374 # Read the changes
 375 if 'GIT_DIR' in os.environ:
 376     # Invocation from hook
 377     for line in sys.stdin:
 378         (oldrev, newrev, refname) = line.split()
 379         if refname == "refs/heads/master":
 380             try:
 381                 oldrev=os.popen('git show-ref --hash refs/remotes/ftp/master').read().strip()
 382                 if not oldrev: raise IOError, "No ref" # Simulate failure if the branch doesn't exist
 383                 git_command = "git diff --name-status %s %s"%(oldrev, newrev)
 384             except IOError:
 385                 # We are run for the first time, so (A)dd all files in the repo.
 386                 oldrev = None
 387                 git_command = r"git ls-tree -r --name-only %s | sed -e 's/\(.*\)/A \1/'"%(newrev);
 388
 389         add_to_change_list(changes, git_command, oldrev, newrev, url.path)
 390 else:
 391     # Manual invocation
 392     newrev = "HEAD"
 393     oldrev = None;
 394     git_command = r"git ls-tree -r --name-only HEAD | sed -e 's/\(.*\)/A \1/'";
 395     add_to_change_list(changes, git_command, oldrev, newrev)
 396
 397 if not changes:
 398     perror("No changes to sync")
 399     sys.exit(0)
 400
 401 # Apply changes
 402 try:
 403     if url.scheme == "ftp":
 404         syncer = FTPSync(FTP(options.host, options.user, options.password))
 405     elif url.scheme == "file":
 406         syncer = LocalSync()
 407     elif url.scheme == "sftp":
 408         syncer = SFTPSync(url)
 409
 410     for change in changes:
 411         syncer.sync(change, options.dir)
 412     syncer.delete_empty_directories(changes, newrev, options.dir)
 413
 414     syncer.close()
 415
 416 # except ftplib.all_errors, detail:
 417 #     perror("FTP synchronization error: %s" % detail);
 418 #     perror("I will try it next time again");
 419 #     sys.exit(1)
 420 except:
 421     raise
 422
 423 # If succeessfull, update remote ref
 424 update_ref(newrev)