3 # Script for mirroring (a part of) git repository to an FTP site. Only
4 # changed files are copied to FTP. Works with bare repositories.
6 # Author: Michal Sojka <sojkam1@fel.cvut.cz>
9 # Typical usage in hooks/post-receive (or pre-receive if you want push
10 # command to fail in case of error during mirroring):
11 # export PASSWORD=secret
12 # /usr/local/bin/git-ftp-sync -H ftp.example.com -u username -r www
14 # /usr/local/bin/git-ftp-sync -d /var/www/projectweb -r www
16 # The first comamnd line mirrors the content of www directory in
17 # repository to FTP site at ftp.example.com as user "username" with
22 from ftplib
import FTP
24 from optparse
import OptionParser
25 from subprocess
import Popen
, PIPE
, call
28 from os
.path
import dirname
, normpath
29 from urlparse
import urlparse
32 defpass
= os
.environ
["PASSWORD"]
33 del os
.environ
["PASSWORD"]
37 opt_parser
= OptionParser(usage
="usage: %prog [options] repository revision")
38 opt_parser
.add_option("-U", "--url", dest
="url", default
="",
39 help="Destination URL (available protocols: sftp, ftp, file). Depricates -H, -u and -d.")
40 opt_parser
.add_option("-p", "--pass", dest
="password", default
=defpass
,
41 help="FTP password (defaults to environment variable PASSWORD)")
42 opt_parser
.add_option("-r", "--repodir", dest
="repodir", default
="",
43 help="Synchronize only this directory (and its subdirectories) from within a repository")
44 opt_parser
.add_option("-H", "--host", dest
="host",
45 help="FTP server address. If not specified update files localy without FTP.")
46 opt_parser
.add_option("-u", "--user", dest
="user",
47 help="FTP login name")
48 opt_parser
.add_option("-d", "--dir", dest
="dir", default
="",
49 help="An existing directory (on FTP site or localhost), where to store synchronized files.")
52 """Represents one line in git diff --name-status"""
53 def __init__(self
, type, path
, oldrev
, newrev
, options
, destroot
=None):
54 self
.type = type.strip() # one of ADMRCUX
56 if path
.startswith(options
.repodir
): # www/something -> something
57 path
= path
[len(options
.repodir
):]
58 if destroot
: # something -> prefix/something
59 path
= destroot
+"/"+path
60 self
.dest_path
= normpath(path
)
65 return self
.repo_path
[-1] == "/"
68 """Dir: empty string or / closed directory name(s), without starting . or /"""
69 return self
.repo_path
.startswith(dir)
72 """Abstract class performing synchronization"""
73 def sync(self
, change
, dest_root
):
74 if change
.type[0] == 'A' and change
.isDir():
75 # Never happens in git
76 self
.mkd(change
.dest_path
)
77 elif change
.type[0] in ["A", "C", "M"]:
78 # Check whether the target directory exists
79 if (normpath(dirname(change
.dest_path
)) != normpath(dest_root
)):
81 if (change
.oldrev
): # If there is previous revision, check for it
82 retcode
= call("git ls-tree %s %s|grep -q ." % (change
.oldrev
, dirname(change
.repo_path
)), shell
=True)
84 dirs
= normpath(dirname(change
.dest_path
)).split("/")
85 #print "YYYYYYYYYYYYY",dirname(change.dest_path)
89 destdir
= os
.path
.join(destdir
, i
);
90 #print "XXXXXXXXXXXXX",destdir
91 #self.mkd(dirname(change.dest_path))
95 print "%s: UPLOAD"%self
.__class
__.__name
__, change
.dest_path
96 pipe
= Popen("git cat-file blob %s:%s" % (change
.newrev
, change
.repo_path
),
97 stdout
=PIPE
, shell
=True)
98 self
._storbinary
(pipe
.stdout
, change
.dest_path
)
99 elif change
.type[0] == "D":
101 # Never happens in git
102 self
.rmd(change
.dest_path
)
104 print "%s: DEL "%self
.__class
__.__name
__, change
.dest_path
105 self
._delete
(change
.dest_path
)
107 perror("Unknown change: %s %s" % (change
.type, change
.dest_path
))
111 print "%s: MKD "%self
.__class
__.__name
__, path
115 print "%s: RMD "%self
.__class
__.__name
__, path
118 def delete_empty_directories(self
, changes
, revision
, dest_root
):
120 for change
in changes
:
121 if change
.type[0] in ["D", "R"]:
122 dirs
[dirname(change
.repo_path
)] = dirname(change
.dest_path
)
123 for d
in dirs
.keys():
124 retcode
= call("git ls-tree %s %s|grep -q ." % (revision
, d
), shell
=True)
125 if (retcode
!= 0 and normpath(dirs
[d
]) != normpath(dest_root
)):
131 class FTPSync(Syncer
):
132 def __init__(self
, ftp
):
138 def _mkd(self
, path
):
141 except ftplib
.error_perm
, detail
:
142 perror("FTP warning: %s %s" % (detail
, path
))
144 def _storbinary(self
, string
, path
):
145 #print >> sys.stderr, self.dest_path
146 self
.ftp
.storbinary("STOR %s"%path
, string
)
148 def _rmd(self
, path
):
150 # FIXME: this should be recursive deletion
152 except ftplib
.error_perm
, detail
:
153 perror("FTP warning: %s %s" % (detail
, path
))
155 def _delete(self
, path
):
157 #print >> sys.stderr, path
158 self
.ftp
.delete(path
)
159 except ftplib
.error_perm
, detail
:
160 perror("FTP warning: %s %s" % (detail
, path
))
162 class SFTPSync(Syncer
):
163 def __init__(self
, url
):
165 # get host key, if we know one
169 host_keys
= paramiko
.util
.load_host_keys(os
.path
.expanduser('~/.ssh/known_hosts'))
172 # try ~/ssh/ too, because windows can't have a folder named ~/.ssh/
173 host_keys
= paramiko
.util
.load_host_keys(os
.path
.expanduser('~/ssh/known_hosts'))
175 print '*** Unable to open host keys file'
178 if host_keys
.has_key(url
.hostname
):
179 hostkeytype
= host_keys
[url
.hostname
].keys()[0]
180 hostkey
= host_keys
[url
.hostname
][hostkeytype
]
181 print 'Using host key of type %s' % hostkeytype
184 # now, connect and use paramiko Transport to negotiate SSH2 across the connection
186 if url
.port
: port
= url
.port
187 self
.t
= paramiko
.Transport((url
.hostname
, port
))
188 password
= url
.password
189 if not password
: password
= options
.password
190 self
.t
.connect(username
=url
.username
, password
=password
, hostkey
=hostkey
)
191 self
.sftp
= paramiko
.SFTPClient
.from_transport(self
.t
)
197 def _mkd(self
, path
):
199 self
.sftp
.mkdir(path
)
200 except IOError, detail
:
201 print "sftp warning:", detail
203 def _storbinary(self
, file, path
):
204 remote
= self
.sftp
.open(path
, 'w')
212 def _rmd(self
, path
):
213 """ Delete everything reachable from the directory named in 'self.dest_path',
214 assuming there are no symbolic links."""
215 self
.sftp
.rmdir(path
)
216 # FIXME: this should be recursive deletion
218 def _delete(self
, path
):
219 self
.sftp
.remove(path
)
222 class LocalSync(Syncer
):
223 def _mkd(self
, path
):
226 except OSError, detail
:
227 print "warning:", detail
229 def _storbinary(self
, file, path
):
237 def _rmd(self
, path
):
238 """ Delete everything reachable from the directory named in 'self.dest_path',
239 assuming there are no symbolic links."""
240 for root
, dirs
, files
in os
.walk(path
, topdown
=False):
242 os
.remove(os
.path
.join(root
, name
))
244 os
.rmdir(os
.path
.join(root
, name
))
247 def _delete(self
, path
):
253 commit_push() { git commit -q -m "$1"; git push $REPO master; }
265 # Commit something to www directory in repo without the activated hook
268 echo 'Bye' > www/dir/dir2/first.html
269 git add www/dir/dir2/first.html
270 commit_push 'Added first.html'
272 # Activate the hook and commit a non-mirrored file
273 echo "%s" > $REPO/hooks/post-receive
274 chmod +x $REPO/hooks/post-receive
275 echo 'abc' > non-mirrored-file
276 git add non-mirrored-file
277 commit_push 'Added non-mirrored-file'
279 # Check that the first commit appears in www even if the hook was
280 # activated later (only for local sync)
281 grep -q PASSWORD $REPO/hooks/post-receive || test -f ../www/dir/dir2/first.html
284 echo 'Hello' > index.html
286 commit_push 'Added index.html'
288 echo 'asdf' > subdir/subfile.html
289 git add subdir/subfile.html
290 commit_push 'Added subdir'
292 git mv subdir subdir2
293 commit_push 'Renamed directory'
295 git mv subdir2/subfile.html subdir2/renamed.html
296 commit_push 'Renamed file'
299 commit_push 'Removed subdir2'
300 git mv index.html index2.html
301 commit_push 'Renamed file index->index2'
303 git rm www/index2.html
304 commit_push 'Removed index2'
305 git rm www/dir/dir2/first.html
306 commit_push 'Removed first.html'
308 rm -rf working repo.git www
310 selfpath
= sys
.argv
[0]
312 print "Checking local sync"
313 hook
= "#!/bin/sh\n%s -d %s/www/ -r www" % (selfpath
, os
.getcwd())
314 ret
= os
.system(commands
% hook
)
317 print "\n\nChecking FTP sync (server: localhost, user: test, password: env. variable PASSWORD)"
319 hook
= "#!/bin/sh\nexport PASSWORD=%(password)s\n%(selfpath)s -r www -H localhost -u test" % locals()
320 ret
= os
.system(commands
% hook
)
323 print "\n\nChecking SFTP sync (server: localhost, user: test, password: env. variable PASSWORD)"
325 hook
= """#!/bin/sh\nexport PASSWORD=%(password)s\n%(selfpath)s -r www --url sftp://test@localhost""" % locals()
326 ret
= os
.system(commands
% hook
)
330 print "Selftest succeeded!"
333 print >>sys
.stderr
, "git-ftp-sync: %s"%str
336 def update_ref(hash):
337 cmd
= "git update-ref refs/remotes/ftp/master %s"%(hash)
338 #print "Runnging", cmd
341 def add_to_change_list(changes
, git_command
, oldrev
, newrev
, destroot
):
343 ##print "Running: ", git_command
344 gitdiff
= Popen(git_command
,
345 stdout
=PIPE
, stderr
=PIPE
, close_fds
=True, shell
=True)
346 change_re
= re
.compile("(\S+)\s+(.*)$");
347 for line
in gitdiff
.stdout
:
349 m
= change_re
.match(line
)
351 change
= RepoChange(m
.group(1), m
.group(2), oldrev
, newrev
, options
, destroot
)
352 if change
.inDir(options
.repodir
):
353 changes
.append(change
)
355 # Parse command line options
356 (options
, args
) = opt_parser
.parse_args()
358 if len(args
) > 0 and args
[0] == "selftest":
362 options
.repodir
= normpath(options
.repodir
).lstrip(".")
363 if options
.repodir
: options
.repodir
+="/"
366 url
= urlparse("ftp://"+options
.user
+"@"+options
.host
+"/"+options
.dir)
367 elif options
.dir and not options
.url
:
368 url
= urlparse("file:///"+options
.dir)
370 url
= urlparse(options
.url
)
375 if 'GIT_DIR' in os
.environ
:
376 # Invocation from hook
377 for line
in sys
.stdin
:
378 (oldrev
, newrev
, refname
) = line
.split()
379 if refname
== "refs/heads/master":
381 oldrev
=os
.popen('git show-ref --hash refs/remotes/ftp/master').read().strip()
382 if not oldrev
: raise IOError, "No ref" # Simulate failure if the branch doesn't exist
383 git_command
= "git diff --name-status %s %s"%(oldrev
, newrev
)
385 # We are run for the first time, so (A)dd all files in the repo.
387 git_command
= r
"git ls-tree -r --name-only %s | sed -e 's/\(.*\)/A \1/'"%(newrev);
389 add_to_change_list(changes
, git_command
, oldrev
, newrev
, url
.path
)
394 git_command
= r
"git ls-tree -r --name-only HEAD | sed -e 's/\(.*\)/A \1/'";
395 add_to_change_list(changes
, git_command
, oldrev
, newrev
)
398 perror("No changes to sync")
403 if url
.scheme
== "ftp":
404 syncer
= FTPSync(FTP(options
.host
, options
.user
, options
.password
))
405 elif url
.scheme
== "file":
407 elif url
.scheme
== "sftp":
408 syncer
= SFTPSync(url
)
410 for change
in changes
:
411 syncer
.sync(change
, options
.dir)
412 syncer
.delete_empty_directories(changes
, newrev
, options
.dir)
416 # except ftplib.all_errors, detail:
417 # perror("FTP synchronization error: %s" % detail);
418 # perror("I will try it next time again");
423 # If succeessfull, update remote ref