3 # Script for mirroring (a part of) git repository to an FTP site. Only
4 # changed files are copied to FTP. Works with bare repositories.
6 # Author: Michal Sojka <sojkam1@fel.cvut.cz>
9 # Typical usage in hooks/post-receive (or pre-receive if you want push
10 # command to fail in case of error during mirroring):
11 # export PASSWORD=secret
12 # /usr/local/bin/git-ftp-sync -H ftp.example.com -u username -r www
14 # /usr/local/bin/git-ftp-sync -d /var/www/projectweb -r www
16 # The first comamnd line mirrors the content of www directory in
17 # repository to FTP site at ftp.example.com as user "username" with
22 from ftplib import FTP
24 from optparse import OptionParser
25 from subprocess import Popen, PIPE, call
28 from os.path import dirname, normpath
29 from urlparse import urlparse
32 defpass = os.environ["PASSWORD"]
33 del os.environ["PASSWORD"]
37 opt_parser = OptionParser(usage="usage: %prog [options] repository revision")
38 opt_parser.add_option("-U", "--url", dest="url", default="",
39 help="Destination URL (available protocols: sftp, ftp, file). Depricates -H, -u and -d.")
40 opt_parser.add_option("-p", "--pass", dest="password", default=defpass,
41 help="FTP password (defaults to environment variable PASSWORD)")
42 opt_parser.add_option("-r", "--repodir", dest="repodir", default="",
43 help="Synchronize only this directory (and its subdirectories) from within a repository")
44 opt_parser.add_option("-H", "--host", dest="host",
45 help="FTP server address. If not specified update files localy without FTP.")
46 opt_parser.add_option("-u", "--user", dest="user",
47 help="FTP login name")
48 opt_parser.add_option("-d", "--dir", dest="dir", default="",
49 help="An existing directory (on FTP site or localhost), where to store synchronized files.")
52 """Represents one line in git diff --name-status"""
53 def __init__(self, type, path, oldrev, newrev, options, destroot=None):
54 self.type = type.strip() # one of ADMRCUX
56 if path.startswith(options.repodir): # www/something -> something
57 path = path[len(options.repodir):]
58 if destroot: # something -> prefix/something
59 path = destroot+"/"+path
60 self.dest_path = normpath(path)
65 return self.repo_path[-1] == "/"
68 """Dir: empty string or / closed directory name(s), without starting . or /"""
69 return self.repo_path.startswith(dir)
72 """Abstract class performing synchronization"""
73 def sync(self, change, dest_root):
74 if change.type[0] == 'A' and change.isDir():
75 # Never happens in git
76 self.mkd(change.dest_path)
77 elif change.type[0] in ["A", "C", "M"]:
78 # Check whether the target directory exists
79 if (normpath(dirname(change.dest_path)) != normpath(dest_root)):
81 if (change.oldrev): # If there is previous revision, check for it
82 retcode = call("git ls-tree %s %s|grep -q ." % (change.oldrev, dirname(change.repo_path)), shell=True)
84 dirs = normpath(dirname(change.dest_path)).split("/")
85 #print "YYYYYYYYYYYYY",dirname(change.dest_path)
89 destdir = os.path.join(destdir, i);
90 #print "XXXXXXXXXXXXX",destdir
91 #self.mkd(dirname(change.dest_path))
95 print "%s: UPLOAD"%self.__class__.__name__, change.dest_path
96 pipe = Popen("git cat-file blob %s:%s" % (change.newrev, change.repo_path),
97 stdout=PIPE, shell=True)
98 self._storbinary(pipe.stdout, change.dest_path)
99 elif change.type[0] == "D":
101 # Never happens in git
102 self.rmd(change.dest_path)
104 print "%s: DEL "%self.__class__.__name__, change.dest_path
105 self._delete(change.dest_path)
107 perror("Unknown change: %s %s" % (change.type, change.dest_path))
111 print "%s: MKD "%self.__class__.__name__, path
115 print "%s: RMD "%self.__class__.__name__, path
118 def delete_empty_directories(self, changes, revision, dest_root):
120 for change in changes:
121 if change.type[0] in ["D", "R"]:
122 dirs[dirname(change.repo_path)] = dirname(change.dest_path)
123 for d in dirs.keys():
124 retcode = call("git ls-tree %s %s|grep -q ." % (revision, d), shell=True)
125 if (retcode != 0 and normpath(dirs[d]) != normpath(dest_root)):
131 class FTPSync(Syncer):
132 def __init__(self, ftp):
138 def _mkd(self, path):
141 except ftplib.error_perm, detail:
142 perror("FTP warning: %s %s" % (detail, path))
144 def _storbinary(self, string, path):
145 #print >> sys.stderr, self.dest_path
146 self.ftp.storbinary("STOR %s"%path, string)
148 def _rmd(self, path):
150 # FIXME: this should be recursive deletion
152 except ftplib.error_perm, detail:
153 perror("FTP warning: %s %s" % (detail, path))
155 def _delete(self, path):
157 #print >> sys.stderr, path
158 self.ftp.delete(path)
159 except ftplib.error_perm, detail:
160 perror("FTP warning: %s %s" % (detail, path))
162 class SFTPSync(Syncer):
163 def __init__(self, url):
165 # get host key, if we know one
169 host_keys = paramiko.util.load_host_keys(os.path.expanduser('~/.ssh/known_hosts'))
172 # try ~/ssh/ too, because windows can't have a folder named ~/.ssh/
173 host_keys = paramiko.util.load_host_keys(os.path.expanduser('~/ssh/known_hosts'))
175 print '*** Unable to open host keys file'
178 if host_keys.has_key(url.hostname):
179 hostkeytype = host_keys[url.hostname].keys()[0]
180 hostkey = host_keys[url.hostname][hostkeytype]
181 print 'Using host key of type %s' % hostkeytype
184 # now, connect and use paramiko Transport to negotiate SSH2 across the connection
186 if url.port: port = url.port
187 self.t = paramiko.Transport((url.hostname, port))
188 password = url.password
189 if not password: password = options.password
190 self.t.connect(username=url.username, password=password, hostkey=hostkey)
191 self.sftp = paramiko.SFTPClient.from_transport(self.t)
197 def _mkd(self, path):
199 self.sftp.mkdir(path)
200 except IOError, detail:
201 print "sftp warning:", detail
203 def _storbinary(self, file, path):
204 remote = self.sftp.open(path, 'w')
212 def _rmd(self, path):
213 """ Delete everything reachable from the directory named in 'self.dest_path',
214 assuming there are no symbolic links."""
215 self.sftp.rmdir(path)
216 # FIXME: this should be recursive deletion
218 def _delete(self, path):
219 self.sftp.remove(path)
222 class LocalSync(Syncer):
223 def _mkd(self, path):
226 except OSError, detail:
227 print "warning:", detail
229 def _storbinary(self, file, path):
237 def _rmd(self, path):
238 """ Delete everything reachable from the directory named in 'self.dest_path',
239 assuming there are no symbolic links."""
240 for root, dirs, files in os.walk(path, topdown=False):
242 os.remove(os.path.join(root, name))
244 os.rmdir(os.path.join(root, name))
247 def _delete(self, path):
253 commit_push() { git commit -q -m "$1"; git push $REPO master; }
265 # Commit something to www directory in repo without the activated hook
268 echo 'Bye' > www/dir/dir2/first.html
269 git add www/dir/dir2/first.html
270 commit_push 'Added first.html'
272 # Activate the hook and commit a non-mirrored file
273 echo "%s" > $REPO/hooks/post-receive
274 chmod +x $REPO/hooks/post-receive
275 echo 'abc' > non-mirrored-file
276 git add non-mirrored-file
277 commit_push 'Added non-mirrored-file'
279 # Check that the first commit appears in www even if the hook was
280 # activated later (only for local sync)
281 grep -q PASSWORD $REPO/hooks/post-receive || test -f ../www/dir/dir2/first.html
284 echo 'Hello' > index.html
286 commit_push 'Added index.html'
288 echo 'asdf' > subdir/subfile.html
289 git add subdir/subfile.html
290 commit_push 'Added subdir'
292 git mv subdir subdir2
293 commit_push 'Renamed directory'
295 git mv subdir2/subfile.html subdir2/renamed.html
296 commit_push 'Renamed file'
299 commit_push 'Removed subdir2'
300 git mv index.html index2.html
301 commit_push 'Renamed file index->index2'
303 git rm www/index2.html
304 commit_push 'Removed index2'
305 git rm www/dir/dir2/first.html
306 commit_push 'Removed first.html'
308 rm -rf working repo.git www
310 selfpath = sys.argv[0]
312 print "Checking local sync"
313 hook = "#!/bin/sh\n%s -d %s/www/ -r www" % (selfpath, os.getcwd())
314 ret = os.system(commands % hook)
317 print "\n\nChecking FTP sync (server: localhost, user: test, password: env. variable PASSWORD)"
319 hook = "#!/bin/sh\nexport PASSWORD=%(password)s\n%(selfpath)s -r www -H localhost -u test" % locals()
320 ret = os.system(commands % hook)
323 print "\n\nChecking SFTP sync (server: localhost, user: test, password: env. variable PASSWORD)"
325 hook = """#!/bin/sh\nexport PASSWORD=%(password)s\n%(selfpath)s -r www --url sftp://test@localhost""" % locals()
326 ret = os.system(commands % hook)
330 print "Selftest succeeded!"
333 print >>sys.stderr, "git-ftp-sync: %s"%str
336 def update_ref(hash):
337 cmd = "git update-ref refs/remotes/ftp/master %s"%(hash)
338 #print "Runnging", cmd
341 def add_to_change_list(changes, git_command, oldrev, newrev, destroot):
343 ##print "Running: ", git_command
344 gitdiff = Popen(git_command,
345 stdout=PIPE, stderr=PIPE, close_fds=True, shell=True)
346 change_re = re.compile("(\S+)\s+(.*)$");
347 for line in gitdiff.stdout:
349 m = change_re.match(line)
351 change = RepoChange(m.group(1), m.group(2), oldrev, newrev, options, destroot)
352 if change.inDir(options.repodir):
353 changes.append(change)
355 # Parse command line options
356 (options, args) = opt_parser.parse_args()
358 if len(args) > 0 and args[0] == "selftest":
362 options.repodir = normpath(options.repodir).lstrip(".")
363 if options.repodir: options.repodir+="/"
366 url = urlparse("ftp://"+options.user+"@"+options.host+"/"+options.dir)
367 elif options.dir and not options.url:
368 url = urlparse("file:///"+options.dir)
370 url = urlparse(options.url)
375 if 'GIT_DIR' in os.environ:
376 # Invocation from hook
377 for line in sys.stdin:
378 (oldrev, newrev, refname) = line.split()
379 if refname == "refs/heads/master":
381 oldrev=os.popen('git show-ref --hash refs/remotes/ftp/master').read().strip()
382 if not oldrev: raise IOError, "No ref" # Simulate failure if the branch doesn't exist
383 git_command = "git diff --name-status %s %s"%(oldrev, newrev)
385 # We are run for the first time, so (A)dd all files in the repo.
387 git_command = r"git ls-tree -r --name-only %s | sed -e 's/\(.*\)/A \1/'"%(newrev);
389 add_to_change_list(changes, git_command, oldrev, newrev, url.path)
394 git_command = r"git ls-tree -r --name-only HEAD | sed -e 's/\(.*\)/A \1/'";
395 add_to_change_list(changes, git_command, oldrev, newrev)
398 perror("No changes to sync")
403 if url.scheme == "ftp":
404 syncer = FTPSync(FTP(options.host, options.user, options.password))
405 elif url.scheme == "file":
407 elif url.scheme == "sftp":
408 syncer = SFTPSync(url)
410 for change in changes:
411 syncer.sync(change, options.dir)
412 syncer.delete_empty_directories(changes, newrev, options.dir)
416 # except ftplib.all_errors, detail:
417 # perror("FTP synchronization error: %s" % detail);
418 # perror("I will try it next time again");
423 # If succeessfull, update remote ref