build/upload.py

   1 #!/usr/bin/python
   2 #
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6 #
   7 # When run directly, this script expects the following environment variables
   8 # to be set:
   9 # UPLOAD_HOST    : host to upload files to
  10 # UPLOAD_USER    : username on that host
  11 # UPLOAD_PATH    : path on that host to put the files in
  12 #
  13 # And will use the following optional environment variables if set:
  14 # UPLOAD_SSH_KEY : path to a ssh private key to use
  15 # UPLOAD_PORT    : port to use for ssh
  16 # POST_UPLOAD_CMD: a commandline to run on the remote host after uploading.
  17 #                  UPLOAD_PATH and the full paths of all files uploaded will
  18 #                  be appended to the commandline.
  19 #
  20 # All files to be uploaded should be passed as commandline arguments to this
  21 # script. The script takes one other parameter, --base-path, which you can use
  22 # to indicate that files should be uploaded including their paths relative
  23 # to the base path.
  24
  25 import sys, os
  26 from optparse import OptionParser
  27 from subprocess import PIPE, Popen, check_call
  28
  29 def RequireEnvironmentVariable(v):
  30     """Return the value of the environment variable named v, or print
  31     an error and exit if it's unset (or empty)."""
  32     if not v in os.environ or os.environ[v] == "":
  33         print "Error: required environment variable %s not set" % v
  34         sys.exit(1)
  35     return os.environ[v]
  36
  37 def OptionalEnvironmentVariable(v):
  38     """Return the value of the environment variable named v, or None
  39     if it's unset (or empty)."""
  40     if v in os.environ and os.environ[v] != "":
  41         return os.environ[v]
  42     return None
  43
  44 def FixupMsysPath(path):
  45     """MSYS helpfully translates absolute pathnames in environment variables
  46     and commandline arguments into Windows native paths. This sucks if you're
  47     trying to pass an absolute path on a remote server. This function attempts
  48     to un-mangle such paths."""
  49     if 'OSTYPE' in os.environ and os.environ['OSTYPE'] == 'msys':
  50         # sort of awful, find out where our shell is (should be in msys/bin)
  51         # and strip the first part of that path out of the other path
  52         if 'SHELL' in os.environ:
  53             sh = os.environ['SHELL']
  54             msys = sh[:sh.find('/bin')]
  55             if path.startswith(msys):
  56                 path = path[len(msys):]
  57     return path
  58
  59 def WindowsPathToMsysPath(path):
  60     """Translate a Windows pathname to an MSYS pathname.
  61     Necessary because we call out to ssh/scp, which are MSYS binaries
  62     and expect MSYS paths."""
  63     if sys.platform != 'win32':
  64         return path
  65     (drive, path) = os.path.splitdrive(os.path.abspath(path))
  66     return "/" + drive[0] + path.replace('\\','/')
  67
  68 def AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key):
  69     """Given optional port and ssh key values, append valid OpenSSH
  70     commandline arguments to the list cmdline if the values are not None."""
  71     if port is not None:
  72         cmdline.append("-P%d" % port)
  73     if ssh_key is not None:
  74         # Don't interpret ~ paths - ssh can handle that on its own
  75         if not ssh_key.startswith('~'):
  76             ssh_key = WindowsPathToMsysPath(ssh_key)
  77         cmdline.extend(["-o", "IdentityFile=%s" % ssh_key])
  78
  79 def DoSSHCommand(command, user, host, port=None, ssh_key=None):
  80     """Execute command on user@host using ssh. Optionally use
  81     port and ssh_key, if provided."""
  82     cmdline = ["ssh"]
  83     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
  84     cmdline.extend(["%s@%s" % (user, host), command])
  85     cmd = Popen(cmdline, stdout=PIPE)
  86     retcode = cmd.wait()
  87     if retcode != 0:
  88         raise Exception("Command %s returned non-zero exit code: %i" % \
  89           (cmdline, retcode))
  90     return cmd.stdout.read().strip()
  91
  92 def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None):
  93     """Upload file to user@host:remote_path using scp. Optionally use
  94     port and ssh_key, if provided."""
  95     cmdline = ["scp"]
  96     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
  97     cmdline.extend([WindowsPathToMsysPath(file),
  98                     "%s@%s:%s" % (user, host, remote_path)])
  99     check_call(cmdline)
 100
 101 def GetRemotePath(path, local_file, base_path):
 102     """Given a remote path to upload to, a full path to a local file, and an
 103     optional full path that is a base path of the local file, construct the
 104     full remote path to place the file in. If base_path is not None, include
 105     the relative path from base_path to file."""
 106     if base_path is None or not local_file.startswith(base_path):
 107         return path
 108     dir = os.path.dirname(local_file)
 109     # strip base_path + extra slash and make it unixy
 110     dir = dir[len(base_path)+1:].replace('\\','/')
 111     return path + dir
 112
 113 def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None, upload_to_temp_dir=False, post_upload_command=None):
 114     """Upload each file in the list files to user@host:path. Optionally pass
 115     port and ssh_key to the ssh commands. If base_path is not None, upload
 116     files including their path relative to base_path. If upload_to_temp_dir is
 117     True files will be uploaded to a temporary directory on the remote server.
 118     Generally, you should have a post upload command specified in these cases
 119     that can move them around to their correct location(s).
 120     If post_upload_command is not None, execute that command on the remote host
 121     after uploading all files, passing it the upload path, and the full paths to
 122     all files uploaded.
 123     If verbose is True, print status updates while working."""
 124     if upload_to_temp_dir:
 125         path = DoSSHCommand("mktemp -d", user, host, port=port, ssh_key=ssh_key)
 126     if not path.endswith("/"):
 127         path += "/"
 128     if base_path is not None:
 129         base_path = os.path.abspath(base_path)
 130     remote_files = []
 131     try:
 132         for file in files:
 133             file = os.path.abspath(file)
 134             if not os.path.isfile(file):
 135                 raise IOError("File not found: %s" % file)
 136             # first ensure that path exists remotely
 137             remote_path = GetRemotePath(path, file, base_path)
 138             DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key)
 139             if verbose:
 140                 print "Uploading " + file
 141             DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key)
 142             remote_files.append(remote_path + '/' + os.path.basename(file))
 143         if post_upload_command is not None:
 144             if verbose:
 145                 print "Running post-upload command: " + post_upload_command
 146             file_list = '"' + '" "'.join(remote_files) + '"'
 147             DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key)
 148     finally:
 149         if upload_to_temp_dir:
 150             DoSSHCommand("rm -rf %s" % path, user, host, port=port,
 151                          ssh_key=ssh_key)
 152     if verbose:
 153         print "Upload complete"
 154
 155 if __name__ == '__main__':
 156     host = RequireEnvironmentVariable('UPLOAD_HOST')
 157     user = RequireEnvironmentVariable('UPLOAD_USER')
 158     path = OptionalEnvironmentVariable('UPLOAD_PATH')
 159     upload_to_temp_dir = OptionalEnvironmentVariable('UPLOAD_TO_TEMP')
 160     port = OptionalEnvironmentVariable('UPLOAD_PORT')
 161     if port is not None:
 162         port = int(port)
 163     key = OptionalEnvironmentVariable('UPLOAD_SSH_KEY')
 164     post_upload_command = OptionalEnvironmentVariable('POST_UPLOAD_CMD')
 165     if (not path and not upload_to_temp_dir) or (path and upload_to_temp_dir):
 166         print "One (and only one of UPLOAD_PATH or UPLOAD_TO_TEMP must be " + \
 167               "defined."
 168         sys.exit(1)
 169     if sys.platform == 'win32':
 170         if path is not None:
 171             path = FixupMsysPath(path)
 172         if post_upload_command is not None:
 173             post_upload_command = FixupMsysPath(post_upload_command)
 174
 175     parser = OptionParser(usage="usage: %prog [options] <files>")
 176     parser.add_option("-b", "--base-path",
 177                       action="store", dest="base_path",
 178                       help="Preserve file paths relative to this path when uploading. If unset, all files will be uploaded directly to UPLOAD_PATH.")
 179     (options, args) = parser.parse_args()
 180     if len(args) < 1:
 181         print "You must specify at least one file to upload"
 182         sys.exit(1)
 183     try:
 184         UploadFiles(user, host, path, args, base_path=options.base_path,
 185                     port=port, ssh_key=key, upload_to_temp_dir=upload_to_temp_dir,
 186                     post_upload_command=post_upload_command,
 187                     verbose=True)
 188     except IOError, (strerror):
 189         print strerror
 190         sys.exit(1)
 191     except Exception, (err):
 192         print err
 193         sys.exit(2)