build/upload.py

   1 #!/usr/bin/python
   2 #
   3 # ***** BEGIN LICENSE BLOCK *****
   4 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   5 #
   6 # The contents of this file are subject to the Mozilla Public License Version
   7 # 1.1 (the "License"); you may not use this file except in compliance with
   8 # the License. You may obtain a copy of the License at
   9 # http://www.mozilla.org/MPL/
  10 #
  11 # Software distributed under the License is distributed on an "AS IS" basis,
  12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  13 # for the specific language governing rights and limitations under the
  14 # License.
  15 #
  16 # The Original Code is mozilla.org code.
  17 #
  18 # The Initial Developer of the Original Code is
  19 # The Mozilla Foundation
  20 # Portions created by the Initial Developer are Copyright (C) 2008
  21 # the Initial Developer. All Rights Reserved.
  22 #
  23 # Contributor(s):
  24 #  Ted Mielczarek <ted.mielczarek@gmail.com>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39 #
  40 # When run directly, this script expects the following environment variables
  41 # to be set:
  42 # UPLOAD_HOST    : host to upload files to
  43 # UPLOAD_USER    : username on that host
  44 # UPLOAD_PATH    : path on that host to put the files in
  45 #
  46 # And will use the following optional environment variables if set:
  47 # UPLOAD_SSH_KEY : path to a ssh private key to use
  48 # UPLOAD_PORT    : port to use for ssh
  49 # POST_UPLOAD_CMD: a commandline to run on the remote host after uploading.
  50 #                  UPLOAD_PATH and the full paths of all files uploaded will
  51 #                  be appended to the commandline.
  52 #
  53 # All files to be uploaded should be passed as commandline arguments to this
  54 # script. The script takes one other parameter, --base-path, which you can use
  55 # to indicate that files should be uploaded including their paths relative
  56 # to the base path.
  57
  58 import sys, os
  59 from optparse import OptionParser
  60 from subprocess import PIPE, Popen, check_call
  61
  62 def RequireEnvironmentVariable(v):
  63     """Return the value of the environment variable named v, or print
  64     an error and exit if it's unset (or empty)."""
  65     if not v in os.environ or os.environ[v] == "":
  66         print "Error: required environment variable %s not set" % v
  67         sys.exit(1)
  68     return os.environ[v]
  69
  70 def OptionalEnvironmentVariable(v):
  71     """Return the value of the environment variable named v, or None
  72     if it's unset (or empty)."""
  73     if v in os.environ and os.environ[v] != "":
  74         return os.environ[v]
  75     return None
  76
  77 def FixupMsysPath(path):
  78     """MSYS helpfully translates absolute pathnames in environment variables
  79     and commandline arguments into Windows native paths. This sucks if you're
  80     trying to pass an absolute path on a remote server. This function attempts
  81     to un-mangle such paths."""
  82     if 'OSTYPE' in os.environ and os.environ['OSTYPE'] == 'msys':
  83         # sort of awful, find out where our shell is (should be in msys/bin)
  84         # and strip the first part of that path out of the other path
  85         if 'SHELL' in os.environ:
  86             sh = os.environ['SHELL']
  87             msys = sh[:sh.find('/bin')]
  88             if path.startswith(msys):
  89                 path = path[len(msys):]
  90     return path
  91
  92 def WindowsPathToMsysPath(path):
  93     """Translate a Windows pathname to an MSYS pathname.
  94     Necessary because we call out to ssh/scp, which are MSYS binaries
  95     and expect MSYS paths."""
  96     if sys.platform != 'win32':
  97         return path
  98     (drive, path) = os.path.splitdrive(os.path.abspath(path))
  99     return "/" + drive[0] + path.replace('\\','/')
 100
 101 def AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key):
 102     """Given optional port and ssh key values, append valid OpenSSH
 103     commandline arguments to the list cmdline if the values are not None."""
 104     if port is not None:
 105         cmdline.append("-P%d" % port)
 106     if ssh_key is not None:
 107         # Don't interpret ~ paths - ssh can handle that on its own
 108         if not ssh_key.startswith('~'):
 109             ssh_key = WindowsPathToMsysPath(ssh_key)
 110         cmdline.extend(["-o", "IdentityFile=%s" % ssh_key])
 111
 112 def DoSSHCommand(command, user, host, port=None, ssh_key=None):
 113     """Execute command on user@host using ssh. Optionally use
 114     port and ssh_key, if provided."""
 115     cmdline = ["ssh"]
 116     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
 117     cmdline.extend(["%s@%s" % (user, host), command])
 118     cmd = Popen(cmdline, stdout=PIPE)
 119     retcode = cmd.wait()
 120     if retcode != 0:
 121         raise Exception("Command %s returned non-zero exit code: %i" % \
 122           (cmdline, retcode))
 123     return cmd.stdout.read().strip()
 124
 125 def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None):
 126     """Upload file to user@host:remote_path using scp. Optionally use
 127     port and ssh_key, if provided."""
 128     cmdline = ["scp"]
 129     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
 130     cmdline.extend([WindowsPathToMsysPath(file),
 131                     "%s@%s:%s" % (user, host, remote_path)])
 132     check_call(cmdline)
 133
 134 def GetRemotePath(path, local_file, base_path):
 135     """Given a remote path to upload to, a full path to a local file, and an
 136     optional full path that is a base path of the local file, construct the
 137     full remote path to place the file in. If base_path is not None, include
 138     the relative path from base_path to file."""
 139     if base_path is None or not local_file.startswith(base_path):
 140         return path
 141     dir = os.path.dirname(local_file)
 142     # strip base_path + extra slash and make it unixy
 143     dir = dir[len(base_path)+1:].replace('\\','/')
 144     return path + dir
 145
 146 def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None, upload_to_temp_dir=False, post_upload_command=None):
 147     """Upload each file in the list files to user@host:path. Optionally pass
 148     port and ssh_key to the ssh commands. If base_path is not None, upload
 149     files including their path relative to base_path. If upload_to_temp_dir is
 150     True files will be uploaded to a temporary directory on the remote server.
 151     Generally, you should have a post upload command specified in these cases
 152     that can move them around to their correct location(s).
 153     If post_upload_command is not None, execute that command on the remote host
 154     after uploading all files, passing it the upload path, and the full paths to
 155     all files uploaded.
 156     If verbose is True, print status updates while working."""
 157     if upload_to_temp_dir:
 158         path = DoSSHCommand("mktemp -d", user, host, port=port, ssh_key=ssh_key)
 159     if not path.endswith("/"):
 160         path += "/"
 161     if base_path is not None:
 162         base_path = os.path.abspath(base_path)
 163     remote_files = []
 164     try:
 165         for file in files:
 166             file = os.path.abspath(file)
 167             if not os.path.isfile(file):
 168                 raise IOError("File not found: %s" % file)
 169             # first ensure that path exists remotely
 170             remote_path = GetRemotePath(path, file, base_path)
 171             DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key)
 172             if verbose:
 173                 print "Uploading " + file
 174             DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key)
 175             remote_files.append(remote_path + '/' + os.path.basename(file))
 176         if post_upload_command is not None:
 177             if verbose:
 178                 print "Running post-upload command: " + post_upload_command
 179             file_list = '"' + '" "'.join(remote_files) + '"'
 180             DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key)
 181     finally:
 182         if upload_to_temp_dir:
 183             DoSSHCommand("rm -rf %s" % path, user, host, port=port,
 184                          ssh_key=ssh_key)
 185     if verbose:
 186         print "Upload complete"
 187
 188 if __name__ == '__main__':
 189     host = RequireEnvironmentVariable('UPLOAD_HOST')
 190     user = RequireEnvironmentVariable('UPLOAD_USER')
 191     path = OptionalEnvironmentVariable('UPLOAD_PATH')
 192     upload_to_temp_dir = OptionalEnvironmentVariable('UPLOAD_TO_TEMP')
 193     port = OptionalEnvironmentVariable('UPLOAD_PORT')
 194     if port is not None:
 195         port = int(port)
 196     key = OptionalEnvironmentVariable('UPLOAD_SSH_KEY')
 197     post_upload_command = OptionalEnvironmentVariable('POST_UPLOAD_CMD')
 198     if (not path and not upload_to_temp_dir) or (path and upload_to_temp_dir):
 199         print "One (and only one of UPLOAD_PATH or UPLOAD_TO_TEMP must be " + \
 200               "defined."
 201         sys.exit(1)
 202     if sys.platform == 'win32':
 203         if path is not None:
 204             path = FixupMsysPath(path)
 205         if post_upload_command is not None:
 206             post_upload_command = FixupMsysPath(post_upload_command)
 207
 208     parser = OptionParser(usage="usage: %prog [options] <files>")
 209     parser.add_option("-b", "--base-path",
 210                       action="store", dest="base_path",
 211                       help="Preserve file paths relative to this path when uploading. If unset, all files will be uploaded directly to UPLOAD_PATH.")
 212     (options, args) = parser.parse_args()
 213     if len(args) < 1:
 214         print "You must specify at least one file to upload"
 215         sys.exit(1)
 216     try:
 217         UploadFiles(user, host, path, args, base_path=options.base_path,
 218                     port=port, ssh_key=key, upload_to_temp_dir=upload_to_temp_dir,
 219                     post_upload_command=post_upload_command,
 220                     verbose=True)
 221     except IOError, (strerror):
 222         print strerror
 223         sys.exit(1)
 224     except Exception, (err):
 225         print err
 226         sys.exit(2)