build/upload.py

   1 #!/usr/bin/python
   2 #
   3 # ***** BEGIN LICENSE BLOCK *****
   4 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   5 #
   6 # The contents of this file are subject to the Mozilla Public License Version
   7 # 1.1 (the "License"); you may not use this file except in compliance with
   8 # the License. You may obtain a copy of the License at
   9 # http://www.mozilla.org/MPL/
  10 #
  11 # Software distributed under the License is distributed on an "AS IS" basis,
  12 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  13 # for the specific language governing rights and limitations under the
  14 # License.
  15 #
  16 # The Original Code is mozilla.org code.
  17 #
  18 # The Initial Developer of the Original Code is
  19 # The Mozilla Foundation
  20 # Portions created by the Initial Developer are Copyright (C) 2008
  21 # the Initial Developer. All Rights Reserved.
  22 #
  23 # Contributor(s):
  24 #  Ted Mielczarek <ted.mielczarek@gmail.com>
  25 #
  26 # Alternatively, the contents of this file may be used under the terms of
  27 # either the GNU General Public License Version 2 or later (the "GPL"), or
  28 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  29 # in which case the provisions of the GPL or the LGPL are applicable instead
  30 # of those above. If you wish to allow use of your version of this file only
  31 # under the terms of either the GPL or the LGPL, and not to allow others to
  32 # use your version of this file under the terms of the MPL, indicate your
  33 # decision by deleting the provisions above and replace them with the notice
  34 # and other provisions required by the GPL or the LGPL. If you do not delete
  35 # the provisions above, a recipient may use your version of this file under
  36 # the terms of any one of the MPL, the GPL or the LGPL.
  37 #
  38 # ***** END LICENSE BLOCK *****
  39 #
  40 # When run directly, this script expects the following environment variables
  41 # to be set:
  42 # UPLOAD_HOST    : host to upload files to
  43 # UPLOAD_USER    : username on that host
  44 # UPLOAD_PATH    : path on that host to put the files in
  45 #
  46 # And will use the following optional environment variables if set:
  47 # UPLOAD_SSH_KEY : path to a ssh private key to use
  48 # UPLOAD_PORT    : port to use for ssh
  49 # POST_UPLOAD_CMD: a commandline to run on the remote host after uploading.
  50 #                  UPLOAD_PATH and the full paths of all files uploaded will
  51 #                  be appended to the commandline.
  52 #
  53 # All files to be uploaded should be passed as commandline arguments to this
  54 # script. The script takes one other parameter, --base-path, which you can use
  55 # to indicate that files should be uploaded including their paths relative
  56 # to the base path.
  57
  58 import sys, os
  59 from optparse import OptionParser
  60 from subprocess import Popen, PIPE
  61 from util import check_call
  62
  63 def RequireEnvironmentVariable(v):
  64     """Return the value of the environment variable named v, or print
  65     an error and exit if it's unset (or empty)."""
  66     if not v in os.environ or os.environ[v] == "":
  67         print "Error: required environment variable %s not set" % v
  68         sys.exit(1)
  69     return os.environ[v]
  70
  71 def OptionalEnvironmentVariable(v):
  72     """Return the value of the environment variable named v, or None
  73     if it's unset (or empty)."""
  74     if v in os.environ and os.environ[v] != "":
  75         return os.environ[v]
  76     return None
  77
  78 def FixupMsysPath(path):
  79     """MSYS helpfully translates absolute pathnames in environment variables
  80     and commandline arguments into Windows native paths. This sucks if you're
  81     trying to pass an absolute path on a remote server. This function attempts
  82     to un-mangle such paths."""
  83     if 'OSTYPE' in os.environ and os.environ['OSTYPE'] == 'msys':
  84         # sort of awful, find out where our shell is (should be in msys/bin)
  85         # and strip the first part of that path out of the other path
  86         if 'SHELL' in os.environ:
  87             sh = os.environ['SHELL']
  88             msys = sh[:sh.find('/bin')]
  89             if path.startswith(msys):
  90                 path = path[len(msys):]
  91     return path
  92
  93 def WindowsPathToMsysPath(path):
  94     """Translate a Windows pathname to an MSYS pathname.
  95     Necessary because we call out to ssh/scp, which are MSYS binaries
  96     and expect MSYS paths."""
  97     if sys.platform != 'win32':
  98         return path
  99     (drive, path) = os.path.splitdrive(os.path.abspath(path))
 100     return "/" + drive[0] + path.replace('\\','/')
 101
 102 def AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key):
 103     """Given optional port and ssh key values, append valid OpenSSH
 104     commandline arguments to the list cmdline if the values are not None."""
 105     if port is not None:
 106         cmdline.append("-P%d" % port)
 107     if ssh_key is not None:
 108         # Don't interpret ~ paths - ssh can handle that on its own
 109         if not ssh_key.startswith('~'):
 110             ssh_key = WindowsPathToMsysPath(ssh_key)
 111         cmdline.extend(["-o", "IdentityFile=%s" % ssh_key])
 112
 113 def DoSSHCommand(command, user, host, port=None, ssh_key=None):
 114     """Execute command on user@host using ssh. Optionally use
 115     port and ssh_key, if provided."""
 116     cmdline = ["ssh"]
 117     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
 118     cmdline.extend(["%s@%s" % (user, host), command])
 119     cmd = Popen(cmdline, stdout=PIPE)
 120     retcode = cmd.wait()
 121     if retcode != 0:
 122         raise Exception("Command %s returned non-zero exit code: %i" % \
 123           (cmdline, retcode))
 124     return cmd.stdout.read().strip()
 125
 126 def DoSCPFile(file, remote_path, user, host, port=None, ssh_key=None):
 127     """Upload file to user@host:remote_path using scp. Optionally use
 128     port and ssh_key, if provided."""
 129     cmdline = ["scp"]
 130     AppendOptionalArgsToSSHCommandline(cmdline, port, ssh_key)
 131     cmdline.extend([WindowsPathToMsysPath(file),
 132                     "%s@%s:%s" % (user, host, remote_path)])
 133     check_call(cmdline)
 134
 135 def GetRemotePath(path, local_file, base_path):
 136     """Given a remote path to upload to, a full path to a local file, and an
 137     optional full path that is a base path of the local file, construct the
 138     full remote path to place the file in. If base_path is not None, include
 139     the relative path from base_path to file."""
 140     if base_path is None or not local_file.startswith(base_path):
 141         return path
 142     dir = os.path.dirname(local_file)
 143     # strip base_path + extra slash and make it unixy
 144     dir = dir[len(base_path)+1:].replace('\\','/')
 145     return path + dir
 146
 147 def UploadFiles(user, host, path, files, verbose=False, port=None, ssh_key=None, base_path=None, upload_to_temp_dir=False, post_upload_command=None):
 148     """Upload each file in the list files to user@host:path. Optionally pass
 149     port and ssh_key to the ssh commands. If base_path is not None, upload
 150     files including their path relative to base_path. If upload_to_temp_dir is
 151     True files will be uploaded to a temporary directory on the remote server.
 152     Generally, you should have a post upload command specified in these cases
 153     that can move them around to their correct location(s).
 154     If post_upload_command is not None, execute that command on the remote host
 155     after uploading all files, passing it the upload path, and the full paths to
 156     all files uploaded.
 157     If verbose is True, print status updates while working."""
 158     if upload_to_temp_dir:
 159         path = DoSSHCommand("mktemp -d", user, host, port=port, ssh_key=ssh_key)
 160     if not path.endswith("/"):
 161         path += "/"
 162     if base_path is not None:
 163         base_path = os.path.abspath(base_path)
 164     remote_files = []
 165     try:
 166         for file in files:
 167             file = os.path.abspath(file)
 168             if not os.path.isfile(file):
 169                 raise IOError("File not found: %s" % file)
 170             # first ensure that path exists remotely
 171             remote_path = GetRemotePath(path, file, base_path)
 172             DoSSHCommand("mkdir -p " + remote_path, user, host, port=port, ssh_key=ssh_key)
 173             if verbose:
 174                 print "Uploading " + file
 175             DoSCPFile(file, remote_path, user, host, port=port, ssh_key=ssh_key)
 176             remote_files.append(remote_path + '/' + os.path.basename(file))
 177         if post_upload_command is not None:
 178             if verbose:
 179                 print "Running post-upload command: " + post_upload_command
 180             file_list = '"' + '" "'.join(remote_files) + '"'
 181             DoSSHCommand('%s "%s" %s' % (post_upload_command, path, file_list), user, host, port=port, ssh_key=ssh_key)
 182     except:
 183         print "Encountered error while uploading"
 184         raise
 185     finally:
 186         if upload_to_temp_dir:
 187             DoSSHCommand("rm -rf %s" % path, user, host, port=port,
 188                          ssh_key=ssh_key)
 189     if verbose:
 190         print "Upload complete"
 191
 192 if __name__ == '__main__':
 193     host = RequireEnvironmentVariable('UPLOAD_HOST')
 194     user = RequireEnvironmentVariable('UPLOAD_USER')
 195     path = OptionalEnvironmentVariable('UPLOAD_PATH')
 196     upload_to_temp_dir = OptionalEnvironmentVariable('UPLOAD_TO_TEMP')
 197     port = OptionalEnvironmentVariable('UPLOAD_PORT')
 198     if port is not None:
 199         port = int(port)
 200     key = OptionalEnvironmentVariable('UPLOAD_SSH_KEY')
 201     post_upload_command = OptionalEnvironmentVariable('POST_UPLOAD_CMD')
 202     if (not path and not upload_to_temp_dir) or (path and upload_to_temp_dir):
 203         print "One (and only one of UPLOAD_PATH or UPLOAD_TO_TEMP must be " + \
 204               "defined."
 205         sys.exit(1)
 206     if sys.platform == 'win32':
 207         if path is not None:
 208             path = FixupMsysPath(path)
 209         if post_upload_command is not None:
 210             post_upload_command = FixupMsysPath(post_upload_command)
 211
 212     parser = OptionParser(usage="usage: %prog [options] <files>")
 213     parser.add_option("-b", "--base-path",
 214                       action="store", dest="base_path",
 215                       help="Preserve file paths relative to this path when uploading. If unset, all files will be uploaded directly to UPLOAD_PATH.")
 216     (options, args) = parser.parse_args()
 217     if len(args) < 1:
 218         print "You must specify at least one file to upload"
 219         sys.exit(1)
 220     try:
 221         UploadFiles(user, host, path, args, base_path=options.base_path,
 222                     port=port, ssh_key=key, upload_to_temp_dir=upload_to_temp_dir,
 223                     post_upload_command=post_upload_command,
 224                     verbose=True)
 225     except IOError, (strerror):
 226         print strerror
 227     except Exception, (err):
 228         print err