utils/common/deploy.py

   1 #!/usr/bin/python
   2 #             __________               __   ___.
   3 #   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4 #   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5 #   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6 #   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7 #                     \/            \/     \/    \/            \/
   8 # $Id$
   9 #
  10 # Copyright (c) 2009 Dominik Riebeling
  11 #
  12 # All files in this archive are subject to the GNU General Public License.
  13 # See the file COPYING in the source tree root for full license agreement.
  14 #
  15 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  16 # KIND, either express or implied.
  17 #
  18 #
  19 # Automate building releases for deployment.
  20 # Run from any folder to build
  21 # - trunk
  22 # - any tag (using the -t option)
  23 # - any local folder (using the -p option)
  24 # Will build a binary archive (tar.bz2 / zip) and source archive.
  25 # The source archive won't be built for local builds. Trunk and
  26 # tag builds will retrieve the sources directly from svn and build
  27 # below the systems temporary folder.
  28 #
  29 # If the required Qt installation isn't in PATH use --qmake option.
  30 # Tested on Linux and MinGW / W32
  31 #
  32 # requires python which package (http://code.google.com/p/which/)
  33 # requires pysvn package.
  34 # requires upx.exe in PATH on Windows.
  35 #
  36
  37 import re
  38 import os
  39 import sys
  40 import tarfile
  41 import zipfile
  42 import shutil
  43 import subprocess
  44 import getopt
  45 import time
  46 import hashlib
  47 import tempfile
  48
  49 # modules that are not part of python itself.
  50 try:
  51     import pysvn
  52 except ImportError:
  53     print "Fatal: This script requires the pysvn package to run."
  54     print "       See http://pysvn.tigris.org/."
  55     sys.exit(-5)
  56 try:
  57     import which
  58 except ImportError:
  59     print "Fatal: This script requires the which package to run."
  60     print "       See http://code.google.com/p/which/."
  61     sys.exit(-5)
  62
  63 # == Global stuff ==
  64 # Windows nees some special treatment. Differentiate between program name
  65 # and executable filename.
  66 program = ""
  67 project = ""
  68 environment = os.environ
  69 progexe = ""
  70 make = "make"
  71 programfiles = []
  72
  73 svnserver = ""
  74 # Paths and files to retrieve from svn when creating a tarball.
  75 # This is a mixed list, holding both paths and filenames.
  76 svnpaths = [ ]
  77 # set this to true to run upx on the resulting binary, false to skip this step.
  78 # only used on w32.
  79 useupx = False
  80
  81 # OS X: files to copy into the bundle. Workaround for out-of-tree builds.
  82 bundlecopy = { }
  83
  84 # == Functions ==
  85 def usage(myself):
  86     print "Usage: %s [options]" % myself
  87     print "       -q, --qmake=<qmake>   path to qmake"
  88     print "       -p, --project=<pro>   path to .pro file for building with local tree"
  89     print "       -t, --tag=<tag>       use specified tag from svn"
  90     print "       -a, --add=<file>      add file to build folder before building"
  91     print "       -s, --source-only     only create source archive"
  92     print "       -b, --binary-only     only create binary archive"
  93     if sys.platform != "darwin":
  94         print "       -d, --dynamic         link dynamically instead of static"
  95     print "       -k, --keep-temp       keep temporary folder on build failure"
  96     print "       -h, --help            this help"
  97     print "  If neither a project file nor tag is specified trunk will get downloaded"
  98     print "  from svn."
  99
 100 def getsources(svnsrv, filelist, dest):
 101     '''Get the files listed in filelist from svnsrv and put it at dest.'''
 102     client = pysvn.Client()
 103     print "Checking out sources from %s, please wait." % svnsrv
 104
 105     for elem in filelist:
 106         url = re.subn('/$', '', svnsrv + elem)[0]
 107         destpath = re.subn('/$', '', dest + elem)[0]
 108         # make sure the destination path does exist
 109         d = os.path.dirname(destpath)
 110         if not os.path.exists(d):
 111             os.makedirs(d)
 112         # get from svn
 113         try:
 114             client.export(url, destpath)
 115         except:
 116             print "SVN client error: %s" % sys.exc_value
 117             print "URL: %s, destination: %s" % (url, destpath)
 118             return -1
 119     print "Checkout finished."
 120     return 0
 121
 122
 123 def gettrunkrev(svnsrv):
 124     '''Get the revision of trunk for svnsrv'''
 125     client = pysvn.Client()
 126     entries = client.info2(svnsrv, recurse=False)
 127     return entries[0][1].rev.number
 128
 129
 130 def findversion(versionfile):
 131     '''figure most recent program version from version.h,
 132     returns version string.'''
 133     h = open(versionfile, "r")
 134     c = h.read()
 135     h.close()
 136     r = re.compile("#define +VERSION +\"(.[0-9\.a-z]+)\"")
 137     m = re.search(r, c)
 138     s = re.compile("\$Revision: +([0-9]+)")
 139     n = re.search(s, c)
 140     if n == None:
 141         print "WARNING: Revision not found!"
 142     return m.group(1)
 143
 144
 145 def findqt():
 146     '''Search for Qt4 installation. Return path to qmake.'''
 147     print "Searching for Qt"
 148     bins = ["qmake", "qmake-qt4"]
 149     for binary in bins:
 150         try:
 151             q = which.which(binary)
 152             if len(q) > 0:
 153                 result = checkqt(q)
 154                 if not result == "":
 155                     return result
 156         except:
 157             print sys.exc_value
 158
 159     return ""
 160
 161
 162 def checkqt(qmakebin):
 163     '''Check if given path to qmake exists and is a suitable version.'''
 164     result = ""
 165     # check if binary exists
 166     if not os.path.exists(qmakebin):
 167         print "Specified qmake path does not exist!"
 168         return result
 169     # check version
 170     output = subprocess.Popen([qmakebin, "-version"], stdout=subprocess.PIPE,
 171             stderr=subprocess.PIPE)
 172     cmdout = output.communicate()
 173     # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
 174     for ou in cmdout:
 175         r = re.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
 176         m = re.search(r, ou)
 177         if not m == None:
 178             print "Qt found: %s" % m.group(1)
 179             s = re.compile("4\..*")
 180             n = re.search(s, m.group(1))
 181             if not n == None:
 182                 result = qmakebin
 183     return result
 184
 185
 186 def qmake(qmake="qmake", projfile=project, wd=".", static=True):
 187     print "Running qmake in %s..." % wd
 188     command = [qmake, "-config", "release", "-config", "noccache"]
 189     if static == True:
 190         command.append("-config")
 191         command.append("static")
 192     command.append(projfile)
 193     output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd, env=environment)
 194     output.communicate()
 195     if not output.returncode == 0:
 196         print "qmake returned an error!"
 197         return -1
 198     return 0
 199
 200
 201 def build(wd="."):
 202     # make
 203     print "Building ..."
 204     output = subprocess.Popen([make], stdout=subprocess.PIPE, cwd=wd)
 205     while True:
 206         c = output.stdout.readline()
 207         sys.stdout.write(".")
 208         sys.stdout.flush()
 209         if not output.poll() == None:
 210             sys.stdout.write("\n")
 211             sys.stdout.flush()
 212             if not output.returncode == 0:
 213                 print "Build failed!"
 214                 return -1
 215             break
 216     if sys.platform != "darwin":
 217         # strip. OS X handles this via macdeployqt.
 218         print "Stripping binary."
 219         output = subprocess.Popen(["strip", progexe], stdout=subprocess.PIPE, cwd=wd)
 220         output.communicate()
 221         if not output.returncode == 0:
 222             print "Stripping failed!"
 223             return -1
 224     return 0
 225
 226
 227 def upxfile(wd="."):
 228     # run upx on binary
 229     print "UPX'ing binary ..."
 230     output = subprocess.Popen(["upx", progexe], stdout=subprocess.PIPE, cwd=wd)
 231     output.communicate()
 232     if not output.returncode == 0:
 233         print "UPX'ing failed!"
 234         return -1
 235     return 0
 236
 237
 238 def zipball(versionstring, buildfolder):
 239     '''package created binary'''
 240     print "Creating binary zipball."
 241     archivebase = program + "-" + versionstring
 242     outfolder = buildfolder + "/" + archivebase
 243     archivename = archivebase + ".zip"
 244     # create output folder
 245     os.mkdir(outfolder)
 246     # move program files to output folder
 247     for f in programfiles:
 248         shutil.copy(buildfolder + "/" + f, outfolder)
 249     # create zipball from output folder
 250     zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED)
 251     for root, dirs, files in os.walk(outfolder):
 252         for name in files:
 253             physname = os.path.join(root, name)
 254             filename = re.sub("^" + buildfolder, "", physname)
 255             zf.write(physname, filename)
 256         for name in dirs:
 257             physname = os.path.join(root, name)
 258             filename = re.sub("^" + buildfolder, "", physname)
 259             zf.write(physname, filename)
 260     zf.close()
 261     # remove output folder
 262     shutil.rmtree(outfolder)
 263     return archivename
 264
 265
 266 def tarball(versionstring, buildfolder):
 267     '''package created binary'''
 268     print "Creating binary tarball."
 269     archivebase = program + "-" + versionstring
 270     outfolder = buildfolder + "/" + archivebase
 271     archivename = archivebase + ".tar.bz2"
 272     # create output folder
 273     os.mkdir(outfolder)
 274     # move program files to output folder
 275     for f in programfiles:
 276         shutil.copy(buildfolder + "/" + f, outfolder)
 277     # create tarball from output folder
 278     tf = tarfile.open(archivename, mode='w:bz2')
 279     tf.add(outfolder, archivebase)
 280     tf.close()
 281     # remove output folder
 282     shutil.rmtree(outfolder)
 283     return archivename
 284
 285
 286 def macdeploy(versionstring, buildfolder):
 287     '''package created binary to dmg'''
 288     dmgfile = program + "-" + versionstring + ".dmg"
 289     appbundle = buildfolder + "/" + progexe
 290
 291     # workaround to Qt issues when building out-of-tree. Copy files into bundle.
 292     sourcebase = buildfolder + re.sub('[^/]+.pro$', '', project) + "/"
 293     print sourcebase
 294     for src in bundlecopy:
 295         shutil.copy(sourcebase + src, appbundle + "/" + bundlecopy[src])
 296     # end of Qt workaround
 297
 298     output = subprocess.Popen(["macdeployqt", progexe, "-dmg"], stdout=subprocess.PIPE, cwd=buildfolder)
 299     output.communicate()
 300     if not output.returncode == 0:
 301         print "macdeployqt failed!"
 302         return -1
 303     # copy dmg to output folder
 304     shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile)
 305     return dmgfile
 306
 307 def filehashes(filename):
 308     '''Calculate md5 and sha1 hashes for a given file.'''
 309     if not os.path.exists(filename):
 310         return ["", ""]
 311     m = hashlib.md5()
 312     s = hashlib.sha1()
 313     f = open(filename, 'rb')
 314     while True:
 315         d = f.read(65536)
 316         if d == "":
 317             break
 318         m.update(d)
 319         s.update(d)
 320     return [m.hexdigest(), s.hexdigest()]
 321
 322
 323 def filestats(filename):
 324     if not os.path.exists(filename):
 325         return
 326     st = os.stat(filename)
 327     print filename, "\n", "-" * len(filename)
 328     print "Size:    %i bytes" % st.st_size
 329     h = filehashes(filename)
 330     print "md5sum:  %s" % h[0]
 331     print "sha1sum: %s" % h[1]
 332     print "-" * len(filename), "\n"
 333
 334
 335 def tempclean(workfolder, nopro):
 336     if nopro == True:
 337         print "Cleaning up working folder %s" % workfolder
 338         shutil.rmtree(workfolder)
 339     else:
 340         print "Project file specified or cleanup disabled!"
 341         print "Temporary files kept at %s" % workfolder
 342
 343
 344 def deploy():
 345     startup = time.time()
 346
 347     try:
 348         opts, args = getopt.getopt(sys.argv[1:], "q:p:t:a:sbdkh",
 349             ["qmake=", "project=", "tag=", "add=", "source-only", "binary-only", "dynamic", "keep-temp", "help"])
 350     except getopt.GetoptError, err:
 351         print str(err)
 352         usage(sys.argv[0])
 353         sys.exit(1)
 354     qt = ""
 355     proj = ""
 356     svnbase = svnserver + "trunk/"
 357     tag = ""
 358     addfiles = []
 359     cleanup = True
 360     binary = True
 361     source = True
 362     keeptemp = False
 363     if sys.platform != "darwin":
 364         static = True
 365     else:
 366         static = False
 367     for o, a in opts:
 368         if o in ("-q", "--qmake"):
 369             qt = a
 370         if o in ("-p", "--project"):
 371             proj = a
 372             cleanup = False
 373         if o in ("-t", "--tag"):
 374             tag = a
 375             svnbase = svnserver + "tags/" + tag + "/"
 376         if o in ("-a", "--add"):
 377             addfiles.append(a)
 378         if o in ("-s", "--source-only"):
 379             binary = False
 380         if o in ("-b", "--binary-only"):
 381             source = False
 382         if o in ("-d", "--dynamic") and sys.platform != "darwin":
 383             static = False
 384         if o in ("-k", "--keep-temp"):
 385             keeptemp = True
 386         if o in ("-h", "--help"):
 387             usage(sys.argv[0])
 388             sys.exit(0)
 389
 390     if source == False and binary == False:
 391         print "Building build neither source nor binary means nothing to do. Exiting."
 392         sys.exit(1)
 393
 394     # search for qmake
 395     if qt == "":
 396         qm = findqt()
 397     else:
 398         qm = checkqt(qt)
 399     if qm == "":
 400         print "ERROR: No suitable Qt installation found."
 401         sys.exit(1)
 402
 403     # create working folder. Use current directory if -p option used.
 404     if proj == "":
 405         w = tempfile.mkdtemp()
 406         # make sure the path doesn't contain backslashes to prevent issues
 407         # later when running on windows.
 408         workfolder = re.sub(r'\\', '/', w)
 409         if not tag == "":
 410             sourcefolder = workfolder + "/" + tag + "/"
 411             archivename = tag + "-src.tar.bz2"
 412             # get numeric version part from tag
 413             ver = "v" + re.sub('^[^\d]+', '', tag)
 414         else:
 415             trunk = gettrunkrev(svnbase)
 416             sourcefolder = workfolder + "/" + program + "-r" + str(trunk) + "/"
 417             archivename = program + "-r" + str(trunk) + "-src.tar.bz2"
 418             ver = "r" + str(trunk)
 419         os.mkdir(sourcefolder)
 420     else:
 421         workfolder = "."
 422         sourcefolder = "."
 423         archivename = ""
 424     # check if project file explicitly given. If yes, don't get sources from svn
 425     if proj == "":
 426         proj = sourcefolder + project
 427         # get sources and pack source tarball
 428         if not getsources(svnbase, svnpaths, sourcefolder) == 0:
 429             tempclean(workfolder, cleanup and not keeptemp)
 430             sys.exit(1)
 431
 432         if source == True:
 433             tf = tarfile.open(archivename, mode='w:bz2')
 434             tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
 435             tf.close()
 436             if binary == False:
 437                 shutil.rmtree(workfolder)
 438                 sys.exit(0)
 439     else:
 440         # figure version from sources. Need to take path to project file into account.
 441         versionfile = re.subn('[\w\.]+$', "version.h", proj)[0]
 442         ver = findversion(versionfile)
 443
 444     # check project file
 445     if not os.path.exists(proj):
 446         print "ERROR: path to project file wrong."
 447         sys.exit(1)
 448
 449     # copy specified (--add) files to working folder
 450     for f in addfiles:
 451         shutil.copy(f, sourcefolder)
 452     buildstart = time.time()
 453     header = "Building %s %s" % (program, ver)
 454     print header
 455     print len(header) * "="
 456
 457     # build it.
 458     if not qmake(qm, proj, sourcefolder, static) == 0:
 459         tempclean(workfolder, cleanup and not keeptemp)
 460         sys.exit(1)
 461     if not build(sourcefolder) == 0:
 462         tempclean(workfolder, cleanup and not keeptemp)
 463         sys.exit(1)
 464     if sys.platform == "win32":
 465         if useupx == True:
 466             if not upxfile(sourcefolder) == 0:
 467                 tempclean(workfolder, cleanup and not keeptemp)
 468                 sys.exit(1)
 469         archive = zipball(ver, sourcefolder)
 470     elif sys.platform == "darwin":
 471         archive = macdeploy(ver, sourcefolder)
 472     else:
 473         if os.uname()[4].endswith("64"):
 474             ver += "-64bit"
 475         archive = tarball(ver, sourcefolder)
 476
 477     # remove temporary files
 478     tempclean(workfolder, cleanup)
 479
 480     # display summary
 481     headline = "Build Summary for %s" % program
 482     print "\n", headline, "\n", "=" * len(headline)
 483     if not archivename == "":
 484         filestats(archivename)
 485     filestats(archive)
 486     duration = time.time() - startup
 487     building = time.time() - buildstart
 488     durmins = (int)(duration / 60)
 489     dursecs = (int)(duration % 60)
 490     buildmins = (int)(building / 60)
 491     buildsecs = (int)(building % 60)
 492     print "Overall time %smin %ssec, building took %smin %ssec." % \
 493         (durmins, dursecs, buildmins, buildsecs)
 494
 495
 496 if __name__ == "__main__":
 497     deploy()
 498