utils/common/deploy.py

   1 #!/usr/bin/python
   2 #             __________               __   ___.
   3 #   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4 #   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5 #   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6 #   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7 #                     \/            \/     \/    \/            \/
   8 # $Id$
   9 #
  10 # Copyright (c) 2009 Dominik Riebeling
  11 #
  12 # All files in this archive are subject to the GNU General Public License.
  13 # See the file COPYING in the source tree root for full license agreement.
  14 #
  15 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  16 # KIND, either express or implied.
  17 #
  18 #
  19 # Automate building releases for deployment.
  20 # Run from any folder to build
  21 # - trunk
  22 # - any tag (using the -t option)
  23 # - any local folder (using the -p option)
  24 # Will build a binary archive (tar.bz2 / zip) and source archive.
  25 # The source archive won't be built for local builds. Trunk and
  26 # tag builds will retrieve the sources directly from svn and build
  27 # below the systems temporary folder.
  28 #
  29 # If the required Qt installation isn't in PATH use --qmake option.
  30 # Tested on Linux and MinGW / W32
  31 #
  32 # requires python which package (http://code.google.com/p/which/)
  33 # requires pysvn package.
  34 # requires upx.exe in PATH on Windows.
  35 #
  36
  37 import re
  38 import os
  39 import sys
  40 import tarfile
  41 import zipfile
  42 import shutil
  43 import subprocess
  44 import getopt
  45 import time
  46 import hashlib
  47 import tempfile
  48
  49 # modules that are not part of python itself.
  50 try:
  51     import pysvn
  52 except ImportError:
  53     print "Fatal: This script requires the pysvn package to run."
  54     print "       See http://pysvn.tigris.org/."
  55     sys.exit(-5)
  56 try:
  57     import which
  58 except ImportError:
  59     print "Fatal: This script requires the which package to run."
  60     print "       See http://code.google.com/p/which/."
  61     sys.exit(-5)
  62 cpus = 1
  63 try:
  64     import multiprocessing
  65     cpus = multiprocessing.cpu_count()
  66     print "Info: %s cores found." % cpus
  67 except ImportError:
  68     print "Warning: multiprocessing module not found. Assuming 1 core."
  69
  70 # == Global stuff ==
  71 # Windows nees some special treatment. Differentiate between program name
  72 # and executable filename.
  73 program = ""
  74 project = ""
  75 environment = os.environ
  76 progexe = ""
  77 make = "make"
  78 programfiles = []
  79 nsisscript = ""
  80
  81 svnserver = ""
  82 # Paths and files to retrieve from svn when creating a tarball.
  83 # This is a mixed list, holding both paths and filenames.
  84 svnpaths = [ ]
  85 # set this to true to run upx on the resulting binary, false to skip this step.
  86 # only used on w32.
  87 useupx = False
  88
  89 # OS X: files to copy into the bundle. Workaround for out-of-tree builds.
  90 bundlecopy = { }
  91
  92 # DLL files to ignore when searching for required DLL files.
  93 systemdlls = ['advapi32.dll',
  94         'comdlg32.dll',
  95         'gdi32.dll',
  96         'imm32.dll',
  97         'kernel32.dll',
  98         'msvcrt.dll',
  99         'msvcrt.dll',
 100         'netapi32.dll',
 101         'ole32.dll',
 102         'oleaut32.dll',
 103         'setupapi.dll',
 104         'shell32.dll',
 105         'user32.dll',
 106         'winmm.dll',
 107         'winspool.drv',
 108         'ws2_32.dll']
 109
 110
 111 # == Functions ==
 112 def usage(myself):
 113     print "Usage: %s [options]" % myself
 114     print "       -q, --qmake=<qmake>   path to qmake"
 115     print "       -p, --project=<pro>   path to .pro file for building with local tree"
 116     print "       -t, --tag=<tag>       use specified tag from svn"
 117     print "       -a, --add=<file>      add file to build folder before building"
 118     print "       -s, --source-only     only create source archive"
 119     print "       -b, --binary-only     only create binary archive"
 120     if nsisscript != "":
 121         print "       -n, --makensis=<file> path to makensis for building Windows setup program."
 122     if sys.platform != "darwin":
 123         print "       -d, --dynamic         link dynamically instead of static"
 124     print "       -k, --keep-temp       keep temporary folder on build failure"
 125     print "       -h, --help            this help"
 126     print "  If neither a project file nor tag is specified trunk will get downloaded"
 127     print "  from svn."
 128
 129 def getsources(svnsrv, filelist, dest):
 130     '''Get the files listed in filelist from svnsrv and put it at dest.'''
 131     client = pysvn.Client()
 132     print "Checking out sources from %s, please wait." % svnsrv
 133
 134     for elem in filelist:
 135         url = re.subn('/$', '', svnsrv + elem)[0]
 136         destpath = re.subn('/$', '', dest + elem)[0]
 137         # make sure the destination path does exist
 138         d = os.path.dirname(destpath)
 139         if not os.path.exists(d):
 140             os.makedirs(d)
 141         # get from svn
 142         try:
 143             client.export(url, destpath)
 144         except:
 145             print "SVN client error: %s" % sys.exc_value
 146             print "URL: %s, destination: %s" % (url, destpath)
 147             return -1
 148     print "Checkout finished."
 149     return 0
 150
 151
 152 def gettrunkrev(svnsrv):
 153     '''Get the revision of trunk for svnsrv'''
 154     client = pysvn.Client()
 155     entries = client.info2(svnsrv, recurse=False)
 156     return entries[0][1].rev.number
 157
 158
 159 def findversion(versionfile):
 160     '''figure most recent program version from version.h,
 161     returns version string.'''
 162     h = open(versionfile, "r")
 163     c = h.read()
 164     h.close()
 165     r = re.compile("#define +VERSION +\"(.[0-9\.a-z]+)\"")
 166     m = re.search(r, c)
 167     s = re.compile("\$Revision: +([0-9]+)")
 168     n = re.search(s, c)
 169     if n == None:
 170         print "WARNING: Revision not found!"
 171     return m.group(1)
 172
 173
 174 def findqt():
 175     '''Search for Qt4 installation. Return path to qmake.'''
 176     print "Searching for Qt"
 177     bins = ["qmake", "qmake-qt4"]
 178     for binary in bins:
 179         try:
 180             q = which.which(binary)
 181             if len(q) > 0:
 182                 result = checkqt(q)
 183                 if not result == "":
 184                     return result
 185         except:
 186             print sys.exc_value
 187
 188     return ""
 189
 190
 191 def checkqt(qmakebin):
 192     '''Check if given path to qmake exists and is a suitable version.'''
 193     result = ""
 194     # check if binary exists
 195     if not os.path.exists(qmakebin):
 196         print "Specified qmake path does not exist!"
 197         return result
 198     # check version
 199     output = subprocess.Popen([qmakebin, "-version"], stdout=subprocess.PIPE,
 200             stderr=subprocess.PIPE)
 201     cmdout = output.communicate()
 202     # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
 203     for ou in cmdout:
 204         r = re.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
 205         m = re.search(r, ou)
 206         if not m == None:
 207             print "Qt found: %s" % m.group(1)
 208             s = re.compile("4\..*")
 209             n = re.search(s, m.group(1))
 210             if not n == None:
 211                 result = qmakebin
 212     return result
 213
 214
 215 def qmake(qmake="qmake", projfile=project, wd=".", static=True):
 216     print "Running qmake in %s..." % wd
 217     command = [qmake, "-config", "release", "-config", "noccache"]
 218     if static == True:
 219         command.append("-config")
 220         command.append("static")
 221     command.append(projfile)
 222     output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd, env=environment)
 223     output.communicate()
 224     if not output.returncode == 0:
 225         print "qmake returned an error!"
 226         return -1
 227     return 0
 228
 229
 230 def build(wd="."):
 231     # make
 232     print "Building ..."
 233     command = [make]
 234     if cpus > 1:
 235         command.append("-j")
 236         command.append(str(cpus))
 237     output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd)
 238     while True:
 239         c = output.stdout.readline()
 240         sys.stdout.write(".")
 241         sys.stdout.flush()
 242         if not output.poll() == None:
 243             sys.stdout.write("\n")
 244             sys.stdout.flush()
 245             if not output.returncode == 0:
 246                 print "Build failed!"
 247                 return -1
 248             break
 249     if sys.platform != "darwin":
 250         # strip. OS X handles this via macdeployqt.
 251         print "Stripping binary."
 252         output = subprocess.Popen(["strip", progexe], stdout=subprocess.PIPE, cwd=wd)
 253         output.communicate()
 254         if not output.returncode == 0:
 255             print "Stripping failed!"
 256             return -1
 257     return 0
 258
 259
 260 def upxfile(wd="."):
 261     # run upx on binary
 262     print "UPX'ing binary ..."
 263     output = subprocess.Popen(["upx", progexe], stdout=subprocess.PIPE, cwd=wd)
 264     output.communicate()
 265     if not output.returncode == 0:
 266         print "UPX'ing failed!"
 267         return -1
 268     return 0
 269
 270
 271 def runnsis(versionstring, nsis, script, srcfolder):
 272     # run script through nsis to create installer.
 273     print "Running NSIS ..."
 274
 275     # Assume the generated installer gets placed in the same folder the nsi
 276     # script lives in.  This seems to be a valid assumption unless the nsi
 277     # script specifies a path. NSIS expects files relative to source folder so
 278     # copy progexe. Additional files are injected into the nsis script.
 279
 280     # FIXME: instead of copying binaries around copy the NSI file and inject
 281     # the correct paths.
 282     b = srcfolder + "/" + os.path.dirname(script) + "/" + os.path.dirname(progexe)
 283     if not os.path.exists(b):
 284         os.mkdir(b)
 285     shutil.copy(srcfolder + "/" + progexe, b)
 286     output = subprocess.Popen([nsis, srcfolder + "/" + script], stdout=subprocess.PIPE)
 287     output.communicate()
 288     if not output.returncode == 0:
 289         print "NSIS failed!"
 290         return -1
 291     setupfile = program + "-" + versionstring + "-setup.exe"
 292     # find output filename in nsis script file
 293     nsissetup = ""
 294     for line in open(srcfolder + "/" + script):
 295         if re.match(r'^[^;]*OutFile\s+', line) != None:
 296             nsissetup = re.sub(r'^[^;]*OutFile\s+"(.+)"', r'\1', line).rstrip()
 297     if nsissetup == "":
 298         print "Could not retrieve output file name!"
 299         return -1
 300     shutil.copy(srcfolder + "/" + os.path.dirname(script) + "/" + nsissetup, setupfile)
 301     return 0
 302
 303
 304 def nsisfileinject(nsis, outscript, filelist):
 305     '''Inject files in filelist into NSIS script file after the File line
 306        containing the main binary. This assumes that the main binary is present
 307        in the NSIS script and that all additiona files (dlls etc) to get placed
 308        into $INSTDIR.'''
 309     output = open(outscript, "w")
 310     for line in open(nsis, "r"):
 311         output.write(line)
 312         # inject files after the progexe binary. Match the basename only to avoid path mismatches.
 313         if re.match(r'^\s*File\s*.*' + os.path.basename(progexe), line, re.IGNORECASE):
 314             for f in filelist:
 315                 injection = "    File /oname=$INSTDIR\\" + os.path.basename(f) + " " + os.path.normcase(f) + "\n"
 316                 output.write(injection)
 317             output.write("    ; end of injected files\n")
 318     output.close()
 319
 320
 321 def finddlls(program, extrapaths = []):
 322     '''Check program for required DLLs. Find all required DLLs except ignored
 323        ones and return a list of DLL filenames (including path).'''
 324     # ask objdump about dependencies.
 325     output = subprocess.Popen(["objdump", "-x", program], stdout=subprocess.PIPE)
 326     cmdout = output.communicate()
 327
 328     # create list of used DLLs. Store as lower case as W32 is case-insensitive.
 329     dlls = []
 330     for line in cmdout[0].split('\n'):
 331         if re.match(r'\s*DLL Name', line) != None:
 332             dll = re.sub(r'^\s*DLL Name:\s+([a-zA-Z_\-0-9\.\+]+).*$', r'\1', line)
 333             dlls.append(dll.lower())
 334
 335     # find DLLs in extrapaths and PATH environment variable.
 336     dllpaths = []
 337     for file in dlls:
 338         if file in systemdlls:
 339             print file + ": System DLL"
 340             continue
 341         dllpath = ""
 342         for path in extrapaths:
 343             if os.path.exists(path + "/" + file):
 344                 dllpath = re.sub(r"\\", r"/", path + "/" + file)
 345                 print file + ": found at " + dllpath
 346                 dllpaths.append(dllpath)
 347                 break
 348         if dllpath == "":
 349             try:
 350                 dllpath = re.sub(r"\\", r"/", which.which(file))
 351                 print file + ": found at " + dllpath
 352                 dllpaths.append(dllpath)
 353             except:
 354                 print file + ": NOT FOUND."
 355     return dllpaths
 356
 357
 358 def zipball(versionstring, buildfolder):
 359     '''package created binary'''
 360     print "Creating binary zipball."
 361     archivebase = program + "-" + versionstring
 362     outfolder = buildfolder + "/" + archivebase
 363     archivename = archivebase + ".zip"
 364     # create output folder
 365     os.mkdir(outfolder)
 366     # move program files to output folder
 367     for f in programfiles:
 368         if re.match(r'^(/|[a-zA-Z]:)', f) != None:
 369             shutil.copy(f, outfolder)
 370         else:
 371             shutil.copy(buildfolder + "/" + f, outfolder)
 372     # create zipball from output folder
 373     zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED)
 374     for root, dirs, files in os.walk(outfolder):
 375         for name in files:
 376             physname = os.path.join(root, name)
 377             filename = re.sub("^" + buildfolder, "", physname)
 378             zf.write(physname, filename)
 379         for name in dirs:
 380             physname = os.path.join(root, name)
 381             filename = re.sub("^" + buildfolder, "", physname)
 382             zf.write(physname, filename)
 383     zf.close()
 384     # remove output folder
 385     shutil.rmtree(outfolder)
 386     return archivename
 387
 388
 389 def tarball(versionstring, buildfolder):
 390     '''package created binary'''
 391     print "Creating binary tarball."
 392     archivebase = program + "-" + versionstring
 393     outfolder = buildfolder + "/" + archivebase
 394     archivename = archivebase + ".tar.bz2"
 395     # create output folder
 396     os.mkdir(outfolder)
 397     # move program files to output folder
 398     for f in programfiles:
 399         shutil.copy(buildfolder + "/" + f, outfolder)
 400     # create tarball from output folder
 401     tf = tarfile.open(archivename, mode='w:bz2')
 402     tf.add(outfolder, archivebase)
 403     tf.close()
 404     # remove output folder
 405     shutil.rmtree(outfolder)
 406     return archivename
 407
 408
 409 def macdeploy(versionstring, buildfolder):
 410     '''package created binary to dmg'''
 411     dmgfile = program + "-" + versionstring + ".dmg"
 412     appbundle = buildfolder + "/" + progexe
 413
 414     # workaround to Qt issues when building out-of-tree. Copy files into bundle.
 415     sourcebase = buildfolder + re.sub('[^/]+.pro$', '', project) + "/"
 416     print sourcebase
 417     for src in bundlecopy:
 418         shutil.copy(sourcebase + src, appbundle + "/" + bundlecopy[src])
 419     # end of Qt workaround
 420
 421     output = subprocess.Popen(["macdeployqt", progexe, "-dmg"], stdout=subprocess.PIPE, cwd=buildfolder)
 422     output.communicate()
 423     if not output.returncode == 0:
 424         print "macdeployqt failed!"
 425         return -1
 426     # copy dmg to output folder
 427     shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile)
 428     return dmgfile
 429
 430 def filehashes(filename):
 431     '''Calculate md5 and sha1 hashes for a given file.'''
 432     if not os.path.exists(filename):
 433         return ["", ""]
 434     m = hashlib.md5()
 435     s = hashlib.sha1()
 436     f = open(filename, 'rb')
 437     while True:
 438         d = f.read(65536)
 439         if d == "":
 440             break
 441         m.update(d)
 442         s.update(d)
 443     return [m.hexdigest(), s.hexdigest()]
 444
 445
 446 def filestats(filename):
 447     if not os.path.exists(filename):
 448         return
 449     st = os.stat(filename)
 450     print filename, "\n", "-" * len(filename)
 451     print "Size:    %i bytes" % st.st_size
 452     h = filehashes(filename)
 453     print "md5sum:  %s" % h[0]
 454     print "sha1sum: %s" % h[1]
 455     print "-" * len(filename), "\n"
 456
 457
 458 def tempclean(workfolder, nopro):
 459     if nopro == True:
 460         print "Cleaning up working folder %s" % workfolder
 461         shutil.rmtree(workfolder)
 462     else:
 463         print "Project file specified or cleanup disabled!"
 464         print "Temporary files kept at %s" % workfolder
 465
 466
 467 def deploy():
 468     startup = time.time()
 469
 470     try:
 471         opts, args = getopt.getopt(sys.argv[1:], "q:p:t:a:n:sbdkh",
 472             ["qmake=", "project=", "tag=", "add=", "makensis=", "source-only", "binary-only", "dynamic", "keep-temp", "help"])
 473     except getopt.GetoptError, err:
 474         print str(err)
 475         usage(sys.argv[0])
 476         sys.exit(1)
 477     qt = ""
 478     proj = ""
 479     svnbase = svnserver + "trunk/"
 480     tag = ""
 481     addfiles = []
 482     cleanup = True
 483     binary = True
 484     source = True
 485     keeptemp = False
 486     makensis = ""
 487     if sys.platform != "darwin":
 488         static = True
 489     else:
 490         static = False
 491     for o, a in opts:
 492         if o in ("-q", "--qmake"):
 493             qt = a
 494         if o in ("-p", "--project"):
 495             proj = a
 496             cleanup = False
 497         if o in ("-t", "--tag"):
 498             tag = a
 499             svnbase = svnserver + "tags/" + tag + "/"
 500         if o in ("-a", "--add"):
 501             addfiles.append(a)
 502         if o in ("-n", "--makensis"):
 503             makensis = a
 504         if o in ("-s", "--source-only"):
 505             binary = False
 506         if o in ("-b", "--binary-only"):
 507             source = False
 508         if o in ("-d", "--dynamic") and sys.platform != "darwin":
 509             static = False
 510         if o in ("-k", "--keep-temp"):
 511             keeptemp = True
 512         if o in ("-h", "--help"):
 513             usage(sys.argv[0])
 514             sys.exit(0)
 515
 516     if source == False and binary == False:
 517         print "Building build neither source nor binary means nothing to do. Exiting."
 518         sys.exit(1)
 519
 520     # search for qmake
 521     if qt == "":
 522         qm = findqt()
 523     else:
 524         qm = checkqt(qt)
 525     if qm == "":
 526         print "ERROR: No suitable Qt installation found."
 527         sys.exit(1)
 528
 529     # create working folder. Use current directory if -p option used.
 530     if proj == "":
 531         w = tempfile.mkdtemp()
 532         # make sure the path doesn't contain backslashes to prevent issues
 533         # later when running on windows.
 534         workfolder = re.sub(r'\\', '/', w)
 535         if not tag == "":
 536             sourcefolder = workfolder + "/" + tag + "/"
 537             archivename = tag + "-src.tar.bz2"
 538             # get numeric version part from tag
 539             ver = "v" + re.sub('^[^\d]+', '', tag)
 540         else:
 541             trunk = gettrunkrev(svnbase)
 542             sourcefolder = workfolder + "/" + program + "-r" + str(trunk) + "/"
 543             archivename = program + "-r" + str(trunk) + "-src.tar.bz2"
 544             ver = "r" + str(trunk)
 545         os.mkdir(sourcefolder)
 546     else:
 547         workfolder = "."
 548         sourcefolder = "."
 549         archivename = ""
 550     # check if project file explicitly given. If yes, don't get sources from svn
 551     if proj == "":
 552         proj = sourcefolder + project
 553         # get sources and pack source tarball
 554         if not getsources(svnbase, svnpaths, sourcefolder) == 0:
 555             tempclean(workfolder, cleanup and not keeptemp)
 556             sys.exit(1)
 557
 558         if source == True:
 559             tf = tarfile.open(archivename, mode='w:bz2')
 560             tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
 561             tf.close()
 562             if binary == False:
 563                 shutil.rmtree(workfolder)
 564                 sys.exit(0)
 565     else:
 566         # figure version from sources. Need to take path to project file into account.
 567         versionfile = re.subn('[\w\.]+$', "version.h", proj)[0]
 568         ver = findversion(versionfile)
 569
 570     # check project file
 571     if not os.path.exists(proj):
 572         print "ERROR: path to project file wrong."
 573         sys.exit(1)
 574
 575     # copy specified (--add) files to working folder
 576     for f in addfiles:
 577         shutil.copy(f, sourcefolder)
 578     buildstart = time.time()
 579     header = "Building %s %s" % (program, ver)
 580     print header
 581     print len(header) * "="
 582
 583     # build it.
 584     if not qmake(qm, proj, sourcefolder, static) == 0:
 585         tempclean(workfolder, cleanup and not keeptemp)
 586         sys.exit(1)
 587     if not build(sourcefolder) == 0:
 588         tempclean(workfolder, cleanup and not keeptemp)
 589         sys.exit(1)
 590     buildtime = time.time() - buildstart
 591     if sys.platform == "win32":
 592         if useupx == True:
 593             if not upxfile(sourcefolder) == 0:
 594                 tempclean(workfolder, cleanup and not keeptemp)
 595                 sys.exit(1)
 596         dllfiles = finddlls(sourcefolder + "/" + progexe, [os.path.dirname(qm)])
 597         if dllfiles.count > 0:
 598             programfiles.extend(dllfiles)
 599         archive = zipball(ver, sourcefolder)
 600         # only when running native right now.
 601         if nsisscript != "" and makensis != "":
 602             nsisfileinject(sourcefolder + "/" + nsisscript, sourcefolder + "/" + nsisscript + ".tmp", dllfiles)
 603             runnsis(ver, makensis, nsisscript + ".tmp", sourcefolder)
 604     elif sys.platform == "darwin":
 605         archive = macdeploy(ver, sourcefolder)
 606     else:
 607         if os.uname()[4].endswith("64"):
 608             ver += "-64bit"
 609         archive = tarball(ver, sourcefolder)
 610
 611     # remove temporary files
 612     tempclean(workfolder, cleanup)
 613
 614     # display summary
 615     headline = "Build Summary for %s" % program
 616     print "\n", headline, "\n", "=" * len(headline)
 617     if not archivename == "":
 618         filestats(archivename)
 619     filestats(archive)
 620     duration = time.time() - startup
 621     durmins = (int)(duration / 60)
 622     dursecs = (int)(duration % 60)
 623     buildmins = (int)(buildtime / 60)
 624     buildsecs = (int)(buildtime % 60)
 625     print "Overall time %smin %ssec, building took %smin %ssec." % \
 626         (durmins, dursecs, buildmins, buildsecs)
 627
 628
 629 if __name__ == "__main__":
 630     deploy()
 631