utils/common/deploy.py

   1 #!/usr/bin/python
   2 #             __________               __   ___.
   3 #   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4 #   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5 #   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6 #   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7 #                     \/            \/     \/    \/            \/
   8 # $Id$
   9 #
  10 # Copyright (c) 2009 Dominik Riebeling
  11 #
  12 # All files in this archive are subject to the GNU General Public License.
  13 # See the file COPYING in the source tree root for full license agreement.
  14 #
  15 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  16 # KIND, either express or implied.
  17 #
  18 #
  19 # Automate building releases for deployment.
  20 # Run from any folder to build
  21 # - trunk
  22 # - any tag (using the -t option)
  23 # - any local folder (using the -p option)
  24 # Will build a binary archive (tar.bz2 / zip) and source archive.
  25 # The source archive won't be built for local builds. Trunk and
  26 # tag builds will retrieve the sources directly from svn and build
  27 # below the systems temporary folder.
  28 #
  29 # If the required Qt installation isn't in PATH use --qmake option.
  30 # Tested on Linux and MinGW / W32
  31 #
  32 # requires python which package (http://code.google.com/p/which/)
  33 # requires pysvn package.
  34 # requires upx.exe in PATH on Windows.
  35 #
  36
  37 import re
  38 import os
  39 import sys
  40 import tarfile
  41 import zipfile
  42 import shutil
  43 import subprocess
  44 import getopt
  45 import time
  46 import hashlib
  47 import tempfile
  48
  49 # modules that are not part of python itself.
  50 try:
  51     import pysvn
  52 except ImportError:
  53     print "Fatal: This script requires the pysvn package to run."
  54     print "       See http://pysvn.tigris.org/."
  55     sys.exit(-5)
  56 try:
  57     import which
  58 except ImportError:
  59     print "Fatal: This script requires the which package to run."
  60     print "       See http://code.google.com/p/which/."
  61     sys.exit(-5)
  62
  63 # == Global stuff ==
  64 # Windows nees some special treatment. Differentiate between program name
  65 # and executable filename.
  66 program = ""
  67 project = ""
  68 environment = os.environ
  69 progexe = ""
  70 make = "make"
  71 programfiles = []
  72 nsisscript = ""
  73
  74 svnserver = ""
  75 # Paths and files to retrieve from svn when creating a tarball.
  76 # This is a mixed list, holding both paths and filenames.
  77 svnpaths = [ ]
  78 # set this to true to run upx on the resulting binary, false to skip this step.
  79 # only used on w32.
  80 useupx = False
  81
  82 # OS X: files to copy into the bundle. Workaround for out-of-tree builds.
  83 bundlecopy = { }
  84
  85 # DLL files to ignore when searching for required DLL files.
  86 systemdlls = ['advapi32.dll',
  87         'comdlg32.dll',
  88         'gdi32.dll',
  89         'imm32.dll',
  90         'kernel32.dll',
  91         'msvcrt.dll',
  92         'msvcrt.dll',
  93         'netapi32.dll',
  94         'ole32.dll',
  95         'oleaut32.dll',
  96         'setupapi.dll',
  97         'shell32.dll',
  98         'user32.dll',
  99         'winmm.dll',
 100         'winspool.drv',
 101         'ws2_32.dll']
 102
 103
 104 # == Functions ==
 105 def usage(myself):
 106     print "Usage: %s [options]" % myself
 107     print "       -q, --qmake=<qmake>   path to qmake"
 108     print "       -p, --project=<pro>   path to .pro file for building with local tree"
 109     print "       -t, --tag=<tag>       use specified tag from svn"
 110     print "       -a, --add=<file>      add file to build folder before building"
 111     print "       -s, --source-only     only create source archive"
 112     print "       -b, --binary-only     only create binary archive"
 113     if nsisscript != "":
 114         print "       -n, --makensis=<file> path to makensis for building Windows setup program."
 115     if sys.platform != "darwin":
 116         print "       -d, --dynamic         link dynamically instead of static"
 117     print "       -k, --keep-temp       keep temporary folder on build failure"
 118     print "       -h, --help            this help"
 119     print "  If neither a project file nor tag is specified trunk will get downloaded"
 120     print "  from svn."
 121
 122 def getsources(svnsrv, filelist, dest):
 123     '''Get the files listed in filelist from svnsrv and put it at dest.'''
 124     client = pysvn.Client()
 125     print "Checking out sources from %s, please wait." % svnsrv
 126
 127     for elem in filelist:
 128         url = re.subn('/$', '', svnsrv + elem)[0]
 129         destpath = re.subn('/$', '', dest + elem)[0]
 130         # make sure the destination path does exist
 131         d = os.path.dirname(destpath)
 132         if not os.path.exists(d):
 133             os.makedirs(d)
 134         # get from svn
 135         try:
 136             client.export(url, destpath)
 137         except:
 138             print "SVN client error: %s" % sys.exc_value
 139             print "URL: %s, destination: %s" % (url, destpath)
 140             return -1
 141     print "Checkout finished."
 142     return 0
 143
 144
 145 def gettrunkrev(svnsrv):
 146     '''Get the revision of trunk for svnsrv'''
 147     client = pysvn.Client()
 148     entries = client.info2(svnsrv, recurse=False)
 149     return entries[0][1].rev.number
 150
 151
 152 def findversion(versionfile):
 153     '''figure most recent program version from version.h,
 154     returns version string.'''
 155     h = open(versionfile, "r")
 156     c = h.read()
 157     h.close()
 158     r = re.compile("#define +VERSION +\"(.[0-9\.a-z]+)\"")
 159     m = re.search(r, c)
 160     s = re.compile("\$Revision: +([0-9]+)")
 161     n = re.search(s, c)
 162     if n == None:
 163         print "WARNING: Revision not found!"
 164     return m.group(1)
 165
 166
 167 def findqt():
 168     '''Search for Qt4 installation. Return path to qmake.'''
 169     print "Searching for Qt"
 170     bins = ["qmake", "qmake-qt4"]
 171     for binary in bins:
 172         try:
 173             q = which.which(binary)
 174             if len(q) > 0:
 175                 result = checkqt(q)
 176                 if not result == "":
 177                     return result
 178         except:
 179             print sys.exc_value
 180
 181     return ""
 182
 183
 184 def checkqt(qmakebin):
 185     '''Check if given path to qmake exists and is a suitable version.'''
 186     result = ""
 187     # check if binary exists
 188     if not os.path.exists(qmakebin):
 189         print "Specified qmake path does not exist!"
 190         return result
 191     # check version
 192     output = subprocess.Popen([qmakebin, "-version"], stdout=subprocess.PIPE,
 193             stderr=subprocess.PIPE)
 194     cmdout = output.communicate()
 195     # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
 196     for ou in cmdout:
 197         r = re.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
 198         m = re.search(r, ou)
 199         if not m == None:
 200             print "Qt found: %s" % m.group(1)
 201             s = re.compile("4\..*")
 202             n = re.search(s, m.group(1))
 203             if not n == None:
 204                 result = qmakebin
 205     return result
 206
 207
 208 def qmake(qmake="qmake", projfile=project, wd=".", static=True):
 209     print "Running qmake in %s..." % wd
 210     command = [qmake, "-config", "release", "-config", "noccache"]
 211     if static == True:
 212         command.append("-config")
 213         command.append("static")
 214     command.append(projfile)
 215     output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd, env=environment)
 216     output.communicate()
 217     if not output.returncode == 0:
 218         print "qmake returned an error!"
 219         return -1
 220     return 0
 221
 222
 223 def build(wd="."):
 224     # make
 225     print "Building ..."
 226     output = subprocess.Popen([make], stdout=subprocess.PIPE, cwd=wd)
 227     while True:
 228         c = output.stdout.readline()
 229         sys.stdout.write(".")
 230         sys.stdout.flush()
 231         if not output.poll() == None:
 232             sys.stdout.write("\n")
 233             sys.stdout.flush()
 234             if not output.returncode == 0:
 235                 print "Build failed!"
 236                 return -1
 237             break
 238     if sys.platform != "darwin":
 239         # strip. OS X handles this via macdeployqt.
 240         print "Stripping binary."
 241         output = subprocess.Popen(["strip", progexe], stdout=subprocess.PIPE, cwd=wd)
 242         output.communicate()
 243         if not output.returncode == 0:
 244             print "Stripping failed!"
 245             return -1
 246     return 0
 247
 248
 249 def upxfile(wd="."):
 250     # run upx on binary
 251     print "UPX'ing binary ..."
 252     output = subprocess.Popen(["upx", progexe], stdout=subprocess.PIPE, cwd=wd)
 253     output.communicate()
 254     if not output.returncode == 0:
 255         print "UPX'ing failed!"
 256         return -1
 257     return 0
 258
 259
 260 def runnsis(versionstring, nsis, script, srcfolder):
 261     # run script through nsis to create installer.
 262     print "Running NSIS ..."
 263
 264     # Assume the generated installer gets placed in the same folder the nsi
 265     # script lives in.  This seems to be a valid assumption unless the nsi
 266     # script specifies a path. NSIS expects files relative to source folder so
 267     # copy progexe. Additional files are injected into the nsis script.
 268
 269     # FIXME: instead of copying binaries around copy the NSI file and inject
 270     # the correct paths.
 271     b = srcfolder + "/" + os.path.dirname(script) + "/" + os.path.dirname(progexe)
 272     if not os.path.exists(b):
 273         os.mkdir(b)
 274     shutil.copy(srcfolder + "/" + progexe, b)
 275     output = subprocess.Popen([nsis, srcfolder + "/" + script], stdout=subprocess.PIPE)
 276     output.communicate()
 277     if not output.returncode == 0:
 278         print "NSIS failed!"
 279         return -1
 280     setupfile = program + "-" + versionstring + "-setup.exe"
 281     # find output filename in nsis script file
 282     nsissetup = ""
 283     for line in open(srcfolder + "/" + script):
 284         if re.match(r'^[^;]*OutFile\s+', line) != None:
 285             nsissetup = re.sub(r'^[^;]*OutFile\s+"(.+)"', r'\1', line).rstrip()
 286     if nsissetup == "":
 287         print "Could not retrieve output file name!"
 288         return -1
 289     shutil.copy(srcfolder + "/" + os.path.dirname(script) + "/" + nsissetup, setupfile)
 290     return 0
 291
 292
 293 def nsisfileinject(nsis, outscript, filelist):
 294     '''Inject files in filelist into NSIS script file after the File line
 295        containing the main binary. This assumes that the main binary is present
 296        in the NSIS script and that all additiona files (dlls etc) to get placed
 297        into $INSTDIR.'''
 298     output = open(outscript, "w")
 299     for line in open(nsis, "r"):
 300         output.write(line)
 301         # inject files after the progexe binary. Match the basename only to avoid path mismatches.
 302         if re.match(r'^\s*File\s*.*' + os.path.basename(progexe), line, re.IGNORECASE):
 303             for f in filelist:
 304                 injection = "    File /oname=$INSTDIR\\" + os.path.basename(f) + " " + os.path.normcase(f) + "\n"
 305                 output.write(injection)
 306             output.write("    ; end of injected files\n")
 307     output.close()
 308
 309
 310 def finddlls(program, extrapaths = []):
 311     '''Check program for required DLLs. Find all required DLLs except ignored
 312        ones and return a list of DLL filenames (including path).'''
 313     # ask objdump about dependencies.
 314     output = subprocess.Popen(["objdump", "-x", program], stdout=subprocess.PIPE)
 315     cmdout = output.communicate()
 316
 317     # create list of used DLLs. Store as lower case as W32 is case-insensitive.
 318     dlls = []
 319     for line in cmdout[0].split('\n'):
 320         if re.match(r'\s*DLL Name', line) != None:
 321             dll = re.sub(r'^\s*DLL Name:\s+([a-zA-Z_\-0-9\.\+]+).*$', r'\1', line)
 322             dlls.append(dll.lower())
 323
 324     # find DLLs in extrapaths and PATH environment variable.
 325     dllpaths = []
 326     for file in dlls:
 327         if file in systemdlls:
 328             print file + ": System DLL"
 329             continue
 330         dllpath = ""
 331         for path in extrapaths:
 332             if os.path.exists(path + "/" + file):
 333                 dllpath = re.sub(r"\\", r"/", path + "/" + file)
 334                 print file + ": found at " + dllpath
 335                 dllpaths.append(dllpath)
 336                 break
 337         if dllpath == "":
 338             try:
 339                 dllpath = re.sub(r"\\", r"/", which.which(file))
 340                 print file + ": found at " + dllpath
 341                 dllpaths.append(dllpath)
 342             except:
 343                 print file + ": NOT FOUND."
 344     return dllpaths
 345
 346
 347 def zipball(versionstring, buildfolder):
 348     '''package created binary'''
 349     print "Creating binary zipball."
 350     archivebase = program + "-" + versionstring
 351     outfolder = buildfolder + "/" + archivebase
 352     archivename = archivebase + ".zip"
 353     # create output folder
 354     os.mkdir(outfolder)
 355     # move program files to output folder
 356     for f in programfiles:
 357         if re.match(r'^(/|[a-zA-Z]:)', f) != None:
 358             shutil.copy(f, outfolder)
 359         else:
 360             shutil.copy(buildfolder + "/" + f, outfolder)
 361     # create zipball from output folder
 362     zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED)
 363     for root, dirs, files in os.walk(outfolder):
 364         for name in files:
 365             physname = os.path.join(root, name)
 366             filename = re.sub("^" + buildfolder, "", physname)
 367             zf.write(physname, filename)
 368         for name in dirs:
 369             physname = os.path.join(root, name)
 370             filename = re.sub("^" + buildfolder, "", physname)
 371             zf.write(physname, filename)
 372     zf.close()
 373     # remove output folder
 374     shutil.rmtree(outfolder)
 375     return archivename
 376
 377
 378 def tarball(versionstring, buildfolder):
 379     '''package created binary'''
 380     print "Creating binary tarball."
 381     archivebase = program + "-" + versionstring
 382     outfolder = buildfolder + "/" + archivebase
 383     archivename = archivebase + ".tar.bz2"
 384     # create output folder
 385     os.mkdir(outfolder)
 386     # move program files to output folder
 387     for f in programfiles:
 388         shutil.copy(buildfolder + "/" + f, outfolder)
 389     # create tarball from output folder
 390     tf = tarfile.open(archivename, mode='w:bz2')
 391     tf.add(outfolder, archivebase)
 392     tf.close()
 393     # remove output folder
 394     shutil.rmtree(outfolder)
 395     return archivename
 396
 397
 398 def macdeploy(versionstring, buildfolder):
 399     '''package created binary to dmg'''
 400     dmgfile = program + "-" + versionstring + ".dmg"
 401     appbundle = buildfolder + "/" + progexe
 402
 403     # workaround to Qt issues when building out-of-tree. Copy files into bundle.
 404     sourcebase = buildfolder + re.sub('[^/]+.pro$', '', project) + "/"
 405     print sourcebase
 406     for src in bundlecopy:
 407         shutil.copy(sourcebase + src, appbundle + "/" + bundlecopy[src])
 408     # end of Qt workaround
 409
 410     output = subprocess.Popen(["macdeployqt", progexe, "-dmg"], stdout=subprocess.PIPE, cwd=buildfolder)
 411     output.communicate()
 412     if not output.returncode == 0:
 413         print "macdeployqt failed!"
 414         return -1
 415     # copy dmg to output folder
 416     shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile)
 417     return dmgfile
 418
 419 def filehashes(filename):
 420     '''Calculate md5 and sha1 hashes for a given file.'''
 421     if not os.path.exists(filename):
 422         return ["", ""]
 423     m = hashlib.md5()
 424     s = hashlib.sha1()
 425     f = open(filename, 'rb')
 426     while True:
 427         d = f.read(65536)
 428         if d == "":
 429             break
 430         m.update(d)
 431         s.update(d)
 432     return [m.hexdigest(), s.hexdigest()]
 433
 434
 435 def filestats(filename):
 436     if not os.path.exists(filename):
 437         return
 438     st = os.stat(filename)
 439     print filename, "\n", "-" * len(filename)
 440     print "Size:    %i bytes" % st.st_size
 441     h = filehashes(filename)
 442     print "md5sum:  %s" % h[0]
 443     print "sha1sum: %s" % h[1]
 444     print "-" * len(filename), "\n"
 445
 446
 447 def tempclean(workfolder, nopro):
 448     if nopro == True:
 449         print "Cleaning up working folder %s" % workfolder
 450         shutil.rmtree(workfolder)
 451     else:
 452         print "Project file specified or cleanup disabled!"
 453         print "Temporary files kept at %s" % workfolder
 454
 455
 456 def deploy():
 457     startup = time.time()
 458
 459     try:
 460         opts, args = getopt.getopt(sys.argv[1:], "q:p:t:a:n:sbdkh",
 461             ["qmake=", "project=", "tag=", "add=", "makensis=", "source-only", "binary-only", "dynamic", "keep-temp", "help"])
 462     except getopt.GetoptError, err:
 463         print str(err)
 464         usage(sys.argv[0])
 465         sys.exit(1)
 466     qt = ""
 467     proj = ""
 468     svnbase = svnserver + "trunk/"
 469     tag = ""
 470     addfiles = []
 471     cleanup = True
 472     binary = True
 473     source = True
 474     keeptemp = False
 475     makensis = ""
 476     if sys.platform != "darwin":
 477         static = True
 478     else:
 479         static = False
 480     for o, a in opts:
 481         if o in ("-q", "--qmake"):
 482             qt = a
 483         if o in ("-p", "--project"):
 484             proj = a
 485             cleanup = False
 486         if o in ("-t", "--tag"):
 487             tag = a
 488             svnbase = svnserver + "tags/" + tag + "/"
 489         if o in ("-a", "--add"):
 490             addfiles.append(a)
 491         if o in ("-n", "--makensis"):
 492             makensis = a
 493         if o in ("-s", "--source-only"):
 494             binary = False
 495         if o in ("-b", "--binary-only"):
 496             source = False
 497         if o in ("-d", "--dynamic") and sys.platform != "darwin":
 498             static = False
 499         if o in ("-k", "--keep-temp"):
 500             keeptemp = True
 501         if o in ("-h", "--help"):
 502             usage(sys.argv[0])
 503             sys.exit(0)
 504
 505     if source == False and binary == False:
 506         print "Building build neither source nor binary means nothing to do. Exiting."
 507         sys.exit(1)
 508
 509     # search for qmake
 510     if qt == "":
 511         qm = findqt()
 512     else:
 513         qm = checkqt(qt)
 514     if qm == "":
 515         print "ERROR: No suitable Qt installation found."
 516         sys.exit(1)
 517
 518     # create working folder. Use current directory if -p option used.
 519     if proj == "":
 520         w = tempfile.mkdtemp()
 521         # make sure the path doesn't contain backslashes to prevent issues
 522         # later when running on windows.
 523         workfolder = re.sub(r'\\', '/', w)
 524         if not tag == "":
 525             sourcefolder = workfolder + "/" + tag + "/"
 526             archivename = tag + "-src.tar.bz2"
 527             # get numeric version part from tag
 528             ver = "v" + re.sub('^[^\d]+', '', tag)
 529         else:
 530             trunk = gettrunkrev(svnbase)
 531             sourcefolder = workfolder + "/" + program + "-r" + str(trunk) + "/"
 532             archivename = program + "-r" + str(trunk) + "-src.tar.bz2"
 533             ver = "r" + str(trunk)
 534         os.mkdir(sourcefolder)
 535     else:
 536         workfolder = "."
 537         sourcefolder = "."
 538         archivename = ""
 539     # check if project file explicitly given. If yes, don't get sources from svn
 540     if proj == "":
 541         proj = sourcefolder + project
 542         # get sources and pack source tarball
 543         if not getsources(svnbase, svnpaths, sourcefolder) == 0:
 544             tempclean(workfolder, cleanup and not keeptemp)
 545             sys.exit(1)
 546
 547         if source == True:
 548             tf = tarfile.open(archivename, mode='w:bz2')
 549             tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
 550             tf.close()
 551             if binary == False:
 552                 shutil.rmtree(workfolder)
 553                 sys.exit(0)
 554     else:
 555         # figure version from sources. Need to take path to project file into account.
 556         versionfile = re.subn('[\w\.]+$', "version.h", proj)[0]
 557         ver = findversion(versionfile)
 558
 559     # check project file
 560     if not os.path.exists(proj):
 561         print "ERROR: path to project file wrong."
 562         sys.exit(1)
 563
 564     # copy specified (--add) files to working folder
 565     for f in addfiles:
 566         shutil.copy(f, sourcefolder)
 567     buildstart = time.time()
 568     header = "Building %s %s" % (program, ver)
 569     print header
 570     print len(header) * "="
 571
 572     # build it.
 573     if not qmake(qm, proj, sourcefolder, static) == 0:
 574         tempclean(workfolder, cleanup and not keeptemp)
 575         sys.exit(1)
 576     if not build(sourcefolder) == 0:
 577         tempclean(workfolder, cleanup and not keeptemp)
 578         sys.exit(1)
 579     buildtime = time.time() - buildstart
 580     if sys.platform == "win32":
 581         if useupx == True:
 582             if not upxfile(sourcefolder) == 0:
 583                 tempclean(workfolder, cleanup and not keeptemp)
 584                 sys.exit(1)
 585         dllfiles = finddlls(sourcefolder + "/" + progexe, [os.path.dirname(qm)])
 586         if dllfiles.count > 0:
 587             programfiles.extend(dllfiles)
 588         archive = zipball(ver, sourcefolder)
 589         # only when running native right now.
 590         if nsisscript != "" and makensis != "":
 591             nsisfileinject(sourcefolder + "/" + nsisscript, sourcefolder + "/" + nsisscript + ".tmp", dllfiles)
 592             runnsis(ver, makensis, nsisscript + ".tmp", sourcefolder)
 593     elif sys.platform == "darwin":
 594         archive = macdeploy(ver, sourcefolder)
 595     else:
 596         if os.uname()[4].endswith("64"):
 597             ver += "-64bit"
 598         archive = tarball(ver, sourcefolder)
 599
 600     # remove temporary files
 601     tempclean(workfolder, cleanup)
 602
 603     # display summary
 604     headline = "Build Summary for %s" % program
 605     print "\n", headline, "\n", "=" * len(headline)
 606     if not archivename == "":
 607         filestats(archivename)
 608     filestats(archive)
 609     duration = time.time() - startup
 610     durmins = (int)(duration / 60)
 611     dursecs = (int)(duration % 60)
 612     buildmins = (int)(buildtime / 60)
 613     buildsecs = (int)(buildtime % 60)
 614     print "Overall time %smin %ssec, building took %smin %ssec." % \
 615         (durmins, dursecs, buildmins, buildsecs)
 616
 617
 618 if __name__ == "__main__":
 619     deploy()
 620