rbutil/rbutilqt/deploy-release.py

   1 #!/usr/bin/python
   2 #             __________               __   ___.
   3 #   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4 #   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5 #   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6 #   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7 #                     \/            \/     \/    \/            \/
   8 # $Id$
   9 #
  10 # Copyright (c) 2009 Dominik Riebeling
  11 #
  12 # All files in this archive are subject to the GNU General Public License.
  13 # See the file COPYING in the source tree root for full license agreement.
  14 #
  15 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  16 # KIND, either express or implied.
  17 #
  18 #
  19 # Automate building releases for deployment.
  20 # Run from any folder to build
  21 # - trunk
  22 # - any tag (using the -t option)
  23 # - any local folder (using the -p option)
  24 # Will build a binary archive (tar.bz2 / zip) and source archive.
  25 # The source archive won't be built for local builds. Trunk and
  26 # tag builds will retrieve the sources directly from svn and build
  27 # below the systems temporary folder.
  28 #
  29 # If the required Qt installation isn't in PATH use --qmake option.
  30 # Tested on Linux and MinGW / W32
  31 #
  32 # requires python which package (http://code.google.com/p/which/)
  33 # requires pysvn package.
  34 # requires upx.exe in PATH on Windows.
  35 #
  36
  37 import re
  38 import os
  39 import sys
  40 import tarfile
  41 import zipfile
  42 import shutil
  43 import subprocess
  44 import getopt
  45 import time
  46 import hashlib
  47 import tempfile
  48
  49 # modules that are not part of python itself.
  50 try:
  51     import pysvn
  52 except ImportError:
  53     print "Fatal: This script requires the pysvn package to run."
  54     print "       See http://pysvn.tigris.org/."
  55     sys.exit(-5)
  56 try:
  57     import which
  58 except ImportError:
  59     print "Fatal: This script requires the which package to run."
  60     print "       See http://code.google.com/p/which/."
  61     sys.exit(-5)
  62
  63 # == Global stuff ==
  64 # Windows nees some special treatment. Differentiate between program name
  65 # and executable filename.
  66 program = "RockboxUtility"
  67 project = "rbutil/rbutilqt/rbutilqt.pro"
  68 environment = os.environ
  69 make = "make"
  70 if sys.platform == "win32":
  71     progexe = "Release/" + program + ".exe"
  72     make = "mingw32-make"
  73 elif sys.platform == "darwin":
  74     progexe = program + ".app"
  75     # OS X 10.6 defaults to gcc 4.2. Building universal binaries that are
  76     # compatible with 10.4 requires using gcc-4.0.
  77     if not "QMAKESPEC" in environment:
  78         environment["QMAKESPEC"] = "macx-g++40"
  79 else:
  80     progexe = program
  81
  82 # all files of the program. Will get put into an archive after building
  83 # (zip on w32, tar.bz2 on Linux). Does not apply on Mac which uses dmg.
  84 programfiles = [ progexe ]
  85
  86 svnserver = "svn://svn.rockbox.org/rockbox/"
  87 # Paths and files to retrieve from svn when creating a tarball.
  88 # This is a mixed list, holding both paths and filenames.
  89 svnpaths = [ "rbutil/",
  90              "tools/ucl",
  91              "tools/rbspeex",
  92              "apps/codecs/libspeex",
  93              "docs/COPYING",
  94              "docs/CREDITS",
  95              "tools/iriver.c",
  96              "tools/Makefile",
  97              "tools/mkboot.h",
  98              "tools/voicefont.c",
  99              "tools/VOICE_PAUSE.wav",
 100              "tools/wavtrim.h",
 101              "tools/iriver.h",
 102              "tools/mkboot.c",
 103              "tools/telechips.c",
 104              "tools/telechips.h",
 105              "tools/voicefont.h",
 106              "tools/wavtrim.c",
 107              "tools/sapi_voice.vbs" ]
 108 # set this to true to run upx on the resulting binary, false to skip this step.
 109 # only used on w32.
 110 useupx = False
 111
 112 # OS X: files to copy into the bundle. Workaround for out-of-tree builds.
 113 bundlecopy = {
 114         "icons/rbutilqt.icns" : "Contents/Resources/",
 115         "Info.plist"          : "Contents/"
 116 }
 117
 118 # == Functions ==
 119 def usage(myself):
 120     print "Usage: %s [options]" % myself
 121     print "       -q, --qmake=<qmake>   path to qmake"
 122     print "       -p, --project=<pro>   path to .pro file for building with local tree"
 123     print "       -t, --tag=<tag>       use specified tag from svn"
 124     print "       -a, --add=<file>      add file to build folder before building"
 125     print "       -s, --source-only     only create source archive"
 126     print "       -b, --binary-only     only create binary archive"
 127     if sys.platform != "darwin":
 128         print "       -d, --dynamic         link dynamically instead of static"
 129     print "       -k, --keep-temp       keep temporary folder on build failure"
 130     print "       -h, --help            this help"
 131     print "  If neither a project file nor tag is specified trunk will get downloaded"
 132     print "  from svn."
 133
 134 def getsources(svnsrv, filelist, dest):
 135     '''Get the files listed in filelist from svnsrv and put it at dest.'''
 136     client = pysvn.Client()
 137     print "Checking out sources from %s, please wait." % svnsrv
 138
 139     for elem in filelist:
 140         url = re.subn('/$', '', svnsrv + elem)[0]
 141         destpath = re.subn('/$', '', dest + elem)[0]
 142         # make sure the destination path does exist
 143         d = os.path.dirname(destpath)
 144         if not os.path.exists(d):
 145             os.makedirs(d)
 146         # get from svn
 147         try:
 148             client.export(url, destpath)
 149         except:
 150             print "SVN client error: %s" % sys.exc_value
 151             print "URL: %s, destination: %s" % (url, destpath)
 152             return -1
 153     print "Checkout finished."
 154     return 0
 155
 156
 157 def gettrunkrev(svnsrv):
 158     '''Get the revision of trunk for svnsrv'''
 159     client = pysvn.Client()
 160     entries = client.info2(svnsrv, recurse=False)
 161     return entries[0][1].rev.number
 162
 163
 164 def findversion(versionfile):
 165     '''figure most recent program version from version.h,
 166     returns version string.'''
 167     h = open(versionfile, "r")
 168     c = h.read()
 169     h.close()
 170     r = re.compile("#define +VERSION +\"(.[0-9\.a-z]+)\"")
 171     m = re.search(r, c)
 172     s = re.compile("\$Revision: +([0-9]+)")
 173     n = re.search(s, c)
 174     if n == None:
 175         print "WARNING: Revision not found!"
 176     return m.group(1)
 177
 178
 179 def findqt():
 180     '''Search for Qt4 installation. Return path to qmake.'''
 181     print "Searching for Qt"
 182     bins = ["qmake", "qmake-qt4"]
 183     for binary in bins:
 184         try:
 185             q = which.which(binary)
 186             if len(q) > 0:
 187                 result = checkqt(q)
 188                 if not result == "":
 189                     return result
 190         except:
 191             print sys.exc_value
 192
 193     return ""
 194
 195
 196 def checkqt(qmakebin):
 197     '''Check if given path to qmake exists and is a suitable version.'''
 198     result = ""
 199     # check if binary exists
 200     if not os.path.exists(qmakebin):
 201         print "Specified qmake path does not exist!"
 202         return result
 203     # check version
 204     output = subprocess.Popen([qmakebin, "-version"], stdout=subprocess.PIPE,
 205             stderr=subprocess.PIPE)
 206     cmdout = output.communicate()
 207     # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
 208     for ou in cmdout:
 209         r = re.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
 210         m = re.search(r, ou)
 211         if not m == None:
 212             print "Qt found: %s" % m.group(1)
 213             s = re.compile("4\..*")
 214             n = re.search(s, m.group(1))
 215             if not n == None:
 216                 result = qmakebin
 217     return result
 218
 219
 220 def qmake(qmake="qmake", projfile=project, wd=".", static=True):
 221     print "Running qmake in %s..." % wd
 222     command = [qmake, "-config", "release", "-config", "noccache"]
 223     if static == True:
 224         command.append("-config")
 225         command.append("static")
 226     command.append(projfile)
 227     output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd, env=environment)
 228     output.communicate()
 229     if not output.returncode == 0:
 230         print "qmake returned an error!"
 231         return -1
 232     return 0
 233
 234
 235 def build(wd="."):
 236     # make
 237     print "Building ..."
 238     output = subprocess.Popen([make], stdout=subprocess.PIPE, cwd=wd)
 239     while True:
 240         c = output.stdout.readline()
 241         sys.stdout.write(".")
 242         sys.stdout.flush()
 243         if not output.poll() == None:
 244             sys.stdout.write("\n")
 245             sys.stdout.flush()
 246             if not output.returncode == 0:
 247                 print "Build failed!"
 248                 return -1
 249             break
 250     if sys.platform != "darwin":
 251         # strip. OS X handles this via macdeployqt.
 252         print "Stripping binary."
 253         output = subprocess.Popen(["strip", progexe], stdout=subprocess.PIPE, cwd=wd)
 254         output.communicate()
 255         if not output.returncode == 0:
 256             print "Stripping failed!"
 257             return -1
 258     return 0
 259
 260
 261 def upxfile(wd="."):
 262     # run upx on binary
 263     print "UPX'ing binary ..."
 264     output = subprocess.Popen(["upx", progexe], stdout=subprocess.PIPE, cwd=wd)
 265     output.communicate()
 266     if not output.returncode == 0:
 267         print "UPX'ing failed!"
 268         return -1
 269     return 0
 270
 271
 272 def zipball(versionstring, buildfolder):
 273     '''package created binary'''
 274     print "Creating binary zipball."
 275     archivebase = program + "-" + versionstring
 276     outfolder = buildfolder + "/" + archivebase
 277     archivename = archivebase + ".zip"
 278     # create output folder
 279     os.mkdir(outfolder)
 280     # move program files to output folder
 281     for f in programfiles:
 282         shutil.copy(buildfolder + "/" + f, outfolder)
 283     # create zipball from output folder
 284     zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED)
 285     for root, dirs, files in os.walk(outfolder):
 286         for name in files:
 287             physname = os.path.join(root, name)
 288             filename = re.sub("^" + buildfolder, "", physname)
 289             zf.write(physname, filename)
 290         for name in dirs:
 291             physname = os.path.join(root, name)
 292             filename = re.sub("^" + buildfolder, "", physname)
 293             zf.write(physname, filename)
 294     zf.close()
 295     # remove output folder
 296     shutil.rmtree(outfolder)
 297     return archivename
 298
 299
 300 def tarball(versionstring, buildfolder):
 301     '''package created binary'''
 302     print "Creating binary tarball."
 303     archivebase = program + "-" + versionstring
 304     outfolder = buildfolder + "/" + archivebase
 305     archivename = archivebase + ".tar.bz2"
 306     # create output folder
 307     os.mkdir(outfolder)
 308     # move program files to output folder
 309     for f in programfiles:
 310         shutil.copy(buildfolder + "/" + f, outfolder)
 311     # create tarball from output folder
 312     tf = tarfile.open(archivename, mode='w:bz2')
 313     tf.add(outfolder, archivebase)
 314     tf.close()
 315     # remove output folder
 316     shutil.rmtree(outfolder)
 317     return archivename
 318
 319
 320 def macdeploy(versionstring, buildfolder):
 321     '''package created binary to dmg'''
 322     dmgfile = program + "-" + versionstring + ".dmg"
 323     appbundle = buildfolder + "/" + progexe
 324
 325     # workaround to Qt issues when building out-of-tree. Copy files into bundle.
 326     sourcebase = buildfolder + re.sub('rbutilqt.pro$', '', project)
 327     for src in bundlecopy:
 328         shutil.copy(sourcebase + src, appbundle + bundlecopy[src])
 329     # end of Qt workaround
 330
 331     output = subprocess.Popen(["macdeployqt", progexe, "-dmg"], stdout=subprocess.PIPE, cwd=buildfolder)
 332     output.communicate()
 333     if not output.returncode == 0:
 334         print "macdeployqt failed!"
 335         return -1
 336     # copy dmg to output folder
 337     shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile)
 338     return dmgfile
 339
 340 def filehashes(filename):
 341     '''Calculate md5 and sha1 hashes for a given file.'''
 342     if not os.path.exists(filename):
 343         return ["", ""]
 344     m = hashlib.md5()
 345     s = hashlib.sha1()
 346     f = open(filename, 'rb')
 347     while True:
 348         d = f.read(65536)
 349         if d == "":
 350             break
 351         m.update(d)
 352         s.update(d)
 353     return [m.hexdigest(), s.hexdigest()]
 354
 355
 356 def filestats(filename):
 357     if not os.path.exists(filename):
 358         return
 359     st = os.stat(filename)
 360     print filename, "\n", "-" * len(filename)
 361     print "Size:    %i bytes" % st.st_size
 362     h = filehashes(filename)
 363     print "md5sum:  %s" % h[0]
 364     print "sha1sum: %s" % h[1]
 365     print "-" * len(filename), "\n"
 366
 367
 368 def tempclean(workfolder, nopro):
 369     if nopro == True:
 370         print "Cleaning up working folder %s" % workfolder
 371         shutil.rmtree(workfolder)
 372     else:
 373         print "Project file specified or cleanup disabled!"
 374         print "Temporary files kept at %s" % workfolder
 375
 376
 377 def main():
 378     startup = time.time()
 379     try:
 380         opts, args = getopt.getopt(sys.argv[1:], "q:p:t:a:sbdkh",
 381             ["qmake=", "project=", "tag=", "add=", "source-only", "binary-only", "dynamic", "keep-temp", "help"])
 382     except getopt.GetoptError, err:
 383         print str(err)
 384         usage(sys.argv[0])
 385         sys.exit(1)
 386     qt = ""
 387     proj = ""
 388     svnbase = svnserver + "trunk/"
 389     tag = ""
 390     addfiles = []
 391     cleanup = True
 392     binary = True
 393     source = True
 394     keeptemp = False
 395     if sys.platform != "darwin":
 396         static = True
 397     else:
 398         static = False
 399     for o, a in opts:
 400         if o in ("-q", "--qmake"):
 401             qt = a
 402         if o in ("-p", "--project"):
 403             proj = a
 404             cleanup = False
 405         if o in ("-t", "--tag"):
 406             tag = a
 407             svnbase = svnserver + "tags/" + tag + "/"
 408         if o in ("-a", "--add"):
 409             addfiles.append(a)
 410         if o in ("-s", "--source-only"):
 411             binary = False
 412         if o in ("-b", "--binary-only"):
 413             source = False
 414         if o in ("-d", "--dynamic") and sys.platform != "darwin":
 415             static = False
 416         if o in ("-k", "--keep-temp"):
 417             keeptemp = True
 418         if o in ("-h", "--help"):
 419             usage(sys.argv[0])
 420             sys.exit(0)
 421
 422     if source == False and binary == False:
 423         print "Building build neither source nor binary means nothing to do. Exiting."
 424         sys.exit(1)
 425
 426     # search for qmake
 427     if qt == "":
 428         qm = findqt()
 429     else:
 430         qm = checkqt(qt)
 431     if qm == "":
 432         print "ERROR: No suitable Qt installation found."
 433         sys.exit(1)
 434
 435     # create working folder. Use current directory if -p option used.
 436     if proj == "":
 437         w = tempfile.mkdtemp()
 438         # make sure the path doesn't contain backslashes to prevent issues
 439         # later when running on windows.
 440         workfolder = re.sub(r'\\', '/', w)
 441         if not tag == "":
 442             sourcefolder = workfolder + "/" + tag + "/"
 443             archivename = tag + "-src.tar.bz2"
 444             # get numeric version part from tag
 445             ver = "v" + re.sub('^[^\d]+', '', tag)
 446         else:
 447             trunk = gettrunkrev(svnbase)
 448             sourcefolder = workfolder + "/" + program + "-r" + str(trunk) + "/"
 449             archivename = program + "-r" + str(trunk) + "-src.tar.bz2"
 450             ver = "r" + str(trunk)
 451         os.mkdir(sourcefolder)
 452     else:
 453         workfolder = "."
 454         sourcefolder = "."
 455         archivename = ""
 456     # check if project file explicitly given. If yes, don't get sources from svn
 457     if proj == "":
 458         proj = sourcefolder + project
 459         # get sources and pack source tarball
 460         if not getsources(svnbase, svnpaths, sourcefolder) == 0:
 461             tempclean(workfolder, cleanup and not keeptemp)
 462             sys.exit(1)
 463
 464         if source == True:
 465             tf = tarfile.open(archivename, mode='w:bz2')
 466             tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
 467             tf.close()
 468             if binary == False:
 469                 shutil.rmtree(workfolder)
 470                 sys.exit(0)
 471     else:
 472         # figure version from sources. Need to take path to project file into account.
 473         versionfile = re.subn('[\w\.]+$', "version.h", proj)[0]
 474         ver = findversion(versionfile)
 475
 476     # check project file
 477     if not os.path.exists(proj):
 478         print "ERROR: path to project file wrong."
 479         sys.exit(1)
 480
 481     # copy specified (--add) files to working folder
 482     for f in addfiles:
 483         shutil.copy(f, sourcefolder)
 484     buildstart = time.time()
 485     header = "Building %s %s" % (program, ver)
 486     print header
 487     print len(header) * "="
 488
 489     # build it.
 490     if not qmake(qm, proj, sourcefolder, static) == 0:
 491         tempclean(workfolder, cleanup and not keeptemp)
 492         sys.exit(1)
 493     if not build(sourcefolder) == 0:
 494         tempclean(workfolder, cleanup and not keeptemp)
 495         sys.exit(1)
 496     if sys.platform == "win32":
 497         if useupx == True:
 498             if not upxfile(sourcefolder) == 0:
 499                 tempclean(workfolder, cleanup and not keeptemp)
 500                 sys.exit(1)
 501         archive = zipball(ver, sourcefolder)
 502     elif sys.platform == "darwin":
 503         archive = macdeploy(ver, sourcefolder)
 504     else:
 505         if os.uname()[4].endswith("64"):
 506             ver += "-64bit"
 507         archive = tarball(ver, sourcefolder)
 508
 509     # remove temporary files
 510     tempclean(workfolder, cleanup)
 511
 512     # display summary
 513     headline = "Build Summary for %s" % program
 514     print "\n", headline, "\n", "=" * len(headline)
 515     if not archivename == "":
 516         filestats(archivename)
 517     filestats(archive)
 518     duration = time.time() - startup
 519     building = time.time() - buildstart
 520     durmins = (int)(duration / 60)
 521     dursecs = (int)(duration % 60)
 522     buildmins = (int)(building / 60)
 523     buildsecs = (int)(building % 60)
 524     print "Overall time %smin %ssec, building took %smin %ssec." % \
 525         (durmins, dursecs, buildmins, buildsecs)
 526
 527
 528 if __name__ == "__main__":
 529     main()
 530