rbutil/rbutilqt/deploy-release.py

   1 #!/usr/bin/python
   2 #             __________               __   ___.
   3 #   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4 #   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5 #   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6 #   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7 #                     \/            \/     \/    \/            \/
   8 # $Id$
   9 #
  10 # Copyright (c) 2009 Dominik Riebeling
  11 #
  12 # All files in this archive are subject to the GNU General Public License.
  13 # See the file COPYING in the source tree root for full license agreement.
  14 #
  15 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  16 # KIND, either express or implied.
  17 #
  18 #
  19 # Automate building releases for deployment.
  20 # Run from any folder to build
  21 # - trunk
  22 # - any tag (using the -t option)
  23 # - any local folder (using the -p option)
  24 # Will build a binary archive (tar.bz2 / zip) and source archive.
  25 # The source archive won't be built for local builds. Trunk and
  26 # tag builds will retrieve the sources directly from svn and build
  27 # below the systems temporary folder.
  28 #
  29 # If the required Qt installation isn't in PATH use --qmake option.
  30 # Tested on Linux and MinGW / W32
  31 #
  32 # requires python which package (http://code.google.com/p/which/)
  33 # requires pysvn package.
  34 # requires upx.exe in PATH on Windows.
  35 #
  36
  37 import re
  38 import os
  39 import sys
  40 import tarfile
  41 import zipfile
  42 import shutil
  43 import subprocess
  44 import getopt
  45 import time
  46 import hashlib
  47 import tempfile
  48
  49 # modules that are not part of python itself.
  50 try:
  51     import pysvn
  52 except ImportError:
  53     print "Fatal: This script requires the pysvn package to run."
  54     print "       See http://pysvn.tigris.org/."
  55     sys.exit(-5)
  56 try:
  57     import which
  58 except ImportError:
  59     print "Fatal: This script requires the which package to run."
  60     print "       See http://code.google.com/p/which/."
  61     sys.exit(-5)
  62
  63 # == Global stuff ==
  64 # Windows nees some special treatment. Differentiate between program name
  65 # and executable filename.
  66 program = "RockboxUtility"
  67 project = "rbutil/rbutilqt/rbutilqt.pro"
  68 environment = os.environ
  69 make = "make"
  70 if sys.platform == "win32":
  71     progexe = "Release/" + program + ".exe"
  72     make = "mingw32-make"
  73 elif sys.platform == "darwin":
  74     progexe = program + ".app"
  75     # OS X 10.6 defaults to gcc 4.2. Building universal binaries that are
  76     # compatible with 10.4 requires using gcc-4.0.
  77     if not "QMAKESPEC" in environment:
  78         environment["QMAKESPEC"] = "macx-g++40"
  79 else:
  80     progexe = program
  81
  82 # all files of the program. Will get put into an archive after building
  83 # (zip on w32, tar.bz2 on Linux). Does not apply on Mac which uses dmg.
  84 programfiles = [ progexe ]
  85
  86 svnserver = "svn://svn.rockbox.org/rockbox/"
  87 # Paths and files to retrieve from svn when creating a tarball.
  88 # This is a mixed list, holding both paths and filenames.
  89 svnpaths = [ "rbutil/",
  90              "tools/ucl",
  91              "tools/rbspeex",
  92              "apps/codecs/libspeex",
  93              "docs/COPYING",
  94              "docs/CREDITS",
  95              "tools/iriver.c",
  96              "tools/Makefile",
  97              "tools/mkboot.h",
  98              "tools/voicefont.c",
  99              "tools/VOICE_PAUSE.wav",
 100              "tools/wavtrim.h",
 101              "tools/iriver.h",
 102              "tools/mkboot.c",
 103              "tools/telechips.c",
 104              "tools/telechips.h",
 105              "tools/voicefont.h",
 106              "tools/wavtrim.c",
 107              "tools/sapi_voice.vbs" ]
 108 # set this to true to run upx on the resulting binary, false to skip this step.
 109 # only used on w32.
 110 useupx = False
 111
 112 # == Functions ==
 113 def usage(myself):
 114     print "Usage: %s [options]" % myself
 115     print "       -q, --qmake=<qmake>   path to qmake"
 116     print "       -p, --project=<pro>   path to .pro file for building with local tree"
 117     print "       -t, --tag=<tag>       use specified tag from svn"
 118     print "       -a, --add=<file>      add file to build folder before building"
 119     print "       -s, --source-only     only create source archive"
 120     print "       -b, --binary-only     only create binary archive"
 121     if sys.platform != "darwin":
 122         print "       -d, --dynamic         link dynamically instead of static"
 123     print "       -k, --keep-temp       keep temporary folder on build failure"
 124     print "       -h, --help            this help"
 125     print "  If neither a project file nor tag is specified trunk will get downloaded"
 126     print "  from svn."
 127
 128 def getsources(svnsrv, filelist, dest):
 129     '''Get the files listed in filelist from svnsrv and put it at dest.'''
 130     client = pysvn.Client()
 131     print "Checking out sources from %s, please wait." % svnsrv
 132
 133     for elem in filelist:
 134         url = re.subn('/$', '', svnsrv + elem)[0]
 135         destpath = re.subn('/$', '', dest + elem)[0]
 136         # make sure the destination path does exist
 137         d = os.path.dirname(destpath)
 138         if not os.path.exists(d):
 139             os.makedirs(d)
 140         # get from svn
 141         try:
 142             client.export(url, destpath)
 143         except:
 144             print "SVN client error: %s" % sys.exc_value
 145             print "URL: %s, destination: %s" % (url, destpath)
 146             return -1
 147     print "Checkout finished."
 148     return 0
 149
 150
 151 def gettrunkrev(svnsrv):
 152     '''Get the revision of trunk for svnsrv'''
 153     client = pysvn.Client()
 154     entries = client.info2(svnsrv, recurse=False)
 155     return entries[0][1].rev.number
 156
 157
 158 def findversion(versionfile):
 159     '''figure most recent program version from version.h,
 160     returns version string.'''
 161     h = open(versionfile, "r")
 162     c = h.read()
 163     h.close()
 164     r = re.compile("#define +VERSION +\"(.[0-9\.a-z]+)\"")
 165     m = re.search(r, c)
 166     s = re.compile("\$Revision: +([0-9]+)")
 167     n = re.search(s, c)
 168     if n == None:
 169         print "WARNING: Revision not found!"
 170     return m.group(1)
 171
 172
 173 def findqt():
 174     '''Search for Qt4 installation. Return path to qmake.'''
 175     print "Searching for Qt"
 176     bins = ["qmake", "qmake-qt4"]
 177     for binary in bins:
 178         try:
 179             q = which.which(binary)
 180             if len(q) > 0:
 181                 result = checkqt(q)
 182                 if not result == "":
 183                     return result
 184         except:
 185             print sys.exc_value
 186
 187     return ""
 188
 189
 190 def checkqt(qmakebin):
 191     '''Check if given path to qmake exists and is a suitable version.'''
 192     result = ""
 193     # check if binary exists
 194     if not os.path.exists(qmakebin):
 195         print "Specified qmake path does not exist!"
 196         return result
 197     # check version
 198     output = subprocess.Popen([qmakebin, "-version"], stdout=subprocess.PIPE,
 199             stderr=subprocess.PIPE)
 200     cmdout = output.communicate()
 201     # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
 202     for ou in cmdout:
 203         r = re.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
 204         m = re.search(r, ou)
 205         if not m == None:
 206             print "Qt found: %s" % m.group(1)
 207             s = re.compile("4\..*")
 208             n = re.search(s, m.group(1))
 209             if not n == None:
 210                 result = qmakebin
 211     return result
 212
 213
 214 def qmake(qmake="qmake", projfile=project, wd=".", static=True):
 215     print "Running qmake in %s..." % wd
 216     command = [qmake, "-config", "release", "-config", "noccache"]
 217     if static == True:
 218         command.append("-config")
 219         command.append("static")
 220     command.append(projfile)
 221     output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd, env=environment)
 222     output.communicate()
 223     if not output.returncode == 0:
 224         print "qmake returned an error!"
 225         return -1
 226     return 0
 227
 228
 229 def build(wd="."):
 230     # make
 231     print "Building ..."
 232     output = subprocess.Popen([make], stdout=subprocess.PIPE, cwd=wd)
 233     while True:
 234         c = output.stdout.readline()
 235         sys.stdout.write(".")
 236         sys.stdout.flush()
 237         if not output.poll() == None:
 238             sys.stdout.write("\n")
 239             sys.stdout.flush()
 240             if not output.returncode == 0:
 241                 print "Build failed!"
 242                 return -1
 243             break
 244     if sys.platform != "darwin":
 245         # strip. OS X handles this via macdeployqt.
 246         print "Stripping binary."
 247         output = subprocess.Popen(["strip", progexe], stdout=subprocess.PIPE, cwd=wd)
 248         output.communicate()
 249         if not output.returncode == 0:
 250             print "Stripping failed!"
 251             return -1
 252     return 0
 253
 254
 255 def upxfile(wd="."):
 256     # run upx on binary
 257     print "UPX'ing binary ..."
 258     output = subprocess.Popen(["upx", progexe], stdout=subprocess.PIPE, cwd=wd)
 259     output.communicate()
 260     if not output.returncode == 0:
 261         print "UPX'ing failed!"
 262         return -1
 263     return 0
 264
 265
 266 def zipball(versionstring, buildfolder):
 267     '''package created binary'''
 268     print "Creating binary zipball."
 269     archivebase = program + "-" + versionstring
 270     outfolder = buildfolder + "/" + archivebase
 271     archivename = archivebase + ".zip"
 272     # create output folder
 273     os.mkdir(outfolder)
 274     # move program files to output folder
 275     for f in programfiles:
 276         shutil.copy(buildfolder + "/" + f, outfolder)
 277     # create zipball from output folder
 278     zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED)
 279     for root, dirs, files in os.walk(outfolder):
 280         for name in files:
 281             physname = os.path.join(root, name)
 282             filename = re.sub("^" + buildfolder, "", physname)
 283             zf.write(physname, filename)
 284         for name in dirs:
 285             physname = os.path.join(root, name)
 286             filename = re.sub("^" + buildfolder, "", physname)
 287             zf.write(physname, filename)
 288     zf.close()
 289     # remove output folder
 290     shutil.rmtree(outfolder)
 291     return archivename
 292
 293
 294 def tarball(versionstring, buildfolder):
 295     '''package created binary'''
 296     print "Creating binary tarball."
 297     archivebase = program + "-" + versionstring
 298     outfolder = buildfolder + "/" + archivebase
 299     archivename = archivebase + ".tar.bz2"
 300     # create output folder
 301     os.mkdir(outfolder)
 302     # move program files to output folder
 303     for f in programfiles:
 304         shutil.copy(buildfolder + "/" + f, outfolder)
 305     # create tarball from output folder
 306     tf = tarfile.open(archivename, mode='w:bz2')
 307     tf.add(outfolder, archivebase)
 308     tf.close()
 309     # remove output folder
 310     shutil.rmtree(outfolder)
 311     return archivename
 312
 313
 314 def macdeploy(versionstring, buildfolder):
 315     '''package created binary to dmg'''
 316     dmgfile = program + "-" + versionstring + ".dmg"
 317     appbundle = buildfolder + "/" + progexe
 318
 319     # workaround to Qt issues when building out-of-tree. Hardcoded for simplicity.
 320     sourcebase = buildfolder + re.sub('rbutilqt.pro$', '', project)
 321     shutil.copy(sourcebase + "icons/rbutilqt.icns", appbundle + "/Contents/Resources/")
 322     shutil.copy(sourcebase + "Info.plist", appbundle + "/Contents/")
 323     # end of Qt workaround
 324
 325     output = subprocess.Popen(["macdeployqt", progexe, "-dmg"], stdout=subprocess.PIPE, cwd=buildfolder)
 326     output.communicate()
 327     if not output.returncode == 0:
 328         print "macdeployqt failed!"
 329         return -1
 330     # copy dmg to output folder
 331     shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile)
 332     return dmgfile
 333
 334 def filehashes(filename):
 335     '''Calculate md5 and sha1 hashes for a given file.'''
 336     if not os.path.exists(filename):
 337         return ["", ""]
 338     m = hashlib.md5()
 339     s = hashlib.sha1()
 340     f = open(filename, 'rb')
 341     while True:
 342         d = f.read(65536)
 343         if d == "":
 344             break
 345         m.update(d)
 346         s.update(d)
 347     return [m.hexdigest(), s.hexdigest()]
 348
 349
 350 def filestats(filename):
 351     if not os.path.exists(filename):
 352         return
 353     st = os.stat(filename)
 354     print filename, "\n", "-" * len(filename)
 355     print "Size:    %i bytes" % st.st_size
 356     h = filehashes(filename)
 357     print "md5sum:  %s" % h[0]
 358     print "sha1sum: %s" % h[1]
 359     print "-" * len(filename), "\n"
 360
 361
 362 def tempclean(workfolder, nopro):
 363     if nopro == True:
 364         print "Cleaning up working folder %s" % workfolder
 365         shutil.rmtree(workfolder)
 366     else:
 367         print "Project file specified or cleanup disabled!"
 368         print "Temporary files kept at %s" % workfolder
 369
 370
 371 def main():
 372     startup = time.time()
 373     try:
 374         opts, args = getopt.getopt(sys.argv[1:], "q:p:t:a:sbdkh",
 375             ["qmake=", "project=", "tag=", "add=", "source-only", "binary-only", "dynamic", "keep-temp", "help"])
 376     except getopt.GetoptError, err:
 377         print str(err)
 378         usage(sys.argv[0])
 379         sys.exit(1)
 380     qt = ""
 381     proj = ""
 382     svnbase = svnserver + "trunk/"
 383     tag = ""
 384     addfiles = []
 385     cleanup = True
 386     binary = True
 387     source = True
 388     keeptemp = False
 389     if sys.platform != "darwin":
 390         static = True
 391     else:
 392         static = False
 393     for o, a in opts:
 394         if o in ("-q", "--qmake"):
 395             qt = a
 396         if o in ("-p", "--project"):
 397             proj = a
 398             cleanup = False
 399         if o in ("-t", "--tag"):
 400             tag = a
 401             svnbase = svnserver + "tags/" + tag + "/"
 402         if o in ("-a", "--add"):
 403             addfiles.append(a)
 404         if o in ("-s", "--source-only"):
 405             binary = False
 406         if o in ("-b", "--binary-only"):
 407             source = False
 408         if o in ("-d", "--dynamic") and sys.platform != "darwin":
 409             static = False
 410         if o in ("-k", "--keep-temp"):
 411             keeptemp = True
 412         if o in ("-h", "--help"):
 413             usage(sys.argv[0])
 414             sys.exit(0)
 415
 416     if source == False and binary == False:
 417         print "Building build neither source nor binary means nothing to do. Exiting."
 418         sys.exit(1)
 419
 420     # search for qmake
 421     if qt == "":
 422         qm = findqt()
 423     else:
 424         qm = checkqt(qt)
 425     if qm == "":
 426         print "ERROR: No suitable Qt installation found."
 427         sys.exit(1)
 428
 429     # create working folder. Use current directory if -p option used.
 430     if proj == "":
 431         w = tempfile.mkdtemp()
 432         # make sure the path doesn't contain backslashes to prevent issues
 433         # later when running on windows.
 434         workfolder = re.sub(r'\\', '/', w)
 435         if not tag == "":
 436             sourcefolder = workfolder + "/" + tag + "/"
 437             archivename = tag + "-src.tar.bz2"
 438             # get numeric version part from tag
 439             ver = "v" + re.sub('^[^\d]+', '', tag)
 440         else:
 441             trunk = gettrunkrev(svnbase)
 442             sourcefolder = workfolder + "/" + program + "-r" + str(trunk) + "/"
 443             archivename = program + "-r" + str(trunk) + "-src.tar.bz2"
 444             ver = "r" + str(trunk)
 445         os.mkdir(sourcefolder)
 446     else:
 447         workfolder = "."
 448         sourcefolder = "."
 449         archivename = ""
 450     # check if project file explicitly given. If yes, don't get sources from svn
 451     if proj == "":
 452         proj = sourcefolder + project
 453         # get sources and pack source tarball
 454         if not getsources(svnbase, svnpaths, sourcefolder) == 0:
 455             tempclean(workfolder, cleanup and not keeptemp)
 456             sys.exit(1)
 457
 458         if source == True:
 459             tf = tarfile.open(archivename, mode='w:bz2')
 460             tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
 461             tf.close()
 462             if binary == False:
 463                 shutil.rmtree(workfolder)
 464                 sys.exit(0)
 465     else:
 466         # figure version from sources. Need to take path to project file into account.
 467         versionfile = re.subn('[\w\.]+$', "version.h", proj)[0]
 468         ver = findversion(versionfile)
 469
 470     # check project file
 471     if not os.path.exists(proj):
 472         print "ERROR: path to project file wrong."
 473         sys.exit(1)
 474
 475     # copy specified (--add) files to working folder
 476     for f in addfiles:
 477         shutil.copy(f, sourcefolder)
 478     buildstart = time.time()
 479     header = "Building %s %s" % (program, ver)
 480     print header
 481     print len(header) * "="
 482
 483     # build it.
 484     if not qmake(qm, proj, sourcefolder, static) == 0:
 485         tempclean(workfolder, cleanup and not keeptemp)
 486         sys.exit(1)
 487     if not build(sourcefolder) == 0:
 488         tempclean(workfolder, cleanup and not keeptemp)
 489         sys.exit(1)
 490     if sys.platform == "win32":
 491         if useupx == True:
 492             if not upxfile(sourcefolder) == 0:
 493                 tempclean(workfolder, cleanup and not keeptemp)
 494                 sys.exit(1)
 495         archive = zipball(ver, sourcefolder)
 496     elif sys.platform == "darwin":
 497         archive = macdeploy(ver, sourcefolder)
 498     else:
 499         archive = tarball(ver, sourcefolder)
 500
 501     # remove temporary files
 502     tempclean(workfolder, cleanup)
 503
 504     # display summary
 505     headline = "Build Summary for %s" % program
 506     print "\n", headline, "\n", "=" * len(headline)
 507     if not archivename == "":
 508         filestats(archivename)
 509     filestats(archive)
 510     duration = time.time() - startup
 511     building = time.time() - buildstart
 512     durmins = (int)(duration / 60)
 513     dursecs = (int)(duration % 60)
 514     buildmins = (int)(building / 60)
 515     buildsecs = (int)(building % 60)
 516     print "Overall time %smin %ssec, building took %smin %ssec." % \
 517         (durmins, dursecs, buildmins, buildsecs)
 518
 519
 520 if __name__ == "__main__":
 521     main()
 522