rbutil/rbutilqt/deploy-release.py

   1 #!/usr/bin/python
   2 #             __________               __   ___.
   3 #   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4 #   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5 #   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6 #   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7 #                     \/            \/     \/    \/            \/
   8 # $Id$
   9 #
  10 # Copyright (c) 2009 Dominik Riebeling
  11 #
  12 # All files in this archive are subject to the GNU General Public License.
  13 # See the file COPYING in the source tree root for full license agreement.
  14 #
  15 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  16 # KIND, either express or implied.
  17 #
  18 #
  19 # Automate building releases for deployment.
  20 # Run from any folder to build
  21 # - trunk
  22 # - any tag (using the -t option)
  23 # - any local folder (using the -p option)
  24 # Will build a binary archive (tar.bz2 / zip) and source archive.
  25 # The source archive won't be built for local builds. Trunk and
  26 # tag builds will retrieve the sources directly from svn and build
  27 # below the systems temporary folder.
  28 #
  29 # If the required Qt installation isn't in PATH use --qmake option.
  30 # Tested on Linux and MinGW / W32
  31 #
  32 # requires python which package (http://code.google.com/p/which/)
  33 # requires pysvn package.
  34 # requires upx.exe in PATH on Windows.
  35 #
  36
  37 import re
  38 import os
  39 import sys
  40 import tarfile
  41 import zipfile
  42 import shutil
  43 import subprocess
  44 import getopt
  45 import time
  46 import hashlib
  47 import tempfile
  48
  49 # modules that are not part of python itself.
  50 try:
  51     import pysvn
  52 except ImportError:
  53     print "Fatal: This script requires the pysvn package to run."
  54     print "       See http://pysvn.tigris.org/."
  55     sys.exit(-5)
  56 try:
  57     import which
  58 except ImportError:
  59     print "Fatal: This script requires the which package to run."
  60     print "       See http://code.google.com/p/which/."
  61     sys.exit(-5)
  62
  63 # == Global stuff ==
  64 # Windows nees some special treatment. Differentiate between program name
  65 # and executable filename.
  66 program = "RockboxUtility"
  67 project = "rbutil/rbutilqt/rbutilqt.pro"
  68 environment = os.environ
  69 make = "make"
  70 if sys.platform == "win32":
  71     progexe = "Release/" + program + ".exe"
  72     make = "mingw32-make"
  73 elif sys.platform == "darwin":
  74     progexe = program + ".app"
  75     # OS X 10.6 defaults to gcc 4.2. Building universal binaries that are
  76     # compatible with 10.4 requires using gcc-4.0.
  77     if not "QMAKESPEC" in environment:
  78         environment["QMAKESPEC"] = "macx-g++40"
  79 else:
  80     progexe = program
  81
  82 programfiles = [ progexe ]
  83
  84 svnserver = "svn://svn.rockbox.org/rockbox/"
  85 # Paths and files to retrieve from svn when creating a tarball.
  86 # This is a mixed list, holding both paths and filenames.
  87 svnpaths = [ "rbutil/",
  88              "tools/ucl",
  89              "tools/rbspeex",
  90              "apps/codecs/libspeex",
  91              "docs/COPYING",
  92              "docs/CREDITS",
  93              "tools/iriver.c",
  94              "tools/Makefile",
  95              "tools/mkboot.h",
  96              "tools/voicefont.c",
  97              "tools/VOICE_PAUSE.wav",
  98              "tools/wavtrim.h",
  99              "tools/iriver.h",
 100              "tools/mkboot.c",
 101              "tools/telechips.c",
 102              "tools/telechips.h",
 103              "tools/voicefont.h",
 104              "tools/wavtrim.c",
 105              "tools/sapi_voice.vbs" ]
 106
 107 # == Functions ==
 108 def usage(myself):
 109     print "Usage: %s [options]" % myself
 110     print "       -q, --qmake=<qmake>   path to qmake"
 111     print "       -p, --project=<pro>   path to .pro file for building with local tree"
 112     print "       -t, --tag=<tag>       use specified tag from svn"
 113     print "       -a, --add=<file>      add file to build folder before building"
 114     print "       -s, --source-only     only create source archive"
 115     print "       -b, --binary-only     only create binary archive"
 116     if sys.platform != "darwin":
 117         print "       -d, --dynamic         link dynamically instead of static"
 118     print "       -k, --keep-temp       keep temporary folder on build failure"
 119     print "       -h, --help            this help"
 120     print "  If neither a project file nor tag is specified trunk will get downloaded"
 121     print "  from svn."
 122
 123 def getsources(svnsrv, filelist, dest):
 124     '''Get the files listed in filelist from svnsrv and put it at dest.'''
 125     client = pysvn.Client()
 126     print "Checking out sources from %s, please wait." % svnsrv
 127
 128     for elem in filelist:
 129         url = re.subn('/$', '', svnsrv + elem)[0]
 130         destpath = re.subn('/$', '', dest + elem)[0]
 131         # make sure the destination path does exist
 132         d = os.path.dirname(destpath)
 133         if not os.path.exists(d):
 134             os.makedirs(d)
 135         # get from svn
 136         try:
 137             client.export(url, destpath)
 138         except:
 139             print "SVN client error: %s" % sys.exc_value
 140             print "URL: %s, destination: %s" % (url, destpath)
 141             return -1
 142     print "Checkout finished."
 143     return 0
 144
 145
 146 def gettrunkrev(svnsrv):
 147     '''Get the revision of trunk for svnsrv'''
 148     client = pysvn.Client()
 149     entries = client.info2(svnsrv, recurse=False)
 150     return entries[0][1].rev.number
 151
 152
 153 def findversion(versionfile):
 154     '''figure most recent program version from version.h,
 155     returns version string.'''
 156     h = open(versionfile, "r")
 157     c = h.read()
 158     h.close()
 159     r = re.compile("#define +VERSION +\"(.[0-9\.a-z]+)\"")
 160     m = re.search(r, c)
 161     s = re.compile("\$Revision: +([0-9]+)")
 162     n = re.search(s, c)
 163     if n == None:
 164         print "WARNING: Revision not found!"
 165     return m.group(1)
 166
 167
 168 def findqt():
 169     '''Search for Qt4 installation. Return path to qmake.'''
 170     print "Searching for Qt"
 171     bins = ["qmake", "qmake-qt4"]
 172     for binary in bins:
 173         try:
 174             q = which.which(binary)
 175             if len(q) > 0:
 176                 result = checkqt(q)
 177                 if not result == "":
 178                     return result
 179         except:
 180             print sys.exc_value
 181
 182     return ""
 183
 184
 185 def checkqt(qmakebin):
 186     '''Check if given path to qmake exists and is a suitable version.'''
 187     result = ""
 188     # check if binary exists
 189     if not os.path.exists(qmakebin):
 190         print "Specified qmake path does not exist!"
 191         return result
 192     # check version
 193     output = subprocess.Popen([qmakebin, "-version"], stdout=subprocess.PIPE,
 194             stderr=subprocess.PIPE)
 195     cmdout = output.communicate()
 196     # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
 197     for ou in cmdout:
 198         r = re.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
 199         m = re.search(r, ou)
 200         if not m == None:
 201             print "Qt found: %s" % m.group(1)
 202             s = re.compile("4\..*")
 203             n = re.search(s, m.group(1))
 204             if not n == None:
 205                 result = qmakebin
 206     return result
 207
 208
 209 def qmake(qmake="qmake", projfile=project, wd=".", static=True):
 210     print "Running qmake in %s..." % wd
 211     command = [qmake, "-config", "release", "-config", "noccache"]
 212     if static == True:
 213         command.append("-config")
 214         command.append("static")
 215     command.append(projfile)
 216     output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd, env=environment)
 217     output.communicate()
 218     if not output.returncode == 0:
 219         print "qmake returned an error!"
 220         return -1
 221     return 0
 222
 223
 224 def build(wd="."):
 225     # make
 226     print "Building ..."
 227     output = subprocess.Popen([make], stdout=subprocess.PIPE, cwd=wd)
 228     while True:
 229         c = output.stdout.readline()
 230         sys.stdout.write(".")
 231         sys.stdout.flush()
 232         if not output.poll() == None:
 233             sys.stdout.write("\n")
 234             sys.stdout.flush()
 235             if not output.returncode == 0:
 236                 print "Build failed!"
 237                 return -1
 238             break
 239     if sys.platform != "darwin":
 240         # strip. OS X handles this via macdeployqt.
 241         print "Stripping binary."
 242         output = subprocess.Popen(["strip", progexe], stdout=subprocess.PIPE, cwd=wd)
 243         output.communicate()
 244         if not output.returncode == 0:
 245             print "Stripping failed!"
 246             return -1
 247     return 0
 248
 249
 250 def upxfile(wd="."):
 251     # run upx on binary
 252     print "UPX'ing binary ..."
 253     output = subprocess.Popen(["upx", progexe], stdout=subprocess.PIPE, cwd=wd)
 254     output.communicate()
 255     if not output.returncode == 0:
 256         print "UPX'ing failed!"
 257         return -1
 258     return 0
 259
 260
 261 def zipball(versionstring, buildfolder):
 262     '''package created binary'''
 263     print "Creating binary zipball."
 264     archivebase = program + "-" + versionstring
 265     outfolder = buildfolder + "/" + archivebase
 266     archivename = archivebase + ".zip"
 267     # create output folder
 268     os.mkdir(outfolder)
 269     # move program files to output folder
 270     for f in programfiles:
 271         shutil.copy(buildfolder + "/" + f, outfolder)
 272     # create zipball from output folder
 273     zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED)
 274     for root, dirs, files in os.walk(outfolder):
 275         for name in files:
 276             physname = os.path.join(root, name)
 277             filename = re.sub("^" + buildfolder, "", physname)
 278             zf.write(physname, filename)
 279         for name in dirs:
 280             physname = os.path.join(root, name)
 281             filename = re.sub("^" + buildfolder, "", physname)
 282             zf.write(physname, filename)
 283     zf.close()
 284     # remove output folder
 285     shutil.rmtree(outfolder)
 286     return archivename
 287
 288
 289 def tarball(versionstring, buildfolder):
 290     '''package created binary'''
 291     print "Creating binary tarball."
 292     archivebase = program + "-" + versionstring
 293     outfolder = buildfolder + "/" + archivebase
 294     archivename = archivebase + ".tar.bz2"
 295     # create output folder
 296     os.mkdir(outfolder)
 297     # move program files to output folder
 298     for f in programfiles:
 299         shutil.copy(buildfolder + "/" + f, outfolder)
 300     # create tarball from output folder
 301     tf = tarfile.open(archivename, mode='w:bz2')
 302     tf.add(outfolder, archivebase)
 303     tf.close()
 304     # remove output folder
 305     shutil.rmtree(outfolder)
 306     return archivename
 307
 308
 309 def macdeploy(versionstring, buildfolder):
 310     '''package created binary to dmg'''
 311     dmgfile = program + "-" + versionstring + ".dmg"
 312     appbundle = buildfolder + "/" + progexe
 313
 314     # workaround to Qt issues when building out-of-tree. Hardcoded for simplicity.
 315     sourcebase = buildfolder + re.sub('rbutilqt.pro$', '', project)
 316     shutil.copy(sourcebase + "icons/rbutilqt.icns", appbundle + "/Contents/Resources/")
 317     shutil.copy(sourcebase + "Info.plist", appbundle + "/Contents/")
 318     # end of Qt workaround
 319
 320     output = subprocess.Popen(["macdeployqt", progexe, "-dmg"], stdout=subprocess.PIPE, cwd=buildfolder)
 321     output.communicate()
 322     if not output.returncode == 0:
 323         print "macdeployqt failed!"
 324         return -1
 325     # copy dmg to output folder
 326     shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile)
 327     return dmgfile
 328
 329 def filehashes(filename):
 330     '''Calculate md5 and sha1 hashes for a given file.'''
 331     if not os.path.exists(filename):
 332         return ["", ""]
 333     m = hashlib.md5()
 334     s = hashlib.sha1()
 335     f = open(filename, 'rb')
 336     while True:
 337         d = f.read(65536)
 338         if d == "":
 339             break
 340         m.update(d)
 341         s.update(d)
 342     return [m.hexdigest(), s.hexdigest()]
 343
 344
 345 def filestats(filename):
 346     if not os.path.exists(filename):
 347         return
 348     st = os.stat(filename)
 349     print filename, "\n", "-" * len(filename)
 350     print "Size:    %i bytes" % st.st_size
 351     h = filehashes(filename)
 352     print "md5sum:  %s" % h[0]
 353     print "sha1sum: %s" % h[1]
 354     print "-" * len(filename), "\n"
 355
 356
 357 def tempclean(workfolder, nopro):
 358     if nopro == True:
 359         print "Cleaning up working folder %s" % workfolder
 360         shutil.rmtree(workfolder)
 361     else:
 362         print "Project file specified or cleanup disabled!"
 363         print "Temporary files kept at %s" % workfolder
 364
 365
 366 def main():
 367     startup = time.time()
 368     try:
 369         opts, args = getopt.getopt(sys.argv[1:], "q:p:t:a:sbdkh",
 370             ["qmake=", "project=", "tag=", "add=", "source-only", "binary-only", "dynamic", "keep-temp", "help"])
 371     except getopt.GetoptError, err:
 372         print str(err)
 373         usage(sys.argv[0])
 374         sys.exit(1)
 375     qt = ""
 376     proj = ""
 377     svnbase = svnserver + "trunk/"
 378     tag = ""
 379     addfiles = []
 380     cleanup = True
 381     binary = True
 382     source = True
 383     keeptemp = False
 384     if sys.platform != "darwin":
 385         static = True
 386     else:
 387         static = False
 388     for o, a in opts:
 389         if o in ("-q", "--qmake"):
 390             qt = a
 391         if o in ("-p", "--project"):
 392             proj = a
 393             cleanup = False
 394         if o in ("-t", "--tag"):
 395             tag = a
 396             svnbase = svnserver + "tags/" + tag + "/"
 397         if o in ("-a", "--add"):
 398             addfiles.append(a)
 399         if o in ("-s", "--source-only"):
 400             binary = False
 401         if o in ("-b", "--binary-only"):
 402             source = False
 403         if o in ("-d", "--dynamic") and sys.platform != "darwin":
 404             static = False
 405         if o in ("-k", "--keep-temp"):
 406             keeptemp = True
 407         if o in ("-h", "--help"):
 408             usage(sys.argv[0])
 409             sys.exit(0)
 410
 411     if source == False and binary == False:
 412         print "Building build neither source nor binary means nothing to do. Exiting."
 413         sys.exit(1)
 414
 415     # search for qmake
 416     if qt == "":
 417         qm = findqt()
 418     else:
 419         qm = checkqt(qt)
 420     if qm == "":
 421         print "ERROR: No suitable Qt installation found."
 422         sys.exit(1)
 423
 424     # create working folder. Use current directory if -p option used.
 425     if proj == "":
 426         w = tempfile.mkdtemp()
 427         # make sure the path doesn't contain backslashes to prevent issues
 428         # later when running on windows.
 429         workfolder = re.sub(r'\\', '/', w)
 430         if not tag == "":
 431             sourcefolder = workfolder + "/" + tag + "/"
 432             archivename = tag + "-src.tar.bz2"
 433             # get numeric version part from tag
 434             ver = "v" + re.sub('^[^\d]+', '', tag)
 435         else:
 436             trunk = gettrunkrev(svnbase)
 437             sourcefolder = workfolder + "/rbutil-r" + str(trunk) + "/"
 438             archivename = "rbutil-r" + str(trunk) + "-src.tar.bz2"
 439             ver = "r" + str(trunk)
 440         os.mkdir(sourcefolder)
 441     else:
 442         workfolder = "."
 443         sourcefolder = "."
 444         archivename = ""
 445     # check if project file explicitly given. If yes, don't get sources from svn
 446     if proj == "":
 447         proj = sourcefolder + project
 448         # get sources and pack source tarball
 449         if not getsources(svnbase, svnpaths, sourcefolder) == 0:
 450             tempclean(workfolder, cleanup and not keeptemp)
 451             sys.exit(1)
 452
 453         if source == True:
 454             tf = tarfile.open(archivename, mode='w:bz2')
 455             tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
 456             tf.close()
 457             if binary == False:
 458                 shutil.rmtree(workfolder)
 459                 sys.exit(0)
 460     else:
 461         # figure version from sources. Need to take path to project file into account.
 462         versionfile = re.subn('[\w\.]+$', "version.h", proj)[0]
 463         ver = findversion(versionfile)
 464
 465     # check project file
 466     if not os.path.exists(proj):
 467         print "ERROR: path to project file wrong."
 468         sys.exit(1)
 469
 470     # copy specified (--add) files to working folder
 471     for f in addfiles:
 472         shutil.copy(f, sourcefolder)
 473     buildstart = time.time()
 474     header = "Building %s %s" % (program, ver)
 475     print header
 476     print len(header) * "="
 477
 478     # build it.
 479     if not qmake(qm, proj, sourcefolder, static) == 0:
 480         tempclean(workfolder, cleanup and not keeptemp)
 481         sys.exit(1)
 482     if not build(sourcefolder) == 0:
 483         tempclean(workfolder, cleanup and not keeptemp)
 484         sys.exit(1)
 485     if sys.platform == "win32":
 486         if not upxfile(sourcefolder) == 0:
 487             tempclean(workfolder, cleanup and not keeptemp)
 488             sys.exit(1)
 489         archive = zipball(ver, sourcefolder)
 490     elif sys.platform == "darwin":
 491         archive = macdeploy(ver, sourcefolder)
 492     else:
 493         archive = tarball(ver, sourcefolder)
 494
 495     # remove temporary files
 496     tempclean(workfolder, cleanup)
 497
 498     # display summary
 499     headline = "Build Summary for %s" % program
 500     print "\n", headline, "\n", "=" * len(headline)
 501     if not archivename == "":
 502         filestats(archivename)
 503     filestats(archive)
 504     duration = time.time() - startup
 505     building = time.time() - buildstart
 506     durmins = (int)(duration / 60)
 507     dursecs = (int)(duration % 60)
 508     buildmins = (int)(building / 60)
 509     buildsecs = (int)(building % 60)
 510     print "Overall time %smin %ssec, building took %smin %ssec." % \
 511         (durmins, dursecs, buildmins, buildsecs)
 512
 513
 514 if __name__ == "__main__":
 515     main()
 516