rbutil/rbutilqt/deploy-release.py

   1 #!/usr/bin/python
   2 #             __________               __   ___.
   3 #   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4 #   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5 #   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6 #   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7 #                     \/            \/     \/    \/            \/
   8 # $Id$
   9 #
  10 # Copyright (c) 2009 Dominik Riebeling
  11 #
  12 # All files in this archive are subject to the GNU General Public License.
  13 # See the file COPYING in the source tree root for full license agreement.
  14 #
  15 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  16 # KIND, either express or implied.
  17 #
  18 #
  19 # Automate building releases for deployment.
  20 # Run from any folder to build
  21 # - trunk
  22 # - any tag (using the -t option)
  23 # - any local folder (using the -p option)
  24 # Will build a binary archive (tar.bz2 / zip) and source archive.
  25 # The source archive won't be built for local builds. Trunk and
  26 # tag builds will retrieve the sources directly from svn and build
  27 # below the systems temporary folder.
  28 #
  29 # If the required Qt installation isn't in PATH use --qmake option.
  30 # Tested on Linux and MinGW / W32
  31 #
  32 # requires python which package (http://code.google.com/p/which/)
  33 # requires pysvn package.
  34 # requires upx.exe in PATH on Windows.
  35 #
  36
  37 import re
  38 import os
  39 import sys
  40 import tarfile
  41 import zipfile
  42 import shutil
  43 import subprocess
  44 import getopt
  45 import time
  46 import hashlib
  47 import tempfile
  48
  49 # modules that are not part of python itself.
  50 try:
  51     import pysvn
  52 except ImportError:
  53     print "Fatal: This script requires the pysvn package to run."
  54     print "       See http://pysvn.tigris.org/."
  55     sys.exit(-5)
  56 try:
  57     import which
  58 except ImportError:
  59     print "Fatal: This script requires the which package to run."
  60     print "       See http://code.google.com/p/which/."
  61     sys.exit(-5)
  62
  63 # == Global stuff ==
  64 # Windows nees some special treatment. Differentiate between program name
  65 # and executable filename.
  66 program = "RockboxUtility"
  67 project = "rbutil/rbutilqt/rbutilqt.pro"
  68 environment = os.environ
  69 make = "make"
  70 if sys.platform == "win32":
  71     progexe = "Release/" + program + ".exe"
  72     make = "mingw32-make"
  73 elif sys.platform == "darwin":
  74     progexe = program + ".app"
  75     # OS X 10.6 defaults to gcc 4.2. Building universal binaries that are
  76     # compatible with 10.4 requires using gcc-4.0.
  77     if not "QMAKESPEC" in environment:
  78         environment["QMAKESPEC"] = "macx-g++40"
  79 else:
  80     progexe = program
  81
  82 programfiles = [ progexe ]
  83
  84 svnserver = "svn://svn.rockbox.org/rockbox/"
  85 # Paths and files to retrieve from svn when creating a tarball.
  86 # This is a mixed list, holding both paths and filenames.
  87 svnpaths = [ "rbutil/",
  88              "tools/ucl",
  89              "tools/rbspeex",
  90              "apps/codecs/libspeex",
  91              "docs/COPYING",
  92              "docs/CREDITS",
  93              "tools/iriver.c",
  94              "tools/Makefile",
  95              "tools/mkboot.h",
  96              "tools/voicefont.c",
  97              "tools/VOICE_PAUSE.wav",
  98              "tools/wavtrim.h",
  99              "tools/iriver.h",
 100              "tools/mkboot.c",
 101              "tools/telechips.c",
 102              "tools/telechips.h",
 103              "tools/voicefont.h",
 104              "tools/wavtrim.c",
 105              "tools/sapi_voice.vbs" ]
 106
 107 # == Functions ==
 108 def usage(myself):
 109     print "Usage: %s [options]" % myself
 110     print "       -q, --qmake=<qmake>   path to qmake"
 111     print "       -p, --project=<pro>   path to .pro file for building with local tree"
 112     print "       -t, --tag=<tag>       use specified tag from svn"
 113     print "       -a, --add=<file>      add file to build folder before building"
 114     print "       -s, --source-only     only create source archive"
 115     print "       -b, --binary-only     only create binary archive"
 116     if sys.platform != "darwin":
 117         print "       -d, --dynamic         link dynamically instead of static"
 118     print "       -h, --help            this help"
 119     print "  If neither a project file nor tag is specified trunk will get downloaded"
 120     print "  from svn."
 121
 122 def getsources(svnsrv, filelist, dest):
 123     '''Get the files listed in filelist from svnsrv and put it at dest.'''
 124     client = pysvn.Client()
 125     print "Checking out sources from %s, please wait." % svnsrv
 126
 127     for elem in filelist:
 128         url = re.subn('/$', '', svnsrv + elem)[0]
 129         destpath = re.subn('/$', '', dest + elem)[0]
 130         # make sure the destination path does exist
 131         d = os.path.dirname(destpath)
 132         if not os.path.exists(d):
 133             os.makedirs(d)
 134         # get from svn
 135         try:
 136             client.export(url, destpath)
 137         except:
 138             print "SVN client error: %s" % sys.exc_value
 139             print "URL: %s, destination: %s" % (url, destpath)
 140             return -1
 141     print "Checkout finished."
 142     return 0
 143
 144
 145 def gettrunkrev(svnsrv):
 146     '''Get the revision of trunk for svnsrv'''
 147     client = pysvn.Client()
 148     entries = client.info2(svnsrv, recurse=False)
 149     return entries[0][1].rev.number
 150
 151
 152 def findversion(versionfile):
 153     '''figure most recent program version from version.h,
 154     returns version string.'''
 155     h = open(versionfile, "r")
 156     c = h.read()
 157     h.close()
 158     r = re.compile("#define +VERSION +\"(.[0-9\.a-z]+)\"")
 159     m = re.search(r, c)
 160     s = re.compile("\$Revision: +([0-9]+)")
 161     n = re.search(s, c)
 162     if n == None:
 163         print "WARNING: Revision not found!"
 164     return m.group(1)
 165
 166
 167 def findqt():
 168     '''Search for Qt4 installation. Return path to qmake.'''
 169     print "Searching for Qt"
 170     bins = ["qmake", "qmake-qt4"]
 171     for binary in bins:
 172         try:
 173             q = which.which(binary)
 174             if len(q) > 0:
 175                 result = checkqt(q)
 176                 if not result == "":
 177                     return result
 178         except:
 179             print sys.exc_value
 180
 181     return ""
 182
 183
 184 def checkqt(qmakebin):
 185     '''Check if given path to qmake exists and is a suitable version.'''
 186     result = ""
 187     # check if binary exists
 188     if not os.path.exists(qmakebin):
 189         print "Specified qmake path does not exist!"
 190         return result
 191     # check version
 192     output = subprocess.Popen([qmakebin, "-version"], stdout=subprocess.PIPE,
 193             stderr=subprocess.PIPE)
 194     cmdout = output.communicate()
 195     # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
 196     for ou in cmdout:
 197         r = re.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
 198         m = re.search(r, ou)
 199         if not m == None:
 200             print "Qt found: %s" % m.group(1)
 201             s = re.compile("4\..*")
 202             n = re.search(s, m.group(1))
 203             if not n == None:
 204                 result = qmakebin
 205     return result
 206
 207
 208 def qmake(qmake="qmake", projfile=project, wd=".", static=True):
 209     print "Running qmake in %s..." % wd
 210     command = [qmake, "-config", "release", "-config", "noccache"]
 211     if static == True:
 212         command.append("-config")
 213         command.append("static")
 214     command.append(projfile)
 215     output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd, env=environment)
 216     output.communicate()
 217     if not output.returncode == 0:
 218         print "qmake returned an error!"
 219         return -1
 220     return 0
 221
 222
 223 def build(wd="."):
 224     # make
 225     print "Building ..."
 226     output = subprocess.Popen([make], stdout=subprocess.PIPE, cwd=wd)
 227     while True:
 228         c = output.stdout.readline()
 229         sys.stdout.write(".")
 230         sys.stdout.flush()
 231         if not output.poll() == None:
 232             sys.stdout.write("\n")
 233             sys.stdout.flush()
 234             if not output.returncode == 0:
 235                 print "Build failed!"
 236                 return -1
 237             break
 238     if sys.platform != "darwin":
 239         # strip. OS X handles this via macdeployqt.
 240         print "Stripping binary."
 241         output = subprocess.Popen(["strip", progexe], stdout=subprocess.PIPE, cwd=wd)
 242         output.communicate()
 243         if not output.returncode == 0:
 244             print "Stripping failed!"
 245             return -1
 246     return 0
 247
 248
 249 def upxfile(wd="."):
 250     # run upx on binary
 251     print "UPX'ing binary ..."
 252     output = subprocess.Popen(["upx", progexe], stdout=subprocess.PIPE, cwd=wd)
 253     output.communicate()
 254     if not output.returncode == 0:
 255         print "UPX'ing failed!"
 256         return -1
 257     return 0
 258
 259
 260 def zipball(versionstring, buildfolder):
 261     '''package created binary'''
 262     print "Creating binary zipball."
 263     archivebase = program + "-" + versionstring
 264     outfolder = buildfolder + "/" + archivebase
 265     archivename = archivebase + ".zip"
 266     # create output folder
 267     os.mkdir(outfolder)
 268     # move program files to output folder
 269     for f in programfiles:
 270         shutil.copy(buildfolder + "/" + f, outfolder)
 271     # create zipball from output folder
 272     zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED)
 273     for root, dirs, files in os.walk(outfolder):
 274         for name in files:
 275             physname = os.path.join(root, name)
 276             filename = re.sub("^" + buildfolder, "", physname)
 277             zf.write(physname, filename)
 278         for name in dirs:
 279             physname = os.path.join(root, name)
 280             filename = re.sub("^" + buildfolder, "", physname)
 281             zf.write(physname, filename)
 282     zf.close()
 283     # remove output folder
 284     shutil.rmtree(outfolder)
 285     return archivename
 286
 287
 288 def tarball(versionstring, buildfolder):
 289     '''package created binary'''
 290     print "Creating binary tarball."
 291     archivebase = program + "-" + versionstring
 292     outfolder = buildfolder + "/" + archivebase
 293     archivename = archivebase + ".tar.bz2"
 294     # create output folder
 295     os.mkdir(outfolder)
 296     # move program files to output folder
 297     for f in programfiles:
 298         shutil.copy(buildfolder + "/" + f, outfolder)
 299     # create tarball from output folder
 300     tf = tarfile.open(archivename, mode='w:bz2')
 301     tf.add(outfolder, archivebase)
 302     tf.close()
 303     # remove output folder
 304     shutil.rmtree(outfolder)
 305     return archivename
 306
 307
 308 def macdeploy(versionstring, buildfolder):
 309     '''package created binary to dmg'''
 310     dmgfile = program + "-" + versionstring + ".dmg"
 311     appbundle = buildfolder + "/" + progexe
 312
 313     # workaround to Qt issues when building out-of-tree. Hardcoded for simplicity.
 314     sourcebase = buildfolder + re.sub('rbutilqt.pro$', '', project)
 315     shutil.copy(sourcebase + "icons/rbutilqt.icns", appbundle + "/Contents/Resources/")
 316     shutil.copy(sourcebase + "Info.plist", appbundle + "/Contents/")
 317     # end of Qt workaround
 318
 319     output = subprocess.Popen(["macdeployqt", progexe, "-dmg"], stdout=subprocess.PIPE, cwd=buildfolder)
 320     output.communicate()
 321     if not output.returncode == 0:
 322         print "macdeployqt failed!"
 323         return -1
 324     # copy dmg to output folder
 325     shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile)
 326     return dmgfile
 327
 328 def filehashes(filename):
 329     '''Calculate md5 and sha1 hashes for a given file.'''
 330     if not os.path.exists(filename):
 331         return ["", ""]
 332     m = hashlib.md5()
 333     s = hashlib.sha1()
 334     f = open(filename, 'rb')
 335     while True:
 336         d = f.read(65536)
 337         if d == "":
 338             break
 339         m.update(d)
 340         s.update(d)
 341     return [m.hexdigest(), s.hexdigest()]
 342
 343
 344 def filestats(filename):
 345     if not os.path.exists(filename):
 346         return
 347     st = os.stat(filename)
 348     print filename, "\n", "-" * len(filename)
 349     print "Size:    %i bytes" % st.st_size
 350     h = filehashes(filename)
 351     print "md5sum:  %s" % h[0]
 352     print "sha1sum: %s" % h[1]
 353     print "-" * len(filename), "\n"
 354
 355
 356 def main():
 357     startup = time.time()
 358     try:
 359         opts, args = getopt.getopt(sys.argv[1:], "q:p:t:a:sbdh",
 360             ["qmake=", "project=", "tag=", "add=", "source-only", "binary-only", "dynamic", "help"])
 361     except getopt.GetoptError, err:
 362         print str(err)
 363         usage(sys.argv[0])
 364         sys.exit(1)
 365     qt = ""
 366     proj = ""
 367     svnbase = svnserver + "trunk/"
 368     tag = ""
 369     addfiles = []
 370     cleanup = True
 371     binary = True
 372     source = True
 373     if sys.platform != "darwin":
 374         static = True
 375     else:
 376         static = False
 377     for o, a in opts:
 378         if o in ("-q", "--qmake"):
 379             qt = a
 380         if o in ("-p", "--project"):
 381             proj = a
 382             cleanup = False
 383         if o in ("-t", "--tag"):
 384             tag = a
 385             svnbase = svnserver + "tags/" + tag + "/"
 386         if o in ("-a", "--add"):
 387             addfiles.append(a)
 388         if o in ("-s", "--source-only"):
 389             binary = False
 390         if o in ("-b", "--binary-only"):
 391             source = False
 392         if o in ("-d", "--dynamic") and sys.platform != "darwin":
 393             static = False
 394         if o in ("-h", "--help"):
 395             usage(sys.argv[0])
 396             sys.exit(0)
 397
 398     if source == False and binary == False:
 399         print "Building build neither source nor binary means nothing to do. Exiting."
 400         sys.exit(1)
 401
 402     # search for qmake
 403     if qt == "":
 404         qm = findqt()
 405     else:
 406         qm = checkqt(qt)
 407     if qm == "":
 408         print "ERROR: No suitable Qt installation found."
 409         sys.exit(1)
 410
 411     # create working folder. Use current directory if -p option used.
 412     if proj == "":
 413         w = tempfile.mkdtemp()
 414         # make sure the path doesn't contain backslashes to prevent issues
 415         # later when running on windows.
 416         workfolder = re.sub(r'\\', '/', w)
 417         if not tag == "":
 418             sourcefolder = workfolder + "/" + tag + "/"
 419             archivename = tag + "-src.tar.bz2"
 420             # get numeric version part from tag
 421             ver = "v" + re.sub('^[^\d]+', '', tag)
 422         else:
 423             trunk = gettrunkrev(svnbase)
 424             sourcefolder = workfolder + "/rbutil-r" + str(trunk) + "/"
 425             archivename = "rbutil-r" + str(trunk) + "-src.tar.bz2"
 426             ver = "r" + str(trunk)
 427         os.mkdir(sourcefolder)
 428     else:
 429         workfolder = "."
 430         sourcefolder = "."
 431         archivename = ""
 432     # check if project file explicitly given. If yes, don't get sources from svn
 433     if proj == "":
 434         proj = sourcefolder + project
 435         # get sources and pack source tarball
 436         if not getsources(svnbase, svnpaths, sourcefolder) == 0:
 437             sys.exit(1)
 438
 439         if source == True:
 440             tf = tarfile.open(archivename, mode='w:bz2')
 441             tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
 442             tf.close()
 443             if binary == False:
 444                 shutil.rmtree(workfolder)
 445                 sys.exit(0)
 446     else:
 447         # figure version from sources. Need to take path to project file into account.
 448         versionfile = re.subn('[\w\.]+$', "version.h", proj)[0]
 449         ver = findversion(versionfile)
 450
 451     # check project file
 452     if not os.path.exists(proj):
 453         print "ERROR: path to project file wrong."
 454         sys.exit(1)
 455
 456     # copy specified (--add) files to working folder
 457     for f in addfiles:
 458         shutil.copy(f, sourcefolder)
 459     buildstart = time.time()
 460     header = "Building %s %s" % (program, ver)
 461     print header
 462     print len(header) * "="
 463
 464     # build it.
 465     if not qmake(qm, proj, sourcefolder, static) == 0:
 466         sys.exit(1)
 467     if not build(sourcefolder) == 0:
 468         sys.exit(1)
 469     if sys.platform == "win32":
 470         if not upxfile(sourcefolder) == 0:
 471             sys.exit(1)
 472         archive = zipball(ver, sourcefolder)
 473     elif sys.platform == "darwin":
 474         archive = macdeploy(ver, sourcefolder)
 475     else:
 476         archive = tarball(ver, sourcefolder)
 477
 478     # remove temporary files
 479     print "Cleaning up working folder %s" % workfolder
 480     if cleanup == True:
 481         shutil.rmtree(workfolder)
 482     else:
 483         print "Project file specified, not cleaning up!"
 484
 485     # display summary
 486     headline = "Build Summary for %s" % program
 487     print "\n", headline, "\n", "=" * len(headline)
 488     if not archivename == "":
 489         filestats(archivename)
 490     filestats(archive)
 491     duration = time.time() - startup
 492     building = time.time() - buildstart
 493     durmins = (int)(duration / 60)
 494     dursecs = (int)(duration % 60)
 495     buildmins = (int)(building / 60)
 496     buildsecs = (int)(building % 60)
 497     print "Overall time %smin %ssec, building took %smin %ssec." % \
 498         (durmins, dursecs, buildmins, buildsecs)
 499
 500
 501 if __name__ == "__main__":
 502     main()
 503