rbutil/rbutilqt/deploy-release.py

   1 #!/usr/bin/python
   2 #             __________               __   ___.
   3 #   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4 #   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5 #   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6 #   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7 #                     \/            \/     \/    \/            \/
   8 # $Id$
   9 #
  10 # Copyright (c) 2009 Dominik Riebeling
  11 #
  12 # All files in this archive are subject to the GNU General Public License.
  13 # See the file COPYING in the source tree root for full license agreement.
  14 #
  15 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  16 # KIND, either express or implied.
  17 #
  18 #
  19 # Automate building releases for deployment.
  20 # Run from any folder to build
  21 # - trunk
  22 # - any tag (using the -t option)
  23 # - any local folder (using the -p option)
  24 # Will build a binary archive (tar.bz2 / zip) and source archive.
  25 # The source archive won't be built for local builds. Trunk and
  26 # tag builds will retrieve the sources directly from svn and build
  27 # below the systems temporary folder.
  28 #
  29 # If the required Qt installation isn't in PATH use --qmake option.
  30 # Tested on Linux and MinGW / W32
  31 #
  32 # requires python which package (http://code.google.com/p/which/)
  33 # requires pysvn package.
  34 # requires upx.exe in PATH on Windows.
  35 #
  36
  37 import re
  38 import os
  39 import sys
  40 import tarfile
  41 import zipfile
  42 import shutil
  43 import subprocess
  44 import getopt
  45 import time
  46 import hashlib
  47 import tempfile
  48
  49 # modules that are not part of python itself.
  50 try:
  51     import pysvn
  52 except ImportError:
  53     print "Fatal: This script requires the pysvn package to run."
  54     print "       See http://pysvn.tigris.org/."
  55     sys.exit(-5)
  56 try:
  57     import which
  58 except ImportError:
  59     print "Fatal: This script requires the which package to run."
  60     print "       See http://code.google.com/p/which/."
  61     sys.exit(-5)
  62
  63 # == Global stuff ==
  64 # Windows nees some special treatment. Differentiate between program name
  65 # and executable filename.
  66 program = "rbutilqt"
  67 project = "rbutil/rbutilqt/rbutilqt.pro"
  68 environment = os.environ
  69 make = "make"
  70 if sys.platform == "win32":
  71     progexe = "Release/rbutilqt.exe"
  72     make = "mingw32-make"
  73 elif sys.platform == "darwin":
  74     progexe = "rbutilqt.app"
  75     # OS X 10.6 defaults to gcc 4.2. Building universal binaries that are
  76     # compatible with 10.4 requires using gcc-4.0.
  77     if not "QMAKESPEC" in environment:
  78         environment["QMAKESPEC"] = "macx-g++40"
  79 else:
  80     progexe = program
  81
  82 programfiles = [ progexe ]
  83
  84 svnserver = "svn://svn.rockbox.org/rockbox/"
  85 # Paths and files to retrieve from svn when creating a tarball.
  86 # This is a mixed list, holding both paths and filenames.
  87 svnpaths = [ "rbutil/",
  88              "tools/ucl",
  89              "tools/rbspeex",
  90              "apps/codecs/libspeex",
  91              "docs/COPYING",
  92              "tools/iriver.c",
  93              "tools/Makefile",
  94              "tools/mkboot.h",
  95              "tools/voicefont.c",
  96              "tools/VOICE_PAUSE.wav",
  97              "tools/wavtrim.h",
  98              "tools/iriver.h",
  99              "tools/mkboot.c",
 100              "tools/telechips.c",
 101              "tools/telechips.h",
 102              "tools/voicefont.h",
 103              "tools/wavtrim.c",
 104              "tools/sapi_voice.vbs" ]
 105
 106 # == Functions ==
 107 def usage(myself):
 108     print "Usage: %s [options]" % myself
 109     print "       -q, --qmake=<qmake>   path to qmake"
 110     print "       -p, --project=<pro>   path to .pro file for building with local tree"
 111     print "       -t, --tag=<tag>       use specified tag from svn"
 112     print "       -a, --add=<file>      add file to build folder before building"
 113     print "       -s, --source-only     only create source archive"
 114     print "       -b, --binary-only     only create binary archive"
 115     if sys.platform != "darwin":
 116         print "       -d, --dynamic         link dynamically instead of static"
 117     print "       -h, --help            this help"
 118     print "  If neither a project file nor tag is specified trunk will get downloaded"
 119     print "  from svn."
 120
 121 def getsources(svnsrv, filelist, dest):
 122     '''Get the files listed in filelist from svnsrv and put it at dest.'''
 123     client = pysvn.Client()
 124     print "Checking out sources from %s, please wait." % svnsrv
 125
 126     for elem in filelist:
 127         url = re.subn('/$', '', svnsrv + elem)[0]
 128         destpath = re.subn('/$', '', dest + elem)[0]
 129         # make sure the destination path does exist
 130         d = os.path.dirname(destpath)
 131         if not os.path.exists(d):
 132             os.makedirs(d)
 133         # get from svn
 134         try:
 135             client.export(url, destpath)
 136         except:
 137             print "SVN client error: %s" % sys.exc_value
 138             print "URL: %s, destination: %s" % (url, destpath)
 139             return -1
 140     print "Checkout finished."
 141     return 0
 142
 143
 144 def gettrunkrev(svnsrv):
 145     '''Get the revision of trunk for svnsrv'''
 146     client = pysvn.Client()
 147     entries = client.info2(svnsrv, recurse=False)
 148     return entries[0][1].rev.number
 149
 150
 151 def findversion(versionfile):
 152     '''figure most recent program version from version.h,
 153     returns version string.'''
 154     h = open(versionfile, "r")
 155     c = h.read()
 156     h.close()
 157     r = re.compile("#define +VERSION +\"(.[0-9\.a-z]+)\"")
 158     m = re.search(r, c)
 159     s = re.compile("\$Revision: +([0-9]+)")
 160     n = re.search(s, c)
 161     if n == None:
 162         print "WARNING: Revision not found!"
 163     return m.group(1)
 164
 165
 166 def findqt():
 167     '''Search for Qt4 installation. Return path to qmake.'''
 168     print "Searching for Qt"
 169     bins = ["qmake", "qmake-qt4"]
 170     for binary in bins:
 171         try:
 172             q = which.which(binary)
 173             if len(q) > 0:
 174                 result = checkqt(q)
 175                 if not result == "":
 176                     return result
 177         except:
 178             print sys.exc_value
 179
 180     return ""
 181
 182
 183 def checkqt(qmakebin):
 184     '''Check if given path to qmake exists and is a suitable version.'''
 185     result = ""
 186     # check if binary exists
 187     if not os.path.exists(qmakebin):
 188         print "Specified qmake path does not exist!"
 189         return result
 190     # check version
 191     output = subprocess.Popen([qmakebin, "-version"], stdout=subprocess.PIPE,
 192             stderr=subprocess.PIPE)
 193     cmdout = output.communicate()
 194     # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
 195     for ou in cmdout:
 196         r = re.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
 197         m = re.search(r, ou)
 198         if not m == None:
 199             print "Qt found: %s" % m.group(1)
 200             s = re.compile("4\..*")
 201             n = re.search(s, m.group(1))
 202             if not n == None:
 203                 result = qmakebin
 204     return result
 205
 206
 207 def qmake(qmake="qmake", projfile=project, wd=".", static=True):
 208     print "Running qmake in %s..." % wd
 209     command = [qmake, "-config", "release", "-config", "noccache"]
 210     if static == True:
 211         command.append("-config")
 212         command.append("static")
 213     command.append(projfile)
 214     output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd, env=environment)
 215     output.communicate()
 216     if not output.returncode == 0:
 217         print "qmake returned an error!"
 218         return -1
 219     return 0
 220
 221
 222 def build(wd="."):
 223     # make
 224     print "Building ..."
 225     output = subprocess.Popen([make], stdout=subprocess.PIPE, cwd=wd)
 226     while True:
 227         c = output.stdout.readline()
 228         sys.stdout.write(".")
 229         sys.stdout.flush()
 230         if not output.poll() == None:
 231             sys.stdout.write("\n")
 232             sys.stdout.flush()
 233             if not output.returncode == 0:
 234                 print "Build failed!"
 235                 return -1
 236             break
 237     if sys.platform != "darwin":
 238         # strip. OS X handles this via macdeployqt.
 239         print "Stripping binary."
 240         output = subprocess.Popen(["strip", progexe], stdout=subprocess.PIPE, cwd=wd)
 241         output.communicate()
 242         if not output.returncode == 0:
 243             print "Stripping failed!"
 244             return -1
 245     return 0
 246
 247
 248 def upxfile(wd="."):
 249     # run upx on binary
 250     print "UPX'ing binary ..."
 251     output = subprocess.Popen(["upx", progexe], stdout=subprocess.PIPE, cwd=wd)
 252     output.communicate()
 253     if not output.returncode == 0:
 254         print "UPX'ing failed!"
 255         return -1
 256     return 0
 257
 258
 259 def zipball(versionstring, buildfolder):
 260     '''package created binary'''
 261     print "Creating binary zipball."
 262     archivebase = program + "-" + versionstring
 263     outfolder = buildfolder + "/" + archivebase
 264     archivename = archivebase + ".zip"
 265     # create output folder
 266     os.mkdir(outfolder)
 267     # move program files to output folder
 268     for f in programfiles:
 269         shutil.copy(buildfolder + "/" + f, outfolder)
 270     # create zipball from output folder
 271     zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED)
 272     for root, dirs, files in os.walk(outfolder):
 273         for name in files:
 274             physname = os.path.join(root, name)
 275             filename = re.sub("^" + buildfolder, "", physname)
 276             zf.write(physname, filename)
 277         for name in dirs:
 278             physname = os.path.join(root, name)
 279             filename = re.sub("^" + buildfolder, "", physname)
 280             zf.write(physname, filename)
 281     zf.close()
 282     # remove output folder
 283     shutil.rmtree(outfolder)
 284     return archivename
 285
 286
 287 def tarball(versionstring, buildfolder):
 288     '''package created binary'''
 289     print "Creating binary tarball."
 290     archivebase = program + "-" + versionstring
 291     outfolder = buildfolder + "/" + archivebase
 292     archivename = archivebase + ".tar.bz2"
 293     # create output folder
 294     os.mkdir(outfolder)
 295     # move program files to output folder
 296     for f in programfiles:
 297         shutil.copy(buildfolder + "/" + f, outfolder)
 298     # create tarball from output folder
 299     tf = tarfile.open(archivename, mode='w:bz2')
 300     tf.add(outfolder, archivebase)
 301     tf.close()
 302     # remove output folder
 303     shutil.rmtree(outfolder)
 304     return archivename
 305
 306
 307 def macdeploy(versionstring, buildfolder):
 308     '''package created binary to dmg'''
 309     dmgfile = program + "-" + versionstring + ".dmg"
 310     appbundle = buildfolder + "/" + progexe
 311
 312     # workaround to Qt issues when building out-of-tree. Hardcoded for simplicity.
 313     sourcebase = buildfolder + re.sub('rbutilqt.pro$', '', project)
 314     shutil.copy(sourcebase + "icons/rbutilqt.icns", appbundle + "/Contents/Resources/")
 315     shutil.copy(sourcebase + "Info.plist", appbundle + "/Contents/")
 316     # end of Qt workaround
 317
 318     output = subprocess.Popen(["macdeployqt", progexe, "-dmg"], stdout=subprocess.PIPE, cwd=buildfolder)
 319     output.communicate()
 320     if not output.returncode == 0:
 321         print "macdeployqt failed!"
 322         return -1
 323     # copy dmg to output folder
 324     shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile)
 325     return dmgfile
 326
 327 def filehashes(filename):
 328     '''Calculate md5 and sha1 hashes for a given file.'''
 329     if not os.path.exists(filename):
 330         return ["", ""]
 331     m = hashlib.md5()
 332     s = hashlib.sha1()
 333     f = open(filename, 'rb')
 334     while True:
 335         d = f.read(65536)
 336         if d == "":
 337             break
 338         m.update(d)
 339         s.update(d)
 340     return [m.hexdigest(), s.hexdigest()]
 341
 342
 343 def filestats(filename):
 344     if not os.path.exists(filename):
 345         return
 346     st = os.stat(filename)
 347     print filename, "\n", "-" * len(filename)
 348     print "Size:    %i bytes" % st.st_size
 349     h = filehashes(filename)
 350     print "md5sum:  %s" % h[0]
 351     print "sha1sum: %s" % h[1]
 352     print "-" * len(filename), "\n"
 353
 354
 355 def main():
 356     startup = time.time()
 357     try:
 358         opts, args = getopt.getopt(sys.argv[1:], "q:p:t:a:sbdh",
 359             ["qmake=", "project=", "tag=", "add=", "source-only", "binary-only", "dynamic", "help"])
 360     except getopt.GetoptError, err:
 361         print str(err)
 362         usage(sys.argv[0])
 363         sys.exit(1)
 364     qt = ""
 365     proj = ""
 366     svnbase = svnserver + "trunk/"
 367     tag = ""
 368     addfiles = []
 369     cleanup = True
 370     binary = True
 371     source = True
 372     if sys.platform != "darwin":
 373         static = True
 374     else:
 375         static = False
 376     for o, a in opts:
 377         if o in ("-q", "--qmake"):
 378             qt = a
 379         if o in ("-p", "--project"):
 380             proj = a
 381             cleanup = False
 382         if o in ("-t", "--tag"):
 383             tag = a
 384             svnbase = svnserver + "tags/" + tag + "/"
 385         if o in ("-a", "--add"):
 386             addfiles.append(a)
 387         if o in ("-s", "--source-only"):
 388             binary = False
 389         if o in ("-b", "--binary-only"):
 390             source = False
 391         if o in ("-d", "--dynamic") and sys.platform != "darwin":
 392             static = False
 393         if o in ("-h", "--help"):
 394             usage(sys.argv[0])
 395             sys.exit(0)
 396
 397     if source == False and binary == False:
 398         print "Building build neither source nor binary means nothing to do. Exiting."
 399         sys.exit(1)
 400
 401     # search for qmake
 402     if qt == "":
 403         qm = findqt()
 404     else:
 405         qm = checkqt(qt)
 406     if qm == "":
 407         print "ERROR: No suitable Qt installation found."
 408         sys.exit(1)
 409
 410     # create working folder. Use current directory if -p option used.
 411     if proj == "":
 412         w = tempfile.mkdtemp()
 413         # make sure the path doesn't contain backslashes to prevent issues
 414         # later when running on windows.
 415         workfolder = re.sub(r'\\', '/', w)
 416         if not tag == "":
 417             sourcefolder = workfolder + "/" + tag + "/"
 418             archivename = tag + "-src.tar.bz2"
 419             # get numeric version part from tag
 420             ver = "v" + re.sub('^[^\d]+', '', tag)
 421         else:
 422             trunk = gettrunkrev(svnbase)
 423             sourcefolder = workfolder + "/rbutil-r" + str(trunk) + "/"
 424             archivename = "rbutil-r" + str(trunk) + "-src.tar.bz2"
 425             ver = "r" + str(trunk)
 426         os.mkdir(sourcefolder)
 427     else:
 428         workfolder = "."
 429         sourcefolder = "."
 430         archivename = ""
 431     # check if project file explicitly given. If yes, don't get sources from svn
 432     if proj == "":
 433         proj = sourcefolder + project
 434         # get sources and pack source tarball
 435         if not getsources(svnbase, svnpaths, sourcefolder) == 0:
 436             sys.exit(1)
 437
 438         if source == True:
 439             tf = tarfile.open(archivename, mode='w:bz2')
 440             tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
 441             tf.close()
 442             if binary == False:
 443                 shutil.rmtree(workfolder)
 444                 sys.exit(0)
 445     else:
 446         # figure version from sources. Need to take path to project file into account.
 447         versionfile = re.subn('[\w\.]+$', "version.h", proj)[0]
 448         ver = findversion(versionfile)
 449
 450     # check project file
 451     if not os.path.exists(proj):
 452         print "ERROR: path to project file wrong."
 453         sys.exit(1)
 454
 455     # copy specified (--add) files to working folder
 456     for f in addfiles:
 457         shutil.copy(f, sourcefolder)
 458     buildstart = time.time()
 459     header = "Building %s %s" % (program, ver)
 460     print header
 461     print len(header) * "="
 462
 463     # build it.
 464     if not qmake(qm, proj, sourcefolder, static) == 0:
 465         sys.exit(1)
 466     if not build(sourcefolder) == 0:
 467         sys.exit(1)
 468     if sys.platform == "win32":
 469         if not upxfile(sourcefolder) == 0:
 470             sys.exit(1)
 471         archive = zipball(ver, sourcefolder)
 472     elif sys.platform == "darwin":
 473         archive = macdeploy(ver, sourcefolder)
 474     else:
 475         archive = tarball(ver, sourcefolder)
 476
 477     # remove temporary files
 478     print "Cleaning up working folder %s" % workfolder
 479     if cleanup == True:
 480         shutil.rmtree(workfolder)
 481     else:
 482         print "Project file specified, not cleaning up!"
 483
 484     # display summary
 485     headline = "Build Summary for %s" % program
 486     print "\n", headline, "\n", "=" * len(headline)
 487     if not archivename == "":
 488         filestats(archivename)
 489     filestats(archive)
 490     duration = time.time() - startup
 491     building = time.time() - buildstart
 492     durmins = (int)(duration / 60)
 493     dursecs = (int)(duration % 60)
 494     buildmins = (int)(building / 60)
 495     buildsecs = (int)(building % 60)
 496     print "Overall time %smin %ssec, building took %smin %ssec." % \
 497         (durmins, dursecs, buildmins, buildsecs)
 498
 499
 500 if __name__ == "__main__":
 501     main()
 502