rbutil/rbutilqt/deploy-release.py

   1 #!/usr/bin/python
   2 #             __________               __   ___.
   3 #   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4 #   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5 #   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6 #   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7 #                     \/            \/     \/    \/            \/
   8 # $Id$
   9 #
  10 # Copyright (c) 2009 Dominik Riebeling
  11 #
  12 # All files in this archive are subject to the GNU General Public License.
  13 # See the file COPYING in the source tree root for full license agreement.
  14 #
  15 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  16 # KIND, either express or implied.
  17 #
  18 #
  19 # Automate building releases for deployment.
  20 # Run from any folder to build
  21 # - trunk
  22 # - any tag (using the -t option)
  23 # - any local folder (using the -p option)
  24 # Will build a binary archive (tar.bz2 / zip) and source archive.
  25 # The source archive won't be built for local builds. Trunk and
  26 # tag builds will retrieve the sources directly from svn and build
  27 # below the systems temporary folder.
  28 #
  29 # If the required Qt installation isn't in PATH use --qmake option.
  30 # Tested on Linux and MinGW / W32
  31 #
  32 # requires python which package (http://code.google.com/p/which/)
  33 # requires pysvn package.
  34 # requires upx.exe in PATH on Windows.
  35 #
  36
  37 import re
  38 import os
  39 import sys
  40 import tarfile
  41 import zipfile
  42 import shutil
  43 import subprocess
  44 import getopt
  45 import time
  46 import hashlib
  47 import tempfile
  48
  49 # modules that are not part of python itself.
  50 try:
  51     import pysvn
  52 except ImportError:
  53     print "Fatal: This script requires the pysvn package to run."
  54     print "       See http://pysvn.tigris.org/."
  55     sys.exit(-5)
  56 try:
  57     import which
  58 except ImportError:
  59     print "Fatal: This script requires the which package to run."
  60     print "       See http://code.google.com/p/which/."
  61     sys.exit(-5)
  62
  63 # == Global stuff ==
  64 # Windows nees some special treatment. Differentiate between program name
  65 # and executable filename.
  66 program = "rbutilqt"
  67 project = "rbutil/rbutilqt/rbutilqt.pro"
  68 if sys.platform == "win32":
  69     progexe = "Release/rbutilqt.exe"
  70     make = "mingw32-make"
  71 else:
  72     progexe = program
  73     make = "make"
  74
  75 programfiles = [ progexe ]
  76
  77 svnserver = "svn://svn.rockbox.org/rockbox/"
  78 # Paths and files to retrieve from svn when creating a tarball.
  79 # This is a mixed list, holding both paths and filenames.
  80 svnpaths = [ "rbutil/",
  81              "tools/ucl",
  82              "tools/rbspeex",
  83              "apps/codecs/libspeex",
  84              "tools/iriver.c",
  85              "tools/Makefile",
  86              "tools/mkboot.h",
  87              "tools/voicefont.c",
  88              "tools/VOICE_PAUSE.wav",
  89              "tools/wavtrim.h",
  90              "tools/iriver.h",
  91              "tools/mkboot.c",
  92              "tools/telechips.c",
  93              "tools/telechips.h",
  94              "tools/voicefont.h",
  95              "tools/wavtrim.c",
  96              "tools/sapi_voice.vbs" ]
  97
  98 # == Functions ==
  99 def usage(myself):
 100     print "Usage: %s [options]" % myself
 101     print "       -q, --qmake=<qmake>   path to qmake"
 102     print "       -p, --project=<pro>   path to .pro file for building with local tree"
 103     print "       -t, --tag=<tag>       use specified tag from svn"
 104     print "       -a, --add=<file>      add file to build folder before building"
 105     print "       -s, --source-only     only create source archive"
 106     print "       -b, --binary-only     only create binary archive"
 107     print "       -d, --dynamic         link dynamically instead of static"
 108     print "       -h, --help            this help"
 109     print "  If neither a project file nor tag is specified trunk will get downloaded"
 110     print "  from svn."
 111
 112 def getsources(svnsrv, filelist, dest):
 113     '''Get the files listed in filelist from svnsrv and put it at dest.'''
 114     client = pysvn.Client()
 115     print "Checking out sources from %s, please wait." % svnsrv
 116
 117     for elem in filelist:
 118         url = re.subn('/$', '', svnsrv + elem)[0]
 119         destpath = re.subn('/$', '', dest + elem)[0]
 120         try:
 121             client.export(url, destpath)
 122         except:
 123             print "SVN client error: %s" % sys.exc_value
 124             return -1
 125     print "Checkout finished."
 126     return 0
 127
 128
 129 def gettrunkrev(svnsrv):
 130     '''Get the revision of trunk for svnsrv'''
 131     client = pysvn.Client()
 132     entries = client.info2(svnsrv, recurse=False)
 133     return entries[0][1].rev.number
 134
 135
 136 def findversion(versionfile):
 137     '''figure most recent program version from version.h,
 138     returns version string.'''
 139     h = open(versionfile, "r")
 140     c = h.read()
 141     h.close()
 142     r = re.compile("#define +VERSION +\"(.[0-9\.a-z]+)\"")
 143     m = re.search(r, c)
 144     s = re.compile("\$Revision: +([0-9]+)")
 145     n = re.search(s, c)
 146     if n == None:
 147         print "WARNING: Revision not found!"
 148     return m.group(1)
 149
 150
 151 def findqt():
 152     '''Search for Qt4 installation. Return path to qmake.'''
 153     print "Searching for Qt"
 154     bins = ["qmake", "qmake-qt4"]
 155     for binary in bins:
 156         try:
 157             q = which.which(binary)
 158             if len(q) > 0:
 159                 result = checkqt(q)
 160                 if not result == "":
 161                     return result
 162         except:
 163             print sys.exc_value
 164
 165     return ""
 166
 167
 168 def checkqt(qmakebin):
 169     '''Check if given path to qmake exists and is a suitable version.'''
 170     result = ""
 171     # check if binary exists
 172     if not os.path.exists(qmakebin):
 173         print "Specified qmake path does not exist!"
 174         return result
 175     # check version
 176     output = subprocess.Popen([qmakebin, "-version"], stdout=subprocess.PIPE,
 177             stderr=subprocess.PIPE)
 178     cmdout = output.communicate()
 179     # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
 180     for ou in cmdout:
 181         r = re.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
 182         m = re.search(r, ou)
 183         if not m == None:
 184             print "Qt found: %s" % m.group(1)
 185             s = re.compile("4\..*")
 186             n = re.search(s, m.group(1))
 187             if not n == None:
 188                 result = qmakebin
 189     return result
 190
 191
 192 def qmake(qmake="qmake", projfile=project, wd=".", static=True):
 193     print "Running qmake in %s..." % wd
 194     command = [qmake, "-config", "release", "-config", "noccache"]
 195     if static == True:
 196         command.append("-config")
 197         command.append("static")
 198     command.append(projfile)
 199     output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd)
 200     output.communicate()
 201     if not output.returncode == 0:
 202         print "qmake returned an error!"
 203         return -1
 204     return 0
 205
 206
 207 def build(wd="."):
 208     # make
 209     print "Building ..."
 210     output = subprocess.Popen([make], stdout=subprocess.PIPE, cwd=wd)
 211     while True:
 212         c = output.stdout.readline()
 213         sys.stdout.write(".")
 214         sys.stdout.flush()
 215         if not output.poll() == None:
 216             sys.stdout.write("\n")
 217             sys.stdout.flush()
 218             if not output.returncode == 0:
 219                 print "Build failed!"
 220                 return -1
 221             break
 222     # strip
 223     print "Stripping binary."
 224     output = subprocess.Popen(["strip", progexe], stdout=subprocess.PIPE, cwd=wd)
 225     output.communicate()
 226     if not output.returncode == 0:
 227         print "Stripping failed!"
 228         return -1
 229     return 0
 230
 231
 232 def upxfile(wd="."):
 233     # run upx on binary
 234     print "UPX'ing binary ..."
 235     output = subprocess.Popen(["upx", progexe], stdout=subprocess.PIPE, cwd=wd)
 236     output.communicate()
 237     if not output.returncode == 0:
 238         print "UPX'ing failed!"
 239         return -1
 240     return 0
 241
 242
 243 def zipball(versionstring, buildfolder):
 244     '''package created binary'''
 245     print "Creating binary zipball."
 246     archivebase = program + "-" + versionstring
 247     outfolder = buildfolder + "/" + archivebase
 248     archivename = archivebase + ".zip"
 249     # create output folder
 250     os.mkdir(outfolder)
 251     # move program files to output folder
 252     for f in programfiles:
 253         shutil.copy(buildfolder + "/" + f, outfolder)
 254     # create zipball from output folder
 255     zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED)
 256     for root, dirs, files in os.walk(outfolder):
 257         for name in files:
 258             physname = os.path.join(root, name)
 259             filename = re.sub("^" + buildfolder, "", physname)
 260             zf.write(physname, filename)
 261         for name in dirs:
 262             physname = os.path.join(root, name)
 263             filename = re.sub("^" + buildfolder, "", physname)
 264             zf.write(physname, filename)
 265     zf.close()
 266     # remove output folder
 267     shutil.rmtree(outfolder)
 268     return archivename
 269
 270
 271 def tarball(versionstring, buildfolder):
 272     '''package created binary'''
 273     print "Creating binary tarball."
 274     archivebase = program + "-" + versionstring
 275     outfolder = buildfolder + "/" + archivebase
 276     archivename = archivebase + ".tar.bz2"
 277     # create output folder
 278     os.mkdir(outfolder)
 279     # move program files to output folder
 280     for f in programfiles:
 281         shutil.copy(buildfolder + "/" + f, outfolder)
 282     # create tarball from output folder
 283     tf = tarfile.open(archivename, mode='w:bz2')
 284     tf.add(outfolder, archivebase)
 285     tf.close()
 286     # remove output folder
 287     shutil.rmtree(outfolder)
 288     return archivename
 289
 290
 291 def filehashes(filename):
 292     '''Calculate md5 and sha1 hashes for a given file.'''
 293     if not os.path.exists(filename):
 294         return ["", ""]
 295     m = hashlib.md5()
 296     s = hashlib.sha1()
 297     f = open(filename, 'rb')
 298     while True:
 299         d = f.read(65536)
 300         if d == "":
 301             break
 302         m.update(d)
 303         s.update(d)
 304     return [m.hexdigest(), s.hexdigest()]
 305
 306
 307 def filestats(filename):
 308     if not os.path.exists(filename):
 309         return
 310     st = os.stat(filename)
 311     print filename, "\n", "-" * len(filename)
 312     print "Size:    %i bytes" % st.st_size
 313     h = filehashes(filename)
 314     print "md5sum:  %s" % h[0]
 315     print "sha1sum: %s" % h[1]
 316     print "-" * len(filename), "\n"
 317
 318
 319 def main():
 320     startup = time.time()
 321     try:
 322         opts, args = getopt.getopt(sys.argv[1:], "q:p:t:a:sbdh",
 323             ["qmake=", "project=", "tag=", "add=", "source-only", "binary-only", "dynamic", "help"])
 324     except getopt.GetoptError, err:
 325         print str(err)
 326         usage(sys.argv[0])
 327         sys.exit(1)
 328     qt = ""
 329     proj = ""
 330     svnbase = svnserver + "trunk/"
 331     tag = ""
 332     addfiles = []
 333     cleanup = True
 334     binary = True
 335     source = True
 336     static = True
 337     for o, a in opts:
 338         if o in ("-q", "--qmake"):
 339             qt = a
 340         if o in ("-p", "--project"):
 341             proj = a
 342             cleanup = False
 343         if o in ("-t", "--tag"):
 344             tag = a
 345             svnbase = svnserver + "tags/" + tag + "/"
 346         if o in ("-a", "--add"):
 347             addfiles.append(a)
 348         if o in ("-s", "--source-only"):
 349             binary = False
 350         if o in ("-b", "--binary-only"):
 351             source = False
 352         if o in ("-d", "--dynamic"):
 353             static = False
 354         if o in ("-h", "--help"):
 355             usage(sys.argv[0])
 356             sys.exit(0)
 357
 358     if source == False and binary == False:
 359         print "Building build neither source nor binary means nothing to do. Exiting."
 360         sys.exit(1)
 361
 362     # search for qmake
 363     if qt == "":
 364         qm = findqt()
 365     else:
 366         qm = checkqt(qt)
 367     if qm == "":
 368         print "ERROR: No suitable Qt installation found."
 369         sys.exit(1)
 370
 371     # create working folder. Use current directory if -p option used.
 372     if proj == "":
 373         w = tempfile.mkdtemp()
 374         # make sure the path doesn't contain backslashes to prevent issues
 375         # later when running on windows.
 376         workfolder = re.sub(r'\\', '/', w)
 377         if not tag == "":
 378             sourcefolder = workfolder + "/" + tag + "/"
 379             archivename = tag + "-src.tar.bz2"
 380             # get numeric version part from tag
 381             ver = "v" + re.sub('^[^\d]+', '', tag)
 382         else:
 383             trunk = gettrunkrev(svnbase)
 384             sourcefolder = workfolder + "/rbutil-r" + str(trunk) + "/"
 385             archivename = "rbutil-r" + str(trunk) + "-src.tar.bz2"
 386             ver = "r" + str(trunk)
 387         os.mkdir(sourcefolder)
 388     else:
 389         workfolder = "."
 390         sourcefolder = "."
 391         archivename = ""
 392     # check if project file explicitly given. If yes, don't get sources from svn
 393     if proj == "":
 394         proj = sourcefolder + project
 395         # get sources and pack source tarball
 396         if not getsources(svnbase, svnpaths, sourcefolder) == 0:
 397             sys.exit(1)
 398
 399         if source == True:
 400             tf = tarfile.open(archivename, mode='w:bz2')
 401             tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
 402             tf.close()
 403             if binary == False:
 404                 shutil.rmtree(workfolder)
 405                 sys.exit(0)
 406     else:
 407         # figure version from sources. Need to take path to project file into account.
 408         versionfile = re.subn('[\w\.]+$', "version.h", proj)[0]
 409         ver = findversion(versionfile)
 410
 411     # check project file
 412     if not os.path.exists(proj):
 413         print "ERROR: path to project file wrong."
 414         sys.exit(1)
 415
 416     # copy specified (--add) files to working folder
 417     for f in addfiles:
 418         shutil.copy(f, sourcefolder)
 419     buildstart = time.time()
 420     header = "Building %s %s" % (program, ver)
 421     print header
 422     print len(header) * "="
 423
 424     # build it.
 425     if not qmake(qm, proj, sourcefolder, static) == 0:
 426         sys.exit(1)
 427     if not build(sourcefolder) == 0:
 428         sys.exit(1)
 429     if sys.platform == "win32":
 430         if not upxfile(sourcefolder) == 0:
 431             sys.exit(1)
 432         archive = zipball(ver, sourcefolder)
 433     else:
 434         archive = tarball(ver, sourcefolder)
 435
 436     # remove temporary files
 437     print "Cleaning up working folder %s" % workfolder
 438     if cleanup == True:
 439         shutil.rmtree(workfolder)
 440     else:
 441         print "Project file specified, not cleaning up!"
 442
 443     # display summary
 444     headline = "Build Summary for %s" % program
 445     print "\n", headline, "\n", "=" * len(headline)
 446     if not archivename == "":
 447         filestats(archivename)
 448     filestats(archive)
 449     duration = time.time() - startup
 450     building = time.time() - buildstart
 451     durmins = (int)(duration / 60)
 452     dursecs = (int)(duration % 60)
 453     buildmins = (int)(building / 60)
 454     buildsecs = (int)(building % 60)
 455     print "Overall time %smin %ssec, building took %smin %ssec." % \
 456         (durmins, dursecs, buildmins, buildsecs)
 457
 458
 459 if __name__ == "__main__":
 460     main()
 461