We have a 3.9 release, update builds.pm
[maemo-rb.git] / utils / common / deploy.py
blob74dfab420cb8e6f408973c2d12fcac4d4eed2dba
1 #!/usr/bin/python
2 # __________ __ ___.
3 # Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 # Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 # Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 # Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 # \/ \/ \/ \/ \/
8 # $Id$
10 # Copyright (c) 2009 Dominik Riebeling
12 # All files in this archive are subject to the GNU General Public License.
13 # See the file COPYING in the source tree root for full license agreement.
15 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 # KIND, either express or implied.
19 # Automate building releases for deployment.
20 # Run from any folder to build
21 # - trunk
22 # - any tag (using the -t option)
23 # - any local folder (using the -p option)
24 # Will build a binary archive (tar.bz2 / zip) and source archive.
25 # The source archive won't be built for local builds. Trunk and
26 # tag builds will retrieve the sources directly from svn and build
27 # below the systems temporary folder.
29 # If the required Qt installation isn't in PATH use --qmake option.
30 # Tested on Linux and MinGW / W32
32 # requires python which package (http://code.google.com/p/which/)
33 # requires pysvn package.
34 # requires upx.exe in PATH on Windows.
37 import re
38 import os
39 import sys
40 import tarfile
41 import zipfile
42 import shutil
43 import subprocess
44 import getopt
45 import time
46 import hashlib
47 import tempfile
48 import string
50 # modules that are not part of python itself.
51 try:
52 import pysvn
53 except ImportError:
54 print "Fatal: This script requires the pysvn package to run."
55 print " See http://pysvn.tigris.org/."
56 sys.exit(-5)
57 try:
58 import which
59 except ImportError:
60 print "Fatal: This script requires the which package to run."
61 print " See http://code.google.com/p/which/."
62 sys.exit(-5)
63 cpus = 1
64 try:
65 import multiprocessing
66 cpus = multiprocessing.cpu_count()
67 print "Info: %s cores found." % cpus
68 except ImportError:
69 print "Warning: multiprocessing module not found. Assuming 1 core."
71 # == Global stuff ==
72 # DLL files to ignore when searching for required DLL files.
73 systemdlls = ['advapi32.dll',
74 'comdlg32.dll',
75 'gdi32.dll',
76 'imm32.dll',
77 'kernel32.dll',
78 'msvcrt.dll',
79 'msvcrt.dll',
80 'netapi32.dll',
81 'ole32.dll',
82 'oleaut32.dll',
83 'setupapi.dll',
84 'shell32.dll',
85 'user32.dll',
86 'winmm.dll',
87 'winspool.drv',
88 'ws2_32.dll']
91 # == Functions ==
92 def usage(myself):
93 print "Usage: %s [options]" % myself
94 print " -q, --qmake=<qmake> path to qmake"
95 print " -p, --project=<pro> path to .pro file for building with local tree"
96 print " -t, --tag=<tag> use specified tag from svn"
97 print " -a, --add=<file> add file to build folder before building"
98 print " -s, --source-only only create source archive"
99 print " -b, --binary-only only create binary archive"
100 if nsisscript != "":
101 print " -n, --makensis=<file> path to makensis for building Windows setup program."
102 if sys.platform != "darwin":
103 print " -d, --dynamic link dynamically instead of static"
104 if sys.platform != "win32":
105 print " -x, --cross= prefix to cross compile for win32"
106 print " -k, --keep-temp keep temporary folder on build failure"
107 print " -h, --help this help"
108 print " If neither a project file nor tag is specified trunk will get downloaded"
109 print " from svn."
112 def getsources(svnsrv, filelist, dest):
113 '''Get the files listed in filelist from svnsrv and put it at dest.'''
114 client = pysvn.Client()
115 print "Checking out sources from %s, please wait." % svnsrv
117 for elem in filelist:
118 url = re.subn('/$', '', svnsrv + elem)[0]
119 destpath = re.subn('/$', '', dest + elem)[0]
120 # make sure the destination path does exist
121 d = os.path.dirname(destpath)
122 if not os.path.exists(d):
123 os.makedirs(d)
124 # get from svn
125 try:
126 client.export(url, destpath)
127 except:
128 print "SVN client error: %s" % sys.exc_value
129 print "URL: %s, destination: %s" % (url, destpath)
130 return -1
131 print "Checkout finished."
132 return 0
135 def getfolderrev(svnsrv):
136 '''Get the most recent revision for svnsrv'''
137 client = pysvn.Client()
138 entries = client.info2(svnsrv, recurse=False)
139 return entries[0][1].rev.number
142 def findversion(versionfile):
143 '''figure most recent program version from version.h,
144 returns version string.'''
145 h = open(versionfile, "r")
146 c = h.read()
147 h.close()
148 r = re.compile("#define +VERSION +\"(.[0-9\.a-z]+)\"")
149 m = re.search(r, c)
150 s = re.compile("\$Revision: +([0-9]+)")
151 n = re.search(s, c)
152 if n == None:
153 print "WARNING: Revision not found!"
154 return m.group(1)
157 def findqt(cross=""):
158 '''Search for Qt4 installation. Return path to qmake.'''
159 print "Searching for Qt"
160 bins = [cross + "qmake", cross + "qmake-qt4"]
161 for binary in bins:
162 try:
163 q = which.which(binary)
164 if len(q) > 0:
165 result = checkqt(q)
166 if not result == "":
167 return result
168 except:
169 print sys.exc_value
171 return ""
174 def checkqt(qmakebin):
175 '''Check if given path to qmake exists and is a suitable version.'''
176 result = ""
177 # check if binary exists
178 if not os.path.exists(qmakebin):
179 print "Specified qmake path does not exist!"
180 return result
181 # check version
182 output = subprocess.Popen([qmakebin, "-version"], stdout=subprocess.PIPE,
183 stderr=subprocess.PIPE)
184 cmdout = output.communicate()
185 # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
186 for ou in cmdout:
187 r = re.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
188 m = re.search(r, ou)
189 if not m == None:
190 print "Qt found: %s" % m.group(1)
191 s = re.compile("4\..*")
192 n = re.search(s, m.group(1))
193 if not n == None:
194 result = qmakebin
195 return result
198 def qmake(qmake, projfile, platform=sys.platform, wd=".", static=True, cross=""):
199 print "Running qmake in %s..." % wd
200 command = [qmake, "-config", "release", "-config", "noccache"]
201 if static == True:
202 command.extend(["-config", "-static"])
203 # special spec required?
204 if len(qmakespec[platform]) > 0:
205 command.extend(["-spec", qmakespec[platform]])
206 # cross compiling prefix set?
207 if len(cross) > 0:
208 command.extend(["-config", "cross"])
209 command.append(projfile)
210 output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd)
211 output.communicate()
212 if not output.returncode == 0:
213 print "qmake returned an error!"
214 return -1
215 return 0
218 def build(wd=".", platform=sys.platform, cross=""):
219 # make
220 print "Building ..."
221 # use the current platforms make here, cross compiling uses the native make.
222 command = [make[sys.platform]]
223 if cpus > 1:
224 command.append("-j")
225 command.append(str(cpus))
226 output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd)
227 while True:
228 c = output.stdout.readline()
229 sys.stdout.write(".")
230 sys.stdout.flush()
231 if not output.poll() == None:
232 sys.stdout.write("\n")
233 sys.stdout.flush()
234 if not output.returncode == 0:
235 print "Build failed!"
236 return -1
237 break
238 if platform != "darwin":
239 # strip. OS X handles this via macdeployqt.
240 print "Stripping binary."
241 output = subprocess.Popen([cross + "strip", progexe[platform]], \
242 stdout=subprocess.PIPE, cwd=wd)
243 output.communicate()
244 if not output.returncode == 0:
245 print "Stripping failed!"
246 return -1
247 return 0
250 def upxfile(wd=".", platform=sys.platform):
251 # run upx on binary
252 print "UPX'ing binary ..."
253 output = subprocess.Popen(["upx", progexe[platform]], \
254 stdout=subprocess.PIPE, cwd=wd)
255 output.communicate()
256 if not output.returncode == 0:
257 print "UPX'ing failed!"
258 return -1
259 return 0
262 def runnsis(versionstring, nsis, script, srcfolder):
263 # run script through nsis to create installer.
264 print "Running NSIS ..."
266 # Assume the generated installer gets placed in the same folder the nsi
267 # script lives in. This seems to be a valid assumption unless the nsi
268 # script specifies a path. NSIS expects files relative to source folder so
269 # copy progexe. Additional files are injected into the nsis script.
271 # FIXME: instead of copying binaries around copy the NSI file and inject
272 # the correct paths.
273 # Only win32 supported as target platform so hard coded.
274 b = srcfolder + "/" + os.path.dirname(script) + "/" \
275 + os.path.dirname(progexe["win32"])
276 if not os.path.exists(b):
277 os.mkdir(b)
278 shutil.copy(srcfolder + "/" + progexe["win32"], b)
279 output = subprocess.Popen([nsis, srcfolder + "/" + script], \
280 stdout=subprocess.PIPE)
281 output.communicate()
282 if not output.returncode == 0:
283 print "NSIS failed!"
284 return -1
285 setupfile = program + "-" + versionstring + "-setup.exe"
286 # find output filename in nsis script file
287 nsissetup = ""
288 for line in open(srcfolder + "/" + script):
289 if re.match(r'^[^;]*OutFile\s+', line) != None:
290 nsissetup = re.sub(r'^[^;]*OutFile\s+"(.+)"', r'\1', line).rstrip()
291 if nsissetup == "":
292 print "Could not retrieve output file name!"
293 return -1
294 shutil.copy(srcfolder + "/" + os.path.dirname(script) + "/" + nsissetup, \
295 setupfile)
296 return 0
299 def nsisfileinject(nsis, outscript, filelist):
300 '''Inject files in filelist into NSIS script file after the File line
301 containing the main binary. This assumes that the main binary is present
302 in the NSIS script and that all additiona files (dlls etc) to get placed
303 into $INSTDIR.'''
304 output = open(outscript, "w")
305 for line in open(nsis, "r"):
306 output.write(line)
307 # inject files after the progexe binary.
308 # Match the basename only to avoid path mismatches.
309 if re.match(r'^\s*File\s*.*' + os.path.basename(progexe["win32"]), \
310 line, re.IGNORECASE):
311 for f in filelist:
312 injection = " File /oname=$INSTDIR\\" + os.path.basename(f) \
313 + " " + os.path.normcase(f) + "\n"
314 output.write(injection)
315 output.write(" ; end of injected files\n")
316 output.close()
319 def finddlls(program, extrapaths=[], cross=""):
320 '''Check program for required DLLs. Find all required DLLs except ignored
321 ones and return a list of DLL filenames (including path).'''
322 # ask objdump about dependencies.
323 output = subprocess.Popen([cross + "objdump", "-x", program], \
324 stdout=subprocess.PIPE)
325 cmdout = output.communicate()
327 # create list of used DLLs. Store as lower case as W32 is case-insensitive.
328 dlls = []
329 for line in cmdout[0].split('\n'):
330 if re.match(r'\s*DLL Name', line) != None:
331 dll = re.sub(r'^\s*DLL Name:\s+([a-zA-Z_\-0-9\.\+]+).*$', r'\1', line)
332 dlls.append(dll.lower())
334 # find DLLs in extrapaths and PATH environment variable.
335 dllpaths = []
336 for file in dlls:
337 if file in systemdlls:
338 print "System DLL: " + file
339 continue
340 dllpath = ""
341 for path in extrapaths:
342 if os.path.exists(path + "/" + file):
343 dllpath = re.sub(r"\\", r"/", path + "/" + file)
344 print file + ": found at " + dllpath
345 dllpaths.append(dllpath)
346 break
347 if dllpath == "":
348 try:
349 dllpath = re.sub(r"\\", r"/", which.which(file))
350 print file + ": found at " + dllpath
351 dllpaths.append(dllpath)
352 except:
353 print "MISSING DLL: " + file
354 return dllpaths
357 def zipball(programfiles, versionstring, buildfolder, platform=sys.platform):
358 '''package created binary'''
359 print "Creating binary zipball."
360 archivebase = program + "-" + versionstring
361 outfolder = buildfolder + "/" + archivebase
362 archivename = archivebase + ".zip"
363 # create output folder
364 os.mkdir(outfolder)
365 # move program files to output folder
366 for f in programfiles:
367 if re.match(r'^(/|[a-zA-Z]:)', f) != None:
368 shutil.copy(f, outfolder)
369 else:
370 shutil.copy(buildfolder + "/" + f, outfolder)
371 # create zipball from output folder
372 zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED)
373 for root, dirs, files in os.walk(outfolder):
374 for name in files:
375 physname = os.path.normpath(os.path.join(root, name))
376 filename = string.replace(physname, os.path.normpath(buildfolder), "")
377 zf.write(physname, filename)
378 for name in dirs:
379 physname = os.path.normpath(os.path.join(root, name))
380 filename = string.replace(physname, os.path.normpath(buildfolder), "")
381 zf.write(physname, filename)
382 zf.close()
383 # remove output folder
384 shutil.rmtree(outfolder)
385 return archivename
388 def tarball(programfiles, versionstring, buildfolder):
389 '''package created binary'''
390 print "Creating binary tarball."
391 archivebase = program + "-" + versionstring
392 outfolder = buildfolder + "/" + archivebase
393 archivename = archivebase + ".tar.bz2"
394 # create output folder
395 os.mkdir(outfolder)
396 # move program files to output folder
397 for f in programfiles:
398 shutil.copy(buildfolder + "/" + f, outfolder)
399 # create tarball from output folder
400 tf = tarfile.open(archivename, mode='w:bz2')
401 tf.add(outfolder, archivebase)
402 tf.close()
403 # remove output folder
404 shutil.rmtree(outfolder)
405 return archivename
408 def macdeploy(versionstring, buildfolder, platform=sys.platform):
409 '''package created binary to dmg'''
410 dmgfile = program + "-" + versionstring + ".dmg"
411 appbundle = buildfolder + "/" + progexe[platform]
413 # workaround to Qt issues when building out-of-tree. Copy files into bundle.
414 sourcebase = buildfolder + re.sub('[^/]+.pro$', '', project) + "/"
415 print sourcebase
416 for src in bundlecopy:
417 shutil.copy(sourcebase + src, appbundle + "/" + bundlecopy[src])
418 # end of Qt workaround
420 output = subprocess.Popen(["macdeployqt", progexe[platform], "-dmg"], \
421 stdout=subprocess.PIPE, cwd=buildfolder)
422 output.communicate()
423 if not output.returncode == 0:
424 print "macdeployqt failed!"
425 return -1
426 # copy dmg to output folder
427 shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile)
428 return dmgfile
431 def filehashes(filename):
432 '''Calculate md5 and sha1 hashes for a given file.'''
433 if not os.path.exists(filename):
434 return ["", ""]
435 m = hashlib.md5()
436 s = hashlib.sha1()
437 f = open(filename, 'rb')
438 while True:
439 d = f.read(65536)
440 if d == "":
441 break
442 m.update(d)
443 s.update(d)
444 return [m.hexdigest(), s.hexdigest()]
447 def filestats(filename):
448 if not os.path.exists(filename):
449 return
450 st = os.stat(filename)
451 print filename, "\n", "-" * len(filename)
452 print "Size: %i bytes" % st.st_size
453 h = filehashes(filename)
454 print "md5sum: %s" % h[0]
455 print "sha1sum: %s" % h[1]
456 print "-" * len(filename), "\n"
459 def tempclean(workfolder, nopro):
460 if nopro == True:
461 print "Cleaning up working folder %s" % workfolder
462 shutil.rmtree(workfolder)
463 else:
464 print "Project file specified or cleanup disabled!"
465 print "Temporary files kept at %s" % workfolder
468 def deploy():
469 startup = time.time()
471 try:
472 opts, args = getopt.getopt(sys.argv[1:], "q:p:t:a:n:sbdkx:i:h",
473 ["qmake=", "project=", "tag=", "add=", "makensis=", "source-only",
474 "binary-only", "dynamic", "keep-temp", "cross=", "buildid=", "help"])
475 except getopt.GetoptError, err:
476 print str(err)
477 usage(sys.argv[0])
478 sys.exit(1)
479 qt = ""
480 proj = ""
481 svnbase = svnserver + "trunk/"
482 tag = ""
483 addfiles = []
484 cleanup = True
485 binary = True
486 source = True
487 keeptemp = False
488 makensis = ""
489 cross = ""
490 buildid = None
491 platform = sys.platform
492 if sys.platform != "darwin":
493 static = True
494 else:
495 static = False
496 for o, a in opts:
497 if o in ("-q", "--qmake"):
498 qt = a
499 if o in ("-p", "--project"):
500 proj = a
501 cleanup = False
502 if o in ("-t", "--tag"):
503 tag = a
504 svnbase = svnserver + "tags/" + tag + "/"
505 if o in ("-a", "--add"):
506 addfiles.append(a)
507 if o in ("-n", "--makensis"):
508 makensis = a
509 if o in ("-s", "--source-only"):
510 binary = False
511 if o in ("-b", "--binary-only"):
512 source = False
513 if o in ("-d", "--dynamic") and sys.platform != "darwin":
514 static = False
515 if o in ("-k", "--keep-temp"):
516 keeptemp = True
517 if o in ("-x", "--cross") and sys.platform != "win32":
518 cross = a
519 platform = "win32"
520 if o in ("-i", "--buildid"):
521 buildid = a
522 if o in ("-h", "--help"):
523 usage(sys.argv[0])
524 sys.exit(0)
526 if source == False and binary == False:
527 print "Building build neither source nor binary means nothing to do. Exiting."
528 sys.exit(1)
530 print "Building " + progexe[platform] + " for " + platform
531 # search for qmake
532 if qt == "":
533 qm = findqt(cross)
534 else:
535 qm = checkqt(qt)
536 if qm == "":
537 print "ERROR: No suitable Qt installation found."
538 sys.exit(1)
540 # create working folder. Use current directory if -p option used.
541 if proj == "":
542 w = tempfile.mkdtemp()
543 # make sure the path doesn't contain backslashes to prevent issues
544 # later when running on windows.
545 workfolder = re.sub(r'\\', '/', w)
546 revision = getfolderrev(svnbase)
547 if buildid == None:
548 versionextra = ""
549 else:
550 versionextra = "-" + buildid
551 if tag != "":
552 sourcefolder = workfolder + "/" + tag + "/"
553 archivename = tag + versionextra + "-src.tar.bz2"
554 # get numeric version part from tag
555 ver = "v" + re.sub('^[^\d]+', '', tag)
556 else:
557 sourcefolder = workfolder + "/" + program + "-r" + str(revision) + versionextra + "/"
558 archivename = program + "-r" + str(revision) + versionextra + "-src.tar.bz2"
559 ver = "r" + str(revision)
560 os.mkdir(sourcefolder)
561 else:
562 workfolder = "."
563 sourcefolder = "."
564 archivename = ""
565 # check if project file explicitly given. If yes, don't get sources from svn
566 if proj == "":
567 proj = sourcefolder + project
568 # get sources and pack source tarball
569 if not getsources(svnbase, svnpaths, sourcefolder) == 0:
570 tempclean(workfolder, cleanup and not keeptemp)
571 sys.exit(1)
573 # replace version strings.
574 print "Updating version information in sources"
575 for f in regreplace:
576 infile = open(sourcefolder + "/" + f, "r")
577 incontents = infile.readlines()
578 infile.close()
580 outfile = open(sourcefolder + "/" + f, "w")
581 for line in incontents:
582 newline = line
583 for r in regreplace[f]:
584 # replacements made on the replacement string:
585 # %REVISION% is replaced with the revision number
586 replacement = re.sub("%REVISION%", str(revision), r[1])
587 # %BUILD% is replace with buildid as passed on the command line
588 if buildid != None:
589 replacement = re.sub("%BUILDID%", str(buildid), replacement)
590 newline = re.sub(r[0], replacement, newline)
591 outfile.write(newline)
592 outfile.close()
594 if source == True:
595 print "Creating source tarball %s\n" % archivename
596 tf = tarfile.open(archivename, mode='w:bz2')
597 tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
598 tf.close()
599 if binary == False:
600 shutil.rmtree(workfolder)
601 sys.exit(0)
602 else:
603 # figure version from sources. Need to take path to project file into account.
604 versionfile = re.subn('[\w\.]+$', "version.h", proj)[0]
605 ver = findversion(versionfile)
606 # append buildid if any.
607 if buildid != None:
608 ver += "-" + buildid
610 # check project file
611 if not os.path.exists(proj):
612 print "ERROR: path to project file wrong."
613 sys.exit(1)
615 # copy specified (--add) files to working folder
616 for f in addfiles:
617 shutil.copy(f, sourcefolder)
618 buildstart = time.time()
619 header = "Building %s %s" % (program, ver)
620 print header
621 print len(header) * "="
623 # build it.
624 if not qmake(qm, proj, platform, sourcefolder, static, cross) == 0:
625 tempclean(workfolder, cleanup and not keeptemp)
626 sys.exit(1)
627 if not build(sourcefolder, platform, cross) == 0:
628 tempclean(workfolder, cleanup and not keeptemp)
629 sys.exit(1)
630 buildtime = time.time() - buildstart
631 progfiles = programfiles
632 progfiles.append(progexe[platform])
633 if platform == "win32":
634 if useupx == True:
635 if not upxfile(sourcefolder, platform) == 0:
636 tempclean(workfolder, cleanup and not keeptemp)
637 sys.exit(1)
638 dllfiles = finddlls(sourcefolder + "/" + progexe[platform], \
639 [os.path.dirname(qm)], cross)
640 if dllfiles.count > 0:
641 progfiles.extend(dllfiles)
642 archive = zipball(progfiles, ver, sourcefolder, platform)
643 # only when running native right now.
644 if nsisscript != "" and makensis != "":
645 nsisfileinject(sourcefolder + "/" + nsisscript, sourcefolder \
646 + "/" + nsisscript + ".tmp", dllfiles)
647 runnsis(ver, makensis, nsisscript + ".tmp", sourcefolder)
648 elif platform == "darwin":
649 archive = macdeploy(ver, sourcefolder, platform)
650 else:
651 if os.uname()[4].endswith("64"):
652 ver += "-64bit"
653 archive = tarball(progfiles, ver, sourcefolder)
655 # remove temporary files
656 tempclean(workfolder, cleanup)
658 # display summary
659 headline = "Build Summary for %s" % program
660 print "\n", headline, "\n", "=" * len(headline)
661 if not archivename == "":
662 filestats(archivename)
663 filestats(archive)
664 duration = time.time() - startup
665 durmins = (int)(duration / 60)
666 dursecs = (int)(duration % 60)
667 buildmins = (int)(buildtime / 60)
668 buildsecs = (int)(buildtime % 60)
669 print "Overall time %smin %ssec, building took %smin %ssec." % \
670 (durmins, dursecs, buildmins, buildsecs)
673 if __name__ == "__main__":
674 print "You cannot run this module directly!"
675 print "Set required environment and call deploy()."