Bump version numbers for 3.13
[maemo-rb.git] / utils / common / deploy.py
blob8f7fd72175e77229119a2f1dfaba300dba1557ae
1 #!/usr/bin/python
2 # __________ __ ___.
3 # Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 # Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 # Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 # Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
7 # \/ \/ \/ \/ \/
8 # $Id$
10 # Copyright (c) 2009 Dominik Riebeling
12 # All files in this archive are subject to the GNU General Public License.
13 # See the file COPYING in the source tree root for full license agreement.
15 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 # KIND, either express or implied.
19 # Automate building releases for deployment.
20 # Run from any folder to build
21 # - trunk
22 # - any tag (using the -t option)
23 # - any local folder (using the -p option)
24 # Will build a binary archive (tar.bz2 / zip) and source archive.
25 # The source archive won't be built for local builds. Trunk and
26 # tag builds will retrieve the sources directly from svn and build
27 # below the systems temporary folder.
29 # If the required Qt installation isn't in PATH use --qmake option.
30 # Tested on Linux and MinGW / W32
32 # requires upx.exe in PATH on Windows.
35 import re
36 import os
37 import sys
38 import tarfile
39 import zipfile
40 import shutil
41 import subprocess
42 import getopt
43 import time
44 import hashlib
45 import tempfile
46 import string
47 import gitscraper
49 # modules that are not part of python itself.
50 cpus = 1
51 try:
52 import multiprocessing
53 cpus = multiprocessing.cpu_count()
54 print "Info: %s cores found." % cpus
55 except ImportError:
56 print "Warning: multiprocessing module not found. Assuming 1 core."
58 # == Global stuff ==
59 # DLL files to ignore when searching for required DLL files.
60 systemdlls = ['advapi32.dll',
61 'comdlg32.dll',
62 'gdi32.dll',
63 'imm32.dll',
64 'kernel32.dll',
65 'msvcrt.dll',
66 'msvcrt.dll',
67 'netapi32.dll',
68 'ole32.dll',
69 'oleaut32.dll',
70 'setupapi.dll',
71 'shell32.dll',
72 'user32.dll',
73 'winmm.dll',
74 'winspool.drv',
75 'ws2_32.dll']
77 gitrepo = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))
80 # == Functions ==
81 def usage(myself):
82 print "Usage: %s [options]" % myself
83 print " -q, --qmake=<qmake> path to qmake"
84 print " -p, --project=<pro> path to .pro file for building with local tree"
85 print " -t, --tag=<tag> use specified tag from svn"
86 print " -a, --add=<file> add file to build folder before building"
87 print " -s, --source-only only create source archive"
88 print " -b, --binary-only only create binary archive"
89 if nsisscript != "":
90 print " -n, --makensis=<file> path to makensis for building Windows setup program."
91 if sys.platform != "darwin":
92 print " -d, --dynamic link dynamically instead of static"
93 if sys.platform != "win32":
94 print " -x, --cross= prefix to cross compile for win32"
95 print " -k, --keep-temp keep temporary folder on build failure"
96 print " -h, --help this help"
97 print " If neither a project file nor tag is specified trunk will get downloaded"
98 print " from svn."
101 def which(executable):
102 path = os.environ.get("PATH", "").split(os.pathsep)
103 for p in path:
104 fullpath = p + "/" + executable
105 if os.path.exists(fullpath):
106 return fullpath
107 print "which: could not find " + executable
108 return ""
111 def getsources(treehash, filelist, dest):
112 '''Get the files listed in filelist from svnsrv and put it at dest.'''
113 gitscraper.scrape_files(gitrepo, treehash, filelist, dest)
114 return 0
117 def getfolderrev(svnsrv):
118 '''Get the most recent revision for svnsrv'''
119 client = pysvn.Client()
120 entries = client.info2(svnsrv, recurse=False)
121 return entries[0][1].rev.number
124 def findversion(versionfile):
125 '''figure most recent program version from version.h,
126 returns version string.'''
127 h = open(versionfile, "r")
128 c = h.read()
129 h.close()
130 r = re.compile("#define +VERSION +\"(.[0-9\.a-z]+)\"")
131 m = re.search(r, c)
132 s = re.compile("\$Revision: +([0-9]+)")
133 n = re.search(s, c)
134 if n == None:
135 print "WARNING: Revision not found!"
136 return m.group(1)
139 def findqt(cross=""):
140 '''Search for Qt4 installation. Return path to qmake.'''
141 print "Searching for Qt"
142 bins = [cross + "qmake", cross + "qmake-qt4"]
143 for binary in bins:
144 try:
145 q = which(binary)
146 if len(q) > 0:
147 result = checkqt(q)
148 if not result == "":
149 return result
150 except:
151 print sys.exc_value
153 return ""
156 def checkqt(qmakebin):
157 '''Check if given path to qmake exists and is a suitable version.'''
158 result = ""
159 # check if binary exists
160 if not os.path.exists(qmakebin):
161 print "Specified qmake path does not exist!"
162 return result
163 # check version
164 output = subprocess.Popen([qmakebin, "-version"], stdout=subprocess.PIPE,
165 stderr=subprocess.PIPE)
166 cmdout = output.communicate()
167 # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
168 for ou in cmdout:
169 r = re.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
170 m = re.search(r, ou)
171 if not m == None:
172 print "Qt found: %s" % m.group(1)
173 s = re.compile("4\..*")
174 n = re.search(s, m.group(1))
175 if not n == None:
176 result = qmakebin
177 return result
180 def qmake(qmake, projfile, platform=sys.platform, wd=".", static=True, cross=""):
181 print "Running qmake in %s..." % wd
182 command = [qmake, "-config", "release", "-config", "noccache"]
183 if static == True:
184 command.extend(["-config", "-static"])
185 # special spec required?
186 if len(qmakespec[platform]) > 0:
187 command.extend(["-spec", qmakespec[platform]])
188 # cross compiling prefix set?
189 if len(cross) > 0:
190 command.extend(["-config", "cross"])
191 command.append(projfile)
192 output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd)
193 output.communicate()
194 if not output.returncode == 0:
195 print "qmake returned an error!"
196 return -1
197 return 0
200 def build(wd=".", platform=sys.platform, cross=""):
201 # make
202 print "Building ..."
203 # use the current platforms make here, cross compiling uses the native make.
204 command = [make[sys.platform]]
205 if cpus > 1:
206 command.append("-j")
207 command.append(str(cpus))
208 output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd)
209 while True:
210 c = output.stdout.readline()
211 sys.stdout.write(".")
212 sys.stdout.flush()
213 if not output.poll() == None:
214 sys.stdout.write("\n")
215 sys.stdout.flush()
216 if not output.returncode == 0:
217 print "Build failed!"
218 return -1
219 break
220 if platform != "darwin":
221 # strip. OS X handles this via macdeployqt.
222 print "Stripping binary."
223 output = subprocess.Popen([cross + "strip", progexe[platform]], \
224 stdout=subprocess.PIPE, cwd=wd)
225 output.communicate()
226 if not output.returncode == 0:
227 print "Stripping failed!"
228 return -1
229 return 0
232 def upxfile(wd=".", platform=sys.platform):
233 # run upx on binary
234 print "UPX'ing binary ..."
235 output = subprocess.Popen(["upx", progexe[platform]], \
236 stdout=subprocess.PIPE, cwd=wd)
237 output.communicate()
238 if not output.returncode == 0:
239 print "UPX'ing failed!"
240 return -1
241 return 0
244 def runnsis(versionstring, nsis, script, srcfolder):
245 # run script through nsis to create installer.
246 print "Running NSIS ..."
248 # Assume the generated installer gets placed in the same folder the nsi
249 # script lives in. This seems to be a valid assumption unless the nsi
250 # script specifies a path. NSIS expects files relative to source folder so
251 # copy progexe. Additional files are injected into the nsis script.
253 # FIXME: instead of copying binaries around copy the NSI file and inject
254 # the correct paths.
255 # Only win32 supported as target platform so hard coded.
256 b = srcfolder + "/" + os.path.dirname(script) + "/" \
257 + os.path.dirname(progexe["win32"])
258 if not os.path.exists(b):
259 os.mkdir(b)
260 shutil.copy(srcfolder + "/" + progexe["win32"], b)
261 output = subprocess.Popen([nsis, srcfolder + "/" + script], \
262 stdout=subprocess.PIPE)
263 output.communicate()
264 if not output.returncode == 0:
265 print "NSIS failed!"
266 return -1
267 setupfile = program + "-" + versionstring + "-setup.exe"
268 # find output filename in nsis script file
269 nsissetup = ""
270 for line in open(srcfolder + "/" + script):
271 if re.match(r'^[^;]*OutFile\s+', line) != None:
272 nsissetup = re.sub(r'^[^;]*OutFile\s+"(.+)"', r'\1', line).rstrip()
273 if nsissetup == "":
274 print "Could not retrieve output file name!"
275 return -1
276 shutil.copy(srcfolder + "/" + os.path.dirname(script) + "/" + nsissetup, \
277 setupfile)
278 return 0
281 def nsisfileinject(nsis, outscript, filelist):
282 '''Inject files in filelist into NSIS script file after the File line
283 containing the main binary. This assumes that the main binary is present
284 in the NSIS script and that all additiona files (dlls etc) to get placed
285 into $INSTDIR.'''
286 output = open(outscript, "w")
287 for line in open(nsis, "r"):
288 output.write(line)
289 # inject files after the progexe binary.
290 # Match the basename only to avoid path mismatches.
291 if re.match(r'^\s*File\s*.*' + os.path.basename(progexe["win32"]), \
292 line, re.IGNORECASE):
293 for f in filelist:
294 injection = " File /oname=$INSTDIR\\" + os.path.basename(f) \
295 + " " + os.path.normcase(f) + "\n"
296 output.write(injection)
297 output.write(" ; end of injected files\n")
298 output.close()
301 def finddlls(program, extrapaths=[], cross=""):
302 '''Check program for required DLLs. Find all required DLLs except ignored
303 ones and return a list of DLL filenames (including path).'''
304 # ask objdump about dependencies.
305 output = subprocess.Popen([cross + "objdump", "-x", program], \
306 stdout=subprocess.PIPE)
307 cmdout = output.communicate()
309 # create list of used DLLs. Store as lower case as W32 is case-insensitive.
310 dlls = []
311 for line in cmdout[0].split('\n'):
312 if re.match(r'\s*DLL Name', line) != None:
313 dll = re.sub(r'^\s*DLL Name:\s+([a-zA-Z_\-0-9\.\+]+).*$', r'\1', line)
314 dlls.append(dll.lower())
316 # find DLLs in extrapaths and PATH environment variable.
317 dllpaths = []
318 for file in dlls:
319 if file in systemdlls:
320 print "System DLL: " + file
321 continue
322 dllpath = ""
323 for path in extrapaths:
324 if os.path.exists(path + "/" + file):
325 dllpath = re.sub(r"\\", r"/", path + "/" + file)
326 print file + ": found at " + dllpath
327 dllpaths.append(dllpath)
328 break
329 if dllpath == "":
330 try:
331 dllpath = re.sub(r"\\", r"/", which(file))
332 print file + ": found at " + dllpath
333 dllpaths.append(dllpath)
334 except:
335 print "MISSING DLL: " + file
336 return dllpaths
339 def zipball(programfiles, versionstring, buildfolder, platform=sys.platform):
340 '''package created binary'''
341 print "Creating binary zipball."
342 archivebase = program + "-" + versionstring
343 outfolder = buildfolder + "/" + archivebase
344 archivename = archivebase + ".zip"
345 # create output folder
346 os.mkdir(outfolder)
347 # move program files to output folder
348 for f in programfiles:
349 if re.match(r'^(/|[a-zA-Z]:)', f) != None:
350 shutil.copy(f, outfolder)
351 else:
352 shutil.copy(buildfolder + "/" + f, outfolder)
353 # create zipball from output folder
354 zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED)
355 for root, dirs, files in os.walk(outfolder):
356 for name in files:
357 physname = os.path.normpath(os.path.join(root, name))
358 filename = string.replace(physname, os.path.normpath(buildfolder), "")
359 zf.write(physname, filename)
360 for name in dirs:
361 physname = os.path.normpath(os.path.join(root, name))
362 filename = string.replace(physname, os.path.normpath(buildfolder), "")
363 zf.write(physname, filename)
364 zf.close()
365 # remove output folder
366 shutil.rmtree(outfolder)
367 return archivename
370 def tarball(programfiles, versionstring, buildfolder):
371 '''package created binary'''
372 print "Creating binary tarball."
373 archivebase = program + "-" + versionstring
374 outfolder = buildfolder + "/" + archivebase
375 archivename = archivebase + ".tar.bz2"
376 # create output folder
377 os.mkdir(outfolder)
378 # move program files to output folder
379 for f in programfiles:
380 shutil.copy(buildfolder + "/" + f, outfolder)
381 # create tarball from output folder
382 tf = tarfile.open(archivename, mode='w:bz2')
383 tf.add(outfolder, archivebase)
384 tf.close()
385 # remove output folder
386 shutil.rmtree(outfolder)
387 return archivename
390 def macdeploy(versionstring, buildfolder, platform=sys.platform):
391 '''package created binary to dmg'''
392 dmgfile = program + "-" + versionstring + ".dmg"
393 appbundle = buildfolder + "/" + progexe[platform]
395 # workaround to Qt issues when building out-of-tree. Copy files into bundle.
396 sourcebase = buildfolder + re.sub('[^/]+.pro$', '', project) + "/"
397 print sourcebase
398 for src in bundlecopy:
399 shutil.copy(sourcebase + src, appbundle + "/" + bundlecopy[src])
400 # end of Qt workaround
402 output = subprocess.Popen(["macdeployqt", progexe[platform], "-dmg"], \
403 stdout=subprocess.PIPE, cwd=buildfolder)
404 output.communicate()
405 if not output.returncode == 0:
406 print "macdeployqt failed!"
407 return -1
408 # copy dmg to output folder
409 shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile)
410 return dmgfile
413 def filehashes(filename):
414 '''Calculate md5 and sha1 hashes for a given file.'''
415 if not os.path.exists(filename):
416 return ["", ""]
417 m = hashlib.md5()
418 s = hashlib.sha1()
419 f = open(filename, 'rb')
420 while True:
421 d = f.read(65536)
422 if d == "":
423 break
424 m.update(d)
425 s.update(d)
426 return [m.hexdigest(), s.hexdigest()]
429 def filestats(filename):
430 if not os.path.exists(filename):
431 return
432 st = os.stat(filename)
433 print filename, "\n", "-" * len(filename)
434 print "Size: %i bytes" % st.st_size
435 h = filehashes(filename)
436 print "md5sum: %s" % h[0]
437 print "sha1sum: %s" % h[1]
438 print "-" * len(filename), "\n"
441 def tempclean(workfolder, nopro):
442 if nopro == True:
443 print "Cleaning up working folder %s" % workfolder
444 shutil.rmtree(workfolder)
445 else:
446 print "Project file specified or cleanup disabled!"
447 print "Temporary files kept at %s" % workfolder
450 def deploy():
451 startup = time.time()
453 try:
454 opts, args = getopt.getopt(sys.argv[1:], "q:p:t:a:n:sbdkx:i:h",
455 ["qmake=", "project=", "tag=", "add=", "makensis=", "source-only",
456 "binary-only", "dynamic", "keep-temp", "cross=", "buildid=", "help"])
457 except getopt.GetoptError, err:
458 print str(err)
459 usage(sys.argv[0])
460 sys.exit(1)
461 qt = ""
462 proj = ""
463 svnbase = svnserver + "trunk/"
464 tag = ""
465 addfiles = []
466 cleanup = True
467 binary = True
468 source = True
469 keeptemp = False
470 makensis = ""
471 cross = ""
472 buildid = None
473 platform = sys.platform
474 treehash = gitscraper.get_refs(gitrepo)['refs/remotes/origin/HEAD']
475 if sys.platform != "darwin":
476 static = True
477 else:
478 static = False
479 for o, a in opts:
480 if o in ("-q", "--qmake"):
481 qt = a
482 if o in ("-p", "--project"):
483 proj = a
484 cleanup = False
485 if o in ("-a", "--add"):
486 addfiles.append(a)
487 if o in ("-n", "--makensis"):
488 makensis = a
489 if o in ("-s", "--source-only"):
490 binary = False
491 if o in ("-b", "--binary-only"):
492 source = False
493 if o in ("-d", "--dynamic") and sys.platform != "darwin":
494 static = False
495 if o in ("-k", "--keep-temp"):
496 keeptemp = True
497 if o in ("-t", "--tree"):
498 treehash = a
499 if o in ("-x", "--cross") and sys.platform != "win32":
500 cross = a
501 platform = "win32"
502 if o in ("-i", "--buildid"):
503 buildid = a
504 if o in ("-h", "--help"):
505 usage(sys.argv[0])
506 sys.exit(0)
508 if source == False and binary == False:
509 print "Building build neither source nor binary means nothing to do. Exiting."
510 sys.exit(1)
512 print "Building " + progexe[platform] + " for " + platform
513 # search for qmake
514 if qt == "":
515 qm = findqt(cross)
516 else:
517 qm = checkqt(qt)
518 if qm == "":
519 print "ERROR: No suitable Qt installation found."
520 sys.exit(1)
522 # create working folder. Use current directory if -p option used.
523 if proj == "":
524 w = tempfile.mkdtemp()
525 # make sure the path doesn't contain backslashes to prevent issues
526 # later when running on windows.
527 workfolder = re.sub(r'\\', '/', w)
528 revision = gitscraper.describe_treehash(gitrepo, treehash)
529 # try to find a version number from describe output.
530 # WARNING: this is broken and just a temporary workaround!
531 v = re.findall('([\d\.a-f]+)', revision)
532 if v:
533 if v[-1].find('.') >= 0:
534 revision = "v" + v[-1]
535 else:
536 revision = v[-1]
537 if buildid == None:
538 versionextra = ""
539 else:
540 versionextra = "-" + buildid
541 sourcefolder = workfolder + "/" + program + "-" + str(revision) + versionextra + "/"
542 archivename = program + "-" + str(revision) + versionextra + "-src.tar.bz2"
543 ver = str(revision)
544 os.mkdir(sourcefolder)
545 else:
546 workfolder = "."
547 sourcefolder = "."
548 archivename = ""
549 # check if project file explicitly given. If yes, don't get sources from svn
550 print "Version: %s" % revision
551 if proj == "":
552 proj = sourcefolder + project
553 # get sources and pack source tarball
554 if not getsources(treehash, svnpaths, sourcefolder) == 0:
555 tempclean(workfolder, cleanup and not keeptemp)
556 sys.exit(1)
558 # replace version strings.
559 print "Updating version information in sources"
560 for f in regreplace:
561 infile = open(sourcefolder + "/" + f, "r")
562 incontents = infile.readlines()
563 infile.close()
565 outfile = open(sourcefolder + "/" + f, "w")
566 for line in incontents:
567 newline = line
568 for r in regreplace[f]:
569 # replacements made on the replacement string:
570 # %REVISION% is replaced with the revision number
571 replacement = re.sub("%REVISION%", str(revision), r[1])
572 newline = re.sub(r[0], replacement, newline)
573 # %BUILD% is replaced with buildid as passed on the command line
574 if buildid != None:
575 replacement = re.sub("%BUILDID%", "-" + str(buildid), replacement)
576 else:
577 replacement = re.sub("%BUILDID%", "", replacement)
578 newline = re.sub(r[0], replacement, newline)
579 outfile.write(newline)
580 outfile.close()
582 if source == True:
583 print "Creating source tarball %s\n" % archivename
584 tf = tarfile.open(archivename, mode='w:bz2')
585 tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
586 tf.close()
587 if binary == False:
588 shutil.rmtree(workfolder)
589 sys.exit(0)
590 else:
591 # figure version from sources. Need to take path to project file into account.
592 versionfile = re.subn('[\w\.]+$', "version.h", proj)[0]
593 ver = findversion(versionfile)
594 # append buildid if any.
595 if buildid != None:
596 ver += "-" + buildid
598 # check project file
599 if not os.path.exists(proj):
600 print "ERROR: path to project file wrong."
601 sys.exit(1)
603 # copy specified (--add) files to working folder
604 for f in addfiles:
605 shutil.copy(f, sourcefolder)
606 buildstart = time.time()
607 header = "Building %s %s" % (program, ver)
608 print header
609 print len(header) * "="
611 # build it.
612 if not qmake(qm, proj, platform, sourcefolder, static, cross) == 0:
613 tempclean(workfolder, cleanup and not keeptemp)
614 sys.exit(1)
615 if not build(sourcefolder, platform, cross) == 0:
616 tempclean(workfolder, cleanup and not keeptemp)
617 sys.exit(1)
618 buildtime = time.time() - buildstart
619 progfiles = programfiles
620 progfiles.append(progexe[platform])
621 if platform == "win32":
622 if useupx == True:
623 if not upxfile(sourcefolder, platform) == 0:
624 tempclean(workfolder, cleanup and not keeptemp)
625 sys.exit(1)
626 dllfiles = finddlls(sourcefolder + "/" + progexe[platform], \
627 [os.path.dirname(qm)], cross)
628 if len(dllfiles) > 0:
629 progfiles.extend(dllfiles)
630 archive = zipball(progfiles, ver, sourcefolder, platform)
631 # only when running native right now.
632 if nsisscript != "" and makensis != "":
633 nsisfileinject(sourcefolder + "/" + nsisscript, sourcefolder \
634 + "/" + nsisscript + ".tmp", dllfiles)
635 runnsis(ver, makensis, nsisscript + ".tmp", sourcefolder)
636 elif platform == "darwin":
637 archive = macdeploy(ver, sourcefolder, platform)
638 else:
639 if platform == "linux2":
640 for p in progfiles:
641 prog = sourcefolder + "/" + p
642 output = subprocess.Popen(["file", prog],
643 stdout=subprocess.PIPE)
644 res = output.communicate()
645 if re.findall("ELF 64-bit", res[0]):
646 ver += "-64bit"
647 break
649 archive = tarball(progfiles, ver, sourcefolder)
651 # remove temporary files
652 tempclean(workfolder, cleanup)
654 # display summary
655 headline = "Build Summary for %s" % program
656 print "\n", headline, "\n", "=" * len(headline)
657 if not archivename == "":
658 filestats(archivename)
659 filestats(archive)
660 duration = time.time() - startup
661 durmins = (int)(duration / 60)
662 dursecs = (int)(duration % 60)
663 buildmins = (int)(buildtime / 60)
664 buildsecs = (int)(buildtime % 60)
665 print "Overall time %smin %ssec, building took %smin %ssec." % \
666 (durmins, dursecs, buildmins, buildsecs)
669 if __name__ == "__main__":
670 print "You cannot run this module directly!"
671 print "Set required environment and call deploy()."