3 # Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 # Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 # Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 # Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
10 # Copyright (c) 2009 Dominik Riebeling
12 # All files in this archive are subject to the GNU General Public License.
13 # See the file COPYING in the source tree root for full license agreement.
15 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 # KIND, either express or implied.
19 # Automate building releases for deployment.
20 # Run from any folder to build
22 # - any tag (using the -t option)
23 # - any local folder (using the -p option)
24 # Will build a binary archive (tar.bz2 / zip) and source archive.
25 # The source archive won't be built for local builds. Trunk and
26 # tag builds will retrieve the sources directly from svn and build
27 # below the systems temporary folder.
29 # If the required Qt installation isn't in PATH use --qmake option.
30 # Tested on Linux and MinGW / W32
32 # requires python which package (http://code.google.com/p/which/)
33 # requires pysvn package.
34 # requires upx.exe in PATH on Windows.
50 # modules that are not part of python itself.
54 print "Fatal: This script requires the pysvn package to run."
55 print " See http://pysvn.tigris.org/."
60 print "Fatal: This script requires the which package to run."
61 print " See http://code.google.com/p/which/."
65 import multiprocessing
66 cpus
= multiprocessing
.cpu_count()
67 print "Info: %s cores found." % cpus
69 print "Warning: multiprocessing module not found. Assuming 1 core."
72 # DLL files to ignore when searching for required DLL files.
73 systemdlls
= ['advapi32.dll',
93 print "Usage: %s [options]" % myself
94 print " -q, --qmake=<qmake> path to qmake"
95 print " -p, --project=<pro> path to .pro file for building with local tree"
96 print " -t, --tag=<tag> use specified tag from svn"
97 print " -a, --add=<file> add file to build folder before building"
98 print " -s, --source-only only create source archive"
99 print " -b, --binary-only only create binary archive"
101 print " -n, --makensis=<file> path to makensis for building Windows setup program."
102 if sys
.platform
!= "darwin":
103 print " -d, --dynamic link dynamically instead of static"
104 if sys
.platform
!= "win32":
105 print " -x, --cross= prefix to cross compile for win32"
106 print " -k, --keep-temp keep temporary folder on build failure"
107 print " -h, --help this help"
108 print " If neither a project file nor tag is specified trunk will get downloaded"
112 def getsources(svnsrv
, filelist
, dest
):
113 '''Get the files listed in filelist from svnsrv and put it at dest.'''
114 client
= pysvn
.Client()
115 print "Checking out sources from %s, please wait." % svnsrv
117 for elem
in filelist
:
118 url
= re
.subn('/$', '', svnsrv
+ elem
)[0]
119 destpath
= re
.subn('/$', '', dest
+ elem
)[0]
120 # make sure the destination path does exist
121 d
= os
.path
.dirname(destpath
)
122 if not os
.path
.exists(d
):
126 client
.export(url
, destpath
)
128 print "SVN client error: %s" % sys
.exc_value
129 print "URL: %s, destination: %s" % (url
, destpath
)
131 print "Checkout finished."
135 def getfolderrev(svnsrv
):
136 '''Get the most recent revision for svnsrv'''
137 client
= pysvn
.Client()
138 entries
= client
.info2(svnsrv
, recurse
=False)
139 return entries
[0][1].rev
.number
142 def findversion(versionfile
):
143 '''figure most recent program version from version.h,
144 returns version string.'''
145 h
= open(versionfile
, "r")
148 r
= re
.compile("#define +VERSION +\"(.[0-9\.a-z]+)\"")
150 s
= re
.compile("\$Revision: +([0-9]+)")
153 print "WARNING: Revision not found!"
157 def findqt(cross
=""):
158 '''Search for Qt4 installation. Return path to qmake.'''
159 print "Searching for Qt"
160 bins
= [cross
+ "qmake", cross
+ "qmake-qt4"]
163 q
= which
.which(binary
)
174 def checkqt(qmakebin
):
175 '''Check if given path to qmake exists and is a suitable version.'''
177 # check if binary exists
178 if not os
.path
.exists(qmakebin
):
179 print "Specified qmake path does not exist!"
182 output
= subprocess
.Popen([qmakebin
, "-version"], stdout
=subprocess
.PIPE
,
183 stderr
=subprocess
.PIPE
)
184 cmdout
= output
.communicate()
185 # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
187 r
= re
.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
190 print "Qt found: %s" % m
.group(1)
191 s
= re
.compile("4\..*")
192 n
= re
.search(s
, m
.group(1))
198 def qmake(qmake
, projfile
, platform
=sys
.platform
, wd
=".", static
=True, cross
=""):
199 print "Running qmake in %s..." % wd
200 command
= [qmake
, "-config", "release", "-config", "noccache"]
202 command
.extend(["-config", "-static"])
203 # special spec required?
204 if len(qmakespec
[platform
]) > 0:
205 command
.extend(["-spec", qmakespec
[platform
]])
206 # cross compiling prefix set?
208 command
.extend(["-config", "cross"])
209 command
.append(projfile
)
210 output
= subprocess
.Popen(command
, stdout
=subprocess
.PIPE
, cwd
=wd
)
212 if not output
.returncode
== 0:
213 print "qmake returned an error!"
218 def build(wd
=".", platform
=sys
.platform
, cross
=""):
221 # use the current platforms make here, cross compiling uses the native make.
222 command
= [make
[sys
.platform
]]
225 command
.append(str(cpus
))
226 output
= subprocess
.Popen(command
, stdout
=subprocess
.PIPE
, cwd
=wd
)
228 c
= output
.stdout
.readline()
229 sys
.stdout
.write(".")
231 if not output
.poll() == None:
232 sys
.stdout
.write("\n")
234 if not output
.returncode
== 0:
235 print "Build failed!"
238 if platform
!= "darwin":
239 # strip. OS X handles this via macdeployqt.
240 print "Stripping binary."
241 output
= subprocess
.Popen([cross
+ "strip", progexe
[platform
]], \
242 stdout
=subprocess
.PIPE
, cwd
=wd
)
244 if not output
.returncode
== 0:
245 print "Stripping failed!"
250 def upxfile(wd
=".", platform
=sys
.platform
):
252 print "UPX'ing binary ..."
253 output
= subprocess
.Popen(["upx", progexe
[platform
]], \
254 stdout
=subprocess
.PIPE
, cwd
=wd
)
256 if not output
.returncode
== 0:
257 print "UPX'ing failed!"
262 def runnsis(versionstring
, nsis
, script
, srcfolder
):
263 # run script through nsis to create installer.
264 print "Running NSIS ..."
266 # Assume the generated installer gets placed in the same folder the nsi
267 # script lives in. This seems to be a valid assumption unless the nsi
268 # script specifies a path. NSIS expects files relative to source folder so
269 # copy progexe. Additional files are injected into the nsis script.
271 # FIXME: instead of copying binaries around copy the NSI file and inject
273 # Only win32 supported as target platform so hard coded.
274 b
= srcfolder
+ "/" + os
.path
.dirname(script
) + "/" \
275 + os
.path
.dirname(progexe
["win32"])
276 if not os
.path
.exists(b
):
278 shutil
.copy(srcfolder
+ "/" + progexe
["win32"], b
)
279 output
= subprocess
.Popen([nsis
, srcfolder
+ "/" + script
], \
280 stdout
=subprocess
.PIPE
)
282 if not output
.returncode
== 0:
285 setupfile
= program
+ "-" + versionstring
+ "-setup.exe"
286 # find output filename in nsis script file
288 for line
in open(srcfolder
+ "/" + script
):
289 if re
.match(r
'^[^;]*OutFile\s+', line
) != None:
290 nsissetup
= re
.sub(r
'^[^;]*OutFile\s+"(.+)"', r
'\1', line
).rstrip()
292 print "Could not retrieve output file name!"
294 shutil
.copy(srcfolder
+ "/" + os
.path
.dirname(script
) + "/" + nsissetup
, \
299 def nsisfileinject(nsis
, outscript
, filelist
):
300 '''Inject files in filelist into NSIS script file after the File line
301 containing the main binary. This assumes that the main binary is present
302 in the NSIS script and that all additiona files (dlls etc) to get placed
304 output
= open(outscript
, "w")
305 for line
in open(nsis
, "r"):
307 # inject files after the progexe binary.
308 # Match the basename only to avoid path mismatches.
309 if re
.match(r
'^\s*File\s*.*' + os
.path
.basename(progexe
["win32"]), \
310 line
, re
.IGNORECASE
):
312 injection
= " File /oname=$INSTDIR\\" + os
.path
.basename(f
) \
313 + " " + os
.path
.normcase(f
) + "\n"
314 output
.write(injection
)
315 output
.write(" ; end of injected files\n")
319 def finddlls(program
, extrapaths
=[], cross
=""):
320 '''Check program for required DLLs. Find all required DLLs except ignored
321 ones and return a list of DLL filenames (including path).'''
322 # ask objdump about dependencies.
323 output
= subprocess
.Popen([cross
+ "objdump", "-x", program
], \
324 stdout
=subprocess
.PIPE
)
325 cmdout
= output
.communicate()
327 # create list of used DLLs. Store as lower case as W32 is case-insensitive.
329 for line
in cmdout
[0].split('\n'):
330 if re
.match(r
'\s*DLL Name', line
) != None:
331 dll
= re
.sub(r
'^\s*DLL Name:\s+([a-zA-Z_\-0-9\.\+]+).*$', r
'\1', line
)
332 dlls
.append(dll
.lower())
334 # find DLLs in extrapaths and PATH environment variable.
337 if file in systemdlls
:
338 print "System DLL: " + file
341 for path
in extrapaths
:
342 if os
.path
.exists(path
+ "/" + file):
343 dllpath
= re
.sub(r
"\\", r
"/", path
+ "/" + file)
344 print file + ": found at " + dllpath
345 dllpaths
.append(dllpath
)
349 dllpath
= re
.sub(r
"\\", r
"/", which
.which(file))
350 print file + ": found at " + dllpath
351 dllpaths
.append(dllpath
)
353 print "MISSING DLL: " + file
357 def zipball(programfiles
, versionstring
, buildfolder
, platform
=sys
.platform
):
358 '''package created binary'''
359 print "Creating binary zipball."
360 archivebase
= program
+ "-" + versionstring
361 outfolder
= buildfolder
+ "/" + archivebase
362 archivename
= archivebase
+ ".zip"
363 # create output folder
365 # move program files to output folder
366 for f
in programfiles
:
367 if re
.match(r
'^(/|[a-zA-Z]:)', f
) != None:
368 shutil
.copy(f
, outfolder
)
370 shutil
.copy(buildfolder
+ "/" + f
, outfolder
)
371 # create zipball from output folder
372 zf
= zipfile
.ZipFile(archivename
, mode
='w', compression
=zipfile
.ZIP_DEFLATED
)
373 for root
, dirs
, files
in os
.walk(outfolder
):
375 physname
= os
.path
.normpath(os
.path
.join(root
, name
))
376 filename
= string
.replace(physname
, os
.path
.normpath(buildfolder
), "")
377 zf
.write(physname
, filename
)
379 physname
= os
.path
.normpath(os
.path
.join(root
, name
))
380 filename
= string
.replace(physname
, os
.path
.normpath(buildfolder
), "")
381 zf
.write(physname
, filename
)
383 # remove output folder
384 shutil
.rmtree(outfolder
)
388 def tarball(programfiles
, versionstring
, buildfolder
):
389 '''package created binary'''
390 print "Creating binary tarball."
391 archivebase
= program
+ "-" + versionstring
392 outfolder
= buildfolder
+ "/" + archivebase
393 archivename
= archivebase
+ ".tar.bz2"
394 # create output folder
396 # move program files to output folder
397 for f
in programfiles
:
398 shutil
.copy(buildfolder
+ "/" + f
, outfolder
)
399 # create tarball from output folder
400 tf
= tarfile
.open(archivename
, mode
='w:bz2')
401 tf
.add(outfolder
, archivebase
)
403 # remove output folder
404 shutil
.rmtree(outfolder
)
408 def macdeploy(versionstring
, buildfolder
, platform
=sys
.platform
):
409 '''package created binary to dmg'''
410 dmgfile
= program
+ "-" + versionstring
+ ".dmg"
411 appbundle
= buildfolder
+ "/" + progexe
[platform
]
413 # workaround to Qt issues when building out-of-tree. Copy files into bundle.
414 sourcebase
= buildfolder
+ re
.sub('[^/]+.pro$', '', project
) + "/"
416 for src
in bundlecopy
:
417 shutil
.copy(sourcebase
+ src
, appbundle
+ "/" + bundlecopy
[src
])
418 # end of Qt workaround
420 output
= subprocess
.Popen(["macdeployqt", progexe
[platform
], "-dmg"], \
421 stdout
=subprocess
.PIPE
, cwd
=buildfolder
)
423 if not output
.returncode
== 0:
424 print "macdeployqt failed!"
426 # copy dmg to output folder
427 shutil
.copy(buildfolder
+ "/" + program
+ ".dmg", dmgfile
)
431 def filehashes(filename
):
432 '''Calculate md5 and sha1 hashes for a given file.'''
433 if not os
.path
.exists(filename
):
437 f
= open(filename
, 'rb')
444 return [m
.hexdigest(), s
.hexdigest()]
447 def filestats(filename
):
448 if not os
.path
.exists(filename
):
450 st
= os
.stat(filename
)
451 print filename
, "\n", "-" * len(filename
)
452 print "Size: %i bytes" % st
.st_size
453 h
= filehashes(filename
)
454 print "md5sum: %s" % h
[0]
455 print "sha1sum: %s" % h
[1]
456 print "-" * len(filename
), "\n"
459 def tempclean(workfolder
, nopro
):
461 print "Cleaning up working folder %s" % workfolder
462 shutil
.rmtree(workfolder
)
464 print "Project file specified or cleanup disabled!"
465 print "Temporary files kept at %s" % workfolder
469 startup
= time
.time()
472 opts
, args
= getopt
.getopt(sys
.argv
[1:], "q:p:t:a:n:sbdkx:i:h",
473 ["qmake=", "project=", "tag=", "add=", "makensis=", "source-only",
474 "binary-only", "dynamic", "keep-temp", "cross=", "buildid=", "help"])
475 except getopt
.GetoptError
, err
:
481 svnbase
= svnserver
+ "trunk/"
491 platform
= sys
.platform
492 if sys
.platform
!= "darwin":
497 if o
in ("-q", "--qmake"):
499 if o
in ("-p", "--project"):
502 if o
in ("-t", "--tag"):
504 svnbase
= svnserver
+ "tags/" + tag
+ "/"
505 if o
in ("-a", "--add"):
507 if o
in ("-n", "--makensis"):
509 if o
in ("-s", "--source-only"):
511 if o
in ("-b", "--binary-only"):
513 if o
in ("-d", "--dynamic") and sys
.platform
!= "darwin":
515 if o
in ("-k", "--keep-temp"):
517 if o
in ("-x", "--cross") and sys
.platform
!= "win32":
520 if o
in ("-i", "--buildid"):
522 if o
in ("-h", "--help"):
526 if source
== False and binary
== False:
527 print "Building build neither source nor binary means nothing to do. Exiting."
530 print "Building " + progexe
[platform
] + " for " + platform
537 print "ERROR: No suitable Qt installation found."
540 # create working folder. Use current directory if -p option used.
542 w
= tempfile
.mkdtemp()
543 # make sure the path doesn't contain backslashes to prevent issues
544 # later when running on windows.
545 workfolder
= re
.sub(r
'\\', '/', w
)
546 revision
= getfolderrev(svnbase
)
550 versionextra
= "-" + buildid
552 sourcefolder
= workfolder
+ "/" + tag
+ "/"
553 archivename
= tag
+ versionextra
+ "-src.tar.bz2"
554 # get numeric version part from tag
555 ver
= "v" + re
.sub('^[^\d]+', '', tag
)
557 sourcefolder
= workfolder
+ "/" + program
+ "-r" + str(revision
) + versionextra
+ "/"
558 archivename
= program
+ "-r" + str(revision
) + versionextra
+ "-src.tar.bz2"
559 ver
= "r" + str(revision
)
560 os
.mkdir(sourcefolder
)
565 # check if project file explicitly given. If yes, don't get sources from svn
567 proj
= sourcefolder
+ project
568 # get sources and pack source tarball
569 if not getsources(svnbase
, svnpaths
, sourcefolder
) == 0:
570 tempclean(workfolder
, cleanup
and not keeptemp
)
573 # replace version strings.
574 print "Updating version information in sources"
576 infile
= open(sourcefolder
+ "/" + f
, "r")
577 incontents
= infile
.readlines()
580 outfile
= open(sourcefolder
+ "/" + f
, "w")
581 for line
in incontents
:
583 for r
in regreplace
[f
]:
584 # replacements made on the replacement string:
585 # %REVISION% is replaced with the revision number
586 replacement
= re
.sub("%REVISION%", str(revision
), r
[1])
587 # %BUILD% is replace with buildid as passed on the command line
589 replacement
= re
.sub("%BUILDID%", str(buildid
), replacement
)
590 newline
= re
.sub(r
[0], replacement
, newline
)
591 outfile
.write(newline
)
595 print "Creating source tarball %s\n" % archivename
596 tf
= tarfile
.open(archivename
, mode
='w:bz2')
597 tf
.add(sourcefolder
, os
.path
.basename(re
.subn('/$', '', sourcefolder
)[0]))
600 shutil
.rmtree(workfolder
)
603 # figure version from sources. Need to take path to project file into account.
604 versionfile
= re
.subn('[\w\.]+$', "version.h", proj
)[0]
605 ver
= findversion(versionfile
)
606 # append buildid if any.
611 if not os
.path
.exists(proj
):
612 print "ERROR: path to project file wrong."
615 # copy specified (--add) files to working folder
617 shutil
.copy(f
, sourcefolder
)
618 buildstart
= time
.time()
619 header
= "Building %s %s" % (program
, ver
)
621 print len(header
) * "="
624 if not qmake(qm
, proj
, platform
, sourcefolder
, static
, cross
) == 0:
625 tempclean(workfolder
, cleanup
and not keeptemp
)
627 if not build(sourcefolder
, platform
, cross
) == 0:
628 tempclean(workfolder
, cleanup
and not keeptemp
)
630 buildtime
= time
.time() - buildstart
631 progfiles
= programfiles
632 progfiles
.append(progexe
[platform
])
633 if platform
== "win32":
635 if not upxfile(sourcefolder
, platform
) == 0:
636 tempclean(workfolder
, cleanup
and not keeptemp
)
638 dllfiles
= finddlls(sourcefolder
+ "/" + progexe
[platform
], \
639 [os
.path
.dirname(qm
)], cross
)
640 if dllfiles
.count
> 0:
641 progfiles
.extend(dllfiles
)
642 archive
= zipball(progfiles
, ver
, sourcefolder
, platform
)
643 # only when running native right now.
644 if nsisscript
!= "" and makensis
!= "":
645 nsisfileinject(sourcefolder
+ "/" + nsisscript
, sourcefolder \
646 + "/" + nsisscript
+ ".tmp", dllfiles
)
647 runnsis(ver
, makensis
, nsisscript
+ ".tmp", sourcefolder
)
648 elif platform
== "darwin":
649 archive
= macdeploy(ver
, sourcefolder
, platform
)
651 if os
.uname()[4].endswith("64"):
653 archive
= tarball(progfiles
, ver
, sourcefolder
)
655 # remove temporary files
656 tempclean(workfolder
, cleanup
)
659 headline
= "Build Summary for %s" % program
660 print "\n", headline
, "\n", "=" * len(headline
)
661 if not archivename
== "":
662 filestats(archivename
)
664 duration
= time
.time() - startup
665 durmins
= (int)(duration
/ 60)
666 dursecs
= (int)(duration
% 60)
667 buildmins
= (int)(buildtime
/ 60)
668 buildsecs
= (int)(buildtime
% 60)
669 print "Overall time %smin %ssec, building took %smin %ssec." % \
670 (durmins
, dursecs
, buildmins
, buildsecs
)
673 if __name__
== "__main__":
674 print "You cannot run this module directly!"
675 print "Set required environment and call deploy()."