3 # Open \______ \ ____ ____ | | _\_ |__ _______ ___
4 # Source | _// _ \_/ ___\| |/ /| __ \ / _ \ \/ /
5 # Jukebox | | ( <_> ) \___| < | \_\ ( <_> > < <
6 # Firmware |____|_ /\____/ \___ >__|_ \|___ /\____/__/\_ \
10 # Copyright (c) 2009 Dominik Riebeling
12 # All files in this archive are subject to the GNU General Public License.
13 # See the file COPYING in the source tree root for full license agreement.
15 # This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
16 # KIND, either express or implied.
19 # Automate building releases for deployment.
20 # Run from any folder to build
22 # - any tag (using the -t option)
23 # - any local folder (using the -p option)
24 # Will build a binary archive (tar.bz2 / zip) and source archive.
25 # The source archive won't be built for local builds. Trunk and
26 # tag builds will retrieve the sources directly from svn and build
27 # below the systems temporary folder.
29 # If the required Qt installation isn't in PATH use --qmake option.
30 # Tested on Linux and MinGW / W32
32 # requires upx.exe in PATH on Windows.
49 # modules that are not part of python itself.
52 import multiprocessing
53 cpus
= multiprocessing
.cpu_count()
54 print "Info: %s cores found." % cpus
56 print "Warning: multiprocessing module not found. Assuming 1 core."
59 # DLL files to ignore when searching for required DLL files.
60 systemdlls
= ['advapi32.dll',
77 gitrepo
= os
.path
.abspath(os
.path
.join(os
.path
.dirname(__file__
), "../.."))
82 print "Usage: %s [options]" % myself
83 print " -q, --qmake=<qmake> path to qmake"
84 print " -p, --project=<pro> path to .pro file for building with local tree"
85 print " -t, --tag=<tag> use specified tag from svn"
86 print " -a, --add=<file> add file to build folder before building"
87 print " -s, --source-only only create source archive"
88 print " -b, --binary-only only create binary archive"
90 print " -n, --makensis=<file> path to makensis for building Windows setup program."
91 if sys
.platform
!= "darwin":
92 print " -d, --dynamic link dynamically instead of static"
93 if sys
.platform
!= "win32":
94 print " -x, --cross= prefix to cross compile for win32"
95 print " -k, --keep-temp keep temporary folder on build failure"
96 print " -h, --help this help"
97 print " If neither a project file nor tag is specified trunk will get downloaded"
101 def which(executable
):
102 path
= os
.environ
.get("PATH", "").split(os
.pathsep
)
104 fullpath
= p
+ "/" + executable
105 if os
.path
.exists(fullpath
):
107 print "which: could not find " + executable
111 def getsources(treehash
, filelist
, dest
):
112 '''Get the files listed in filelist from svnsrv and put it at dest.'''
113 gitscraper
.scrape_files(gitrepo
, treehash
, filelist
, dest
)
117 def getfolderrev(svnsrv
):
118 '''Get the most recent revision for svnsrv'''
119 client
= pysvn
.Client()
120 entries
= client
.info2(svnsrv
, recurse
=False)
121 return entries
[0][1].rev
.number
124 def findversion(versionfile
):
125 '''figure most recent program version from version.h,
126 returns version string.'''
127 h
= open(versionfile
, "r")
130 r
= re
.compile("#define +VERSION +\"(.[0-9\.a-z]+)\"")
132 s
= re
.compile("\$Revision: +([0-9]+)")
135 print "WARNING: Revision not found!"
139 def findqt(cross
=""):
140 '''Search for Qt4 installation. Return path to qmake.'''
141 print "Searching for Qt"
142 bins
= [cross
+ "qmake", cross
+ "qmake-qt4"]
156 def checkqt(qmakebin
):
157 '''Check if given path to qmake exists and is a suitable version.'''
159 # check if binary exists
160 if not os
.path
.exists(qmakebin
):
161 print "Specified qmake path does not exist!"
164 output
= subprocess
.Popen([qmakebin
, "-version"], stdout
=subprocess
.PIPE
,
165 stderr
=subprocess
.PIPE
)
166 cmdout
= output
.communicate()
167 # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
169 r
= re
.compile("Qt[^0-9]+([0-9\.]+[a-z]*)")
172 print "Qt found: %s" % m
.group(1)
173 s
= re
.compile("4\..*")
174 n
= re
.search(s
, m
.group(1))
180 def qmake(qmake
, projfile
, platform
=sys
.platform
, wd
=".", static
=True, cross
=""):
181 print "Running qmake in %s..." % wd
182 command
= [qmake
, "-config", "release", "-config", "noccache"]
184 command
.extend(["-config", "-static"])
185 # special spec required?
186 if len(qmakespec
[platform
]) > 0:
187 command
.extend(["-spec", qmakespec
[platform
]])
188 # cross compiling prefix set?
190 command
.extend(["-config", "cross"])
191 command
.append(projfile
)
192 output
= subprocess
.Popen(command
, stdout
=subprocess
.PIPE
, cwd
=wd
)
194 if not output
.returncode
== 0:
195 print "qmake returned an error!"
200 def build(wd
=".", platform
=sys
.platform
, cross
=""):
203 # use the current platforms make here, cross compiling uses the native make.
204 command
= [make
[sys
.platform
]]
207 command
.append(str(cpus
))
208 output
= subprocess
.Popen(command
, stdout
=subprocess
.PIPE
, cwd
=wd
)
210 c
= output
.stdout
.readline()
211 sys
.stdout
.write(".")
213 if not output
.poll() == None:
214 sys
.stdout
.write("\n")
216 if not output
.returncode
== 0:
217 print "Build failed!"
220 if platform
!= "darwin":
221 # strip. OS X handles this via macdeployqt.
222 print "Stripping binary."
223 output
= subprocess
.Popen([cross
+ "strip", progexe
[platform
]], \
224 stdout
=subprocess
.PIPE
, cwd
=wd
)
226 if not output
.returncode
== 0:
227 print "Stripping failed!"
232 def upxfile(wd
=".", platform
=sys
.platform
):
234 print "UPX'ing binary ..."
235 output
= subprocess
.Popen(["upx", progexe
[platform
]], \
236 stdout
=subprocess
.PIPE
, cwd
=wd
)
238 if not output
.returncode
== 0:
239 print "UPX'ing failed!"
244 def runnsis(versionstring
, nsis
, script
, srcfolder
):
245 # run script through nsis to create installer.
246 print "Running NSIS ..."
248 # Assume the generated installer gets placed in the same folder the nsi
249 # script lives in. This seems to be a valid assumption unless the nsi
250 # script specifies a path. NSIS expects files relative to source folder so
251 # copy progexe. Additional files are injected into the nsis script.
253 # FIXME: instead of copying binaries around copy the NSI file and inject
255 # Only win32 supported as target platform so hard coded.
256 b
= srcfolder
+ "/" + os
.path
.dirname(script
) + "/" \
257 + os
.path
.dirname(progexe
["win32"])
258 if not os
.path
.exists(b
):
260 shutil
.copy(srcfolder
+ "/" + progexe
["win32"], b
)
261 output
= subprocess
.Popen([nsis
, srcfolder
+ "/" + script
], \
262 stdout
=subprocess
.PIPE
)
264 if not output
.returncode
== 0:
267 setupfile
= program
+ "-" + versionstring
+ "-setup.exe"
268 # find output filename in nsis script file
270 for line
in open(srcfolder
+ "/" + script
):
271 if re
.match(r
'^[^;]*OutFile\s+', line
) != None:
272 nsissetup
= re
.sub(r
'^[^;]*OutFile\s+"(.+)"', r
'\1', line
).rstrip()
274 print "Could not retrieve output file name!"
276 shutil
.copy(srcfolder
+ "/" + os
.path
.dirname(script
) + "/" + nsissetup
, \
281 def nsisfileinject(nsis
, outscript
, filelist
):
282 '''Inject files in filelist into NSIS script file after the File line
283 containing the main binary. This assumes that the main binary is present
284 in the NSIS script and that all additiona files (dlls etc) to get placed
286 output
= open(outscript
, "w")
287 for line
in open(nsis
, "r"):
289 # inject files after the progexe binary.
290 # Match the basename only to avoid path mismatches.
291 if re
.match(r
'^\s*File\s*.*' + os
.path
.basename(progexe
["win32"]), \
292 line
, re
.IGNORECASE
):
294 injection
= " File /oname=$INSTDIR\\" + os
.path
.basename(f
) \
295 + " " + os
.path
.normcase(f
) + "\n"
296 output
.write(injection
)
297 output
.write(" ; end of injected files\n")
301 def finddlls(program
, extrapaths
=[], cross
=""):
302 '''Check program for required DLLs. Find all required DLLs except ignored
303 ones and return a list of DLL filenames (including path).'''
304 # ask objdump about dependencies.
305 output
= subprocess
.Popen([cross
+ "objdump", "-x", program
], \
306 stdout
=subprocess
.PIPE
)
307 cmdout
= output
.communicate()
309 # create list of used DLLs. Store as lower case as W32 is case-insensitive.
311 for line
in cmdout
[0].split('\n'):
312 if re
.match(r
'\s*DLL Name', line
) != None:
313 dll
= re
.sub(r
'^\s*DLL Name:\s+([a-zA-Z_\-0-9\.\+]+).*$', r
'\1', line
)
314 dlls
.append(dll
.lower())
316 # find DLLs in extrapaths and PATH environment variable.
319 if file in systemdlls
:
320 print "System DLL: " + file
323 for path
in extrapaths
:
324 if os
.path
.exists(path
+ "/" + file):
325 dllpath
= re
.sub(r
"\\", r
"/", path
+ "/" + file)
326 print file + ": found at " + dllpath
327 dllpaths
.append(dllpath
)
331 dllpath
= re
.sub(r
"\\", r
"/", which(file))
332 print file + ": found at " + dllpath
333 dllpaths
.append(dllpath
)
335 print "MISSING DLL: " + file
339 def zipball(programfiles
, versionstring
, buildfolder
, platform
=sys
.platform
):
340 '''package created binary'''
341 print "Creating binary zipball."
342 archivebase
= program
+ "-" + versionstring
343 outfolder
= buildfolder
+ "/" + archivebase
344 archivename
= archivebase
+ ".zip"
345 # create output folder
347 # move program files to output folder
348 for f
in programfiles
:
349 if re
.match(r
'^(/|[a-zA-Z]:)', f
) != None:
350 shutil
.copy(f
, outfolder
)
352 shutil
.copy(buildfolder
+ "/" + f
, outfolder
)
353 # create zipball from output folder
354 zf
= zipfile
.ZipFile(archivename
, mode
='w', compression
=zipfile
.ZIP_DEFLATED
)
355 for root
, dirs
, files
in os
.walk(outfolder
):
357 physname
= os
.path
.normpath(os
.path
.join(root
, name
))
358 filename
= string
.replace(physname
, os
.path
.normpath(buildfolder
), "")
359 zf
.write(physname
, filename
)
361 physname
= os
.path
.normpath(os
.path
.join(root
, name
))
362 filename
= string
.replace(physname
, os
.path
.normpath(buildfolder
), "")
363 zf
.write(physname
, filename
)
365 # remove output folder
366 shutil
.rmtree(outfolder
)
370 def tarball(programfiles
, versionstring
, buildfolder
):
371 '''package created binary'''
372 print "Creating binary tarball."
373 archivebase
= program
+ "-" + versionstring
374 outfolder
= buildfolder
+ "/" + archivebase
375 archivename
= archivebase
+ ".tar.bz2"
376 # create output folder
378 # move program files to output folder
379 for f
in programfiles
:
380 shutil
.copy(buildfolder
+ "/" + f
, outfolder
)
381 # create tarball from output folder
382 tf
= tarfile
.open(archivename
, mode
='w:bz2')
383 tf
.add(outfolder
, archivebase
)
385 # remove output folder
386 shutil
.rmtree(outfolder
)
390 def macdeploy(versionstring
, buildfolder
, platform
=sys
.platform
):
391 '''package created binary to dmg'''
392 dmgfile
= program
+ "-" + versionstring
+ ".dmg"
393 appbundle
= buildfolder
+ "/" + progexe
[platform
]
395 # workaround to Qt issues when building out-of-tree. Copy files into bundle.
396 sourcebase
= buildfolder
+ re
.sub('[^/]+.pro$', '', project
) + "/"
398 for src
in bundlecopy
:
399 shutil
.copy(sourcebase
+ src
, appbundle
+ "/" + bundlecopy
[src
])
400 # end of Qt workaround
402 output
= subprocess
.Popen(["macdeployqt", progexe
[platform
], "-dmg"], \
403 stdout
=subprocess
.PIPE
, cwd
=buildfolder
)
405 if not output
.returncode
== 0:
406 print "macdeployqt failed!"
408 # copy dmg to output folder
409 shutil
.copy(buildfolder
+ "/" + program
+ ".dmg", dmgfile
)
413 def filehashes(filename
):
414 '''Calculate md5 and sha1 hashes for a given file.'''
415 if not os
.path
.exists(filename
):
419 f
= open(filename
, 'rb')
426 return [m
.hexdigest(), s
.hexdigest()]
429 def filestats(filename
):
430 if not os
.path
.exists(filename
):
432 st
= os
.stat(filename
)
433 print filename
, "\n", "-" * len(filename
)
434 print "Size: %i bytes" % st
.st_size
435 h
= filehashes(filename
)
436 print "md5sum: %s" % h
[0]
437 print "sha1sum: %s" % h
[1]
438 print "-" * len(filename
), "\n"
441 def tempclean(workfolder
, nopro
):
443 print "Cleaning up working folder %s" % workfolder
444 shutil
.rmtree(workfolder
)
446 print "Project file specified or cleanup disabled!"
447 print "Temporary files kept at %s" % workfolder
451 startup
= time
.time()
454 opts
, args
= getopt
.getopt(sys
.argv
[1:], "q:p:t:a:n:sbdkx:i:h",
455 ["qmake=", "project=", "tag=", "add=", "makensis=", "source-only",
456 "binary-only", "dynamic", "keep-temp", "cross=", "buildid=", "help"])
457 except getopt
.GetoptError
, err
:
463 svnbase
= svnserver
+ "trunk/"
473 platform
= sys
.platform
474 treehash
= gitscraper
.get_refs(gitrepo
)['refs/remotes/origin/HEAD']
475 if sys
.platform
!= "darwin":
480 if o
in ("-q", "--qmake"):
482 if o
in ("-p", "--project"):
485 if o
in ("-a", "--add"):
487 if o
in ("-n", "--makensis"):
489 if o
in ("-s", "--source-only"):
491 if o
in ("-b", "--binary-only"):
493 if o
in ("-d", "--dynamic") and sys
.platform
!= "darwin":
495 if o
in ("-k", "--keep-temp"):
497 if o
in ("-t", "--tree"):
499 if o
in ("-x", "--cross") and sys
.platform
!= "win32":
502 if o
in ("-i", "--buildid"):
504 if o
in ("-h", "--help"):
508 if source
== False and binary
== False:
509 print "Building build neither source nor binary means nothing to do. Exiting."
512 print "Building " + progexe
[platform
] + " for " + platform
519 print "ERROR: No suitable Qt installation found."
522 # create working folder. Use current directory if -p option used.
524 w
= tempfile
.mkdtemp()
525 # make sure the path doesn't contain backslashes to prevent issues
526 # later when running on windows.
527 workfolder
= re
.sub(r
'\\', '/', w
)
528 revision
= gitscraper
.describe_treehash(gitrepo
, treehash
)
529 # try to find a version number from describe output.
530 # WARNING: this is broken and just a temporary workaround!
531 v
= re
.findall('([\d\.a-f]+)', revision
)
533 if v
[-1].find('.') >= 0:
534 revision
= "v" + v
[-1]
540 versionextra
= "-" + buildid
541 sourcefolder
= workfolder
+ "/" + program
+ "-" + str(revision
) + versionextra
+ "/"
542 archivename
= program
+ "-" + str(revision
) + versionextra
+ "-src.tar.bz2"
544 os
.mkdir(sourcefolder
)
549 # check if project file explicitly given. If yes, don't get sources from svn
550 print "Version: %s" % revision
552 proj
= sourcefolder
+ project
553 # get sources and pack source tarball
554 if not getsources(treehash
, svnpaths
, sourcefolder
) == 0:
555 tempclean(workfolder
, cleanup
and not keeptemp
)
558 # replace version strings.
559 print "Updating version information in sources"
561 infile
= open(sourcefolder
+ "/" + f
, "r")
562 incontents
= infile
.readlines()
565 outfile
= open(sourcefolder
+ "/" + f
, "w")
566 for line
in incontents
:
568 for r
in regreplace
[f
]:
569 # replacements made on the replacement string:
570 # %REVISION% is replaced with the revision number
571 replacement
= re
.sub("%REVISION%", str(revision
), r
[1])
572 newline
= re
.sub(r
[0], replacement
, newline
)
573 # %BUILD% is replaced with buildid as passed on the command line
575 replacement
= re
.sub("%BUILDID%", "-" + str(buildid
), replacement
)
577 replacement
= re
.sub("%BUILDID%", "", replacement
)
578 newline
= re
.sub(r
[0], replacement
, newline
)
579 outfile
.write(newline
)
583 print "Creating source tarball %s\n" % archivename
584 tf
= tarfile
.open(archivename
, mode
='w:bz2')
585 tf
.add(sourcefolder
, os
.path
.basename(re
.subn('/$', '', sourcefolder
)[0]))
588 shutil
.rmtree(workfolder
)
591 # figure version from sources. Need to take path to project file into account.
592 versionfile
= re
.subn('[\w\.]+$', "version.h", proj
)[0]
593 ver
= findversion(versionfile
)
594 # append buildid if any.
599 if not os
.path
.exists(proj
):
600 print "ERROR: path to project file wrong."
603 # copy specified (--add) files to working folder
605 shutil
.copy(f
, sourcefolder
)
606 buildstart
= time
.time()
607 header
= "Building %s %s" % (program
, ver
)
609 print len(header
) * "="
612 if not qmake(qm
, proj
, platform
, sourcefolder
, static
, cross
) == 0:
613 tempclean(workfolder
, cleanup
and not keeptemp
)
615 if not build(sourcefolder
, platform
, cross
) == 0:
616 tempclean(workfolder
, cleanup
and not keeptemp
)
618 buildtime
= time
.time() - buildstart
619 progfiles
= programfiles
620 progfiles
.append(progexe
[platform
])
621 if platform
== "win32":
623 if not upxfile(sourcefolder
, platform
) == 0:
624 tempclean(workfolder
, cleanup
and not keeptemp
)
626 dllfiles
= finddlls(sourcefolder
+ "/" + progexe
[platform
], \
627 [os
.path
.dirname(qm
)], cross
)
628 if len(dllfiles
) > 0:
629 progfiles
.extend(dllfiles
)
630 archive
= zipball(progfiles
, ver
, sourcefolder
, platform
)
631 # only when running native right now.
632 if nsisscript
!= "" and makensis
!= "":
633 nsisfileinject(sourcefolder
+ "/" + nsisscript
, sourcefolder \
634 + "/" + nsisscript
+ ".tmp", dllfiles
)
635 runnsis(ver
, makensis
, nsisscript
+ ".tmp", sourcefolder
)
636 elif platform
== "darwin":
637 archive
= macdeploy(ver
, sourcefolder
, platform
)
639 if platform
== "linux2":
641 prog
= sourcefolder
+ "/" + p
642 output
= subprocess
.Popen(["file", prog
],
643 stdout
=subprocess
.PIPE
)
644 res
= output
.communicate()
645 if re
.findall("ELF 64-bit", res
[0]):
649 archive
= tarball(progfiles
, ver
, sourcefolder
)
651 # remove temporary files
652 tempclean(workfolder
, cleanup
)
655 headline
= "Build Summary for %s" % program
656 print "\n", headline
, "\n", "=" * len(headline
)
657 if not archivename
== "":
658 filestats(archivename
)
660 duration
= time
.time() - startup
661 durmins
= (int)(duration
/ 60)
662 dursecs
= (int)(duration
% 60)
663 buildmins
= (int)(buildtime
/ 60)
664 buildsecs
= (int)(buildtime
% 60)
665 print "Overall time %smin %ssec, building took %smin %ssec." % \
666 (durmins
, dursecs
, buildmins
, buildsecs
)
669 if __name__
== "__main__":
670 print "You cannot run this module directly!"
671 print "Set required environment and call deploy()."