Start development series 0.24.2-post
[0compile.git] / build.py
bloba6289e7e401db502465c151e5af79136d5c23194
1 # Copyright (C) 2006, Thomas Leonard
2 # See http://0install.net/0compile.html
4 import sys, os, __main__, time, shutil, glob, codecs, subprocess
5 from os.path import join
6 from logging import info, warn
7 from xml.dom import minidom, XMLNS_NAMESPACE
8 from optparse import OptionParser
9 import tempfile
11 from zeroinstall import SafeException
12 from zeroinstall.injector import model, namespaces
13 from zeroinstall.injector.iface_cache import iface_cache
15 from support import BuildEnv, ensure_dir, XMLNS_0COMPILE, is_package_impl, parse_bool, depth
16 from support import spawn_and_check, find_in_path, ENV_FILE, lookup, spawn_maybe_sandboxed, Prefixes
18 if hasattr(os.path, 'relpath'):
19 relpath = os.path.relpath
20 else:
21 # Copied from Python 2.6 (GPL compatible license)
22 # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation; All Rights Reserved
24 # Return the longest prefix of all list elements.
25 def commonprefix(m):
26 "Given a list of pathnames, returns the longest common leading component"
27 if not m: return ''
28 s1 = min(m)
29 s2 = max(m)
30 for i, c in enumerate(s1):
31 if c != s2[i]:
32 return s1[:i]
33 return s1
35 def relpath(path, start):
36 """Return a relative version of a path"""
38 if not path:
39 raise ValueError("no path specified")
41 start_list = os.path.abspath(start).split('/')
42 path_list = os.path.abspath(path).split('/')
44 # Work out how much of the filepath is shared by start and path.
45 i = len(commonprefix([start_list, path_list]))
47 rel_list = ['..'] * (len(start_list)-i) + path_list[i:]
48 if not rel_list:
49 return '.'
50 return join(*rel_list)
52 # If we have to modify any pkg-config files, we put the new versions in $TMPDIR/PKG_CONFIG_OVERRIDES
53 PKG_CONFIG_OVERRIDES = 'pkg-config-overrides'
55 def env(name, value):
56 os.environ[name] = value
57 print "%s=%s" % (name, value)
59 def do_env_binding(binding, path):
60 env(binding.name, binding.get_value(path, os.environ.get(binding.name, None)))
62 def correct_for_64bit(base, rel_path):
63 """If rel_path starts lib or usr/lib and doesn't exist, try with lib64 instead."""
64 if os.path.exists(os.path.join(base, rel_path)):
65 return rel_path
67 if rel_path.startswith('lib/') or rel_path.startswith('usr/lib/'):
68 new_rel_path = rel_path.replace('lib/', 'lib64/', 1)
69 if os.path.exists(os.path.join(base, new_rel_path)):
70 return new_rel_path
72 return rel_path
74 def write_pc(name, lines):
75 overrides_dir = os.path.join(os.environ['TMPDIR'], PKG_CONFIG_OVERRIDES)
76 if not os.path.isdir(overrides_dir):
77 os.mkdir(overrides_dir)
78 stream = open(os.path.join(overrides_dir, name), 'w')
79 stream.write(''.join(lines))
80 stream.close()
82 def do_pkg_config_binding(binding, impl):
83 feed_name = impl.feed.split('/')[-1]
84 path = lookup(impl)
85 new_insert = correct_for_64bit(path, binding.insert)
86 if new_insert != binding.insert:
87 print "PKG_CONFIG_PATH dir <%s>/%s not found; using %s instead" % (feed_name, binding.insert, new_insert)
88 binding = model.EnvironmentBinding(binding.name,
89 new_insert,
90 binding.default,
91 binding.mode)
93 orig_path = os.path.join(path, binding.insert)
94 if os.path.isdir(orig_path):
95 for pc in os.listdir(orig_path):
96 stream = open(os.path.join(orig_path, pc))
97 lines = stream.readlines()
98 stream.close()
99 for i, line in enumerate(lines):
100 if '=' not in line: continue
101 name, value = [x.strip() for x in line.split('=', 1)]
102 if name == 'prefix' and value.startswith('/'):
103 print "Absolute prefix=%s in %s; overriding..." % (value, feed_name)
104 lines[i] = 'prefix=%s/%s\n' % (path, value[1:])
105 write_pc(pc, lines)
106 break
107 do_env_binding(binding, path)
109 def fixup_generated_pkgconfig_file(pc_file):
110 stream = open(pc_file)
111 lines = stream.readlines()
112 stream.close()
113 for i, line in enumerate(lines):
114 if '=' not in line: continue
115 name, value = [x.strip() for x in line.split('=', 1)]
116 if name == 'prefix' and value.startswith('/'):
117 print "Absolute prefix=%s in %s; fixing..." % (value, pc_file)
118 rel_path = relpath(value, os.path.dirname(pc_file)) # Requires Python 2.6
119 lines[i] = 'prefix=${pcfiledir}/%s\n' % rel_path
120 write_pc(pc_file, lines)
121 break
123 # After doing a build, check that we didn't generate pkgconfig files with absolute paths
124 # Rewrite if so
125 def fixup_generated_pkgconfig_files():
126 for root, dirs, files in os.walk(os.environ['DISTDIR']):
127 if os.path.basename(root) == 'pkgconfig':
128 for f in files:
129 if f.endswith('.pc'):
130 info("Checking generated pkgconfig file '%s'", f)
131 fixup_generated_pkgconfig_file(os.path.join(root, f))
133 def remove_la_file(path):
134 # Read the contents...
135 stream = open(path)
136 data = stream.read()
137 stream.close()
139 # Check it really is a libtool archive...
140 if 'Please DO NOT delete this file' not in data:
141 warn("Ignoring %s; doesn't look like a libtool archive", path)
142 return
144 os.unlink(path)
145 print "Removed %s (.la files contain absolute paths)" % path
147 # libtool archives contain hard-coded paths. Lucky, modern systems don't need them, so remove
148 # them.
149 def remove_la_files():
150 for root, dirs, files in os.walk(os.environ['DISTDIR']):
151 if os.path.basename(root) == 'lib':
152 for f in files:
153 if f.endswith('.la'):
154 remove_la_file(os.path.join(root, f))
155 if f.endswith('.a'):
156 warn("Found static archive '%s'; maybe build with --disable-static?", f)
158 def do_build_internal(options, args):
159 """build-internal"""
160 # If a sandbox is being used, we're in it now.
161 import getpass, socket
163 buildenv = BuildEnv()
164 sels = buildenv.get_selections()
166 builddir = os.path.realpath('build')
167 ensure_dir(buildenv.metadir)
169 build_env_xml = join(buildenv.metadir, 'build-environment.xml')
171 buildenv_doc = sels.toDOM()
173 # Create build-environment.xml file
174 root = buildenv_doc.documentElement
175 info = buildenv_doc.createElementNS(XMLNS_0COMPILE, 'build-info')
176 root.appendChild(info)
177 info.setAttributeNS(None, 'time', time.strftime('%Y-%m-%d %H:%M').strip())
178 info.setAttributeNS(None, 'host', socket.getfqdn())
179 info.setAttributeNS(None, 'user', getpass.getuser())
180 uname = os.uname()
181 info.setAttributeNS(None, 'arch', '%s-%s' % (uname[0], uname[4]))
182 stream = file(build_env_xml, 'w')
183 buildenv_doc.writexml(stream, addindent=" ", newl="\n")
184 stream.close()
186 # Create local binary interface file
187 src_iface = iface_cache.get_interface(buildenv.interface)
188 src_impl = buildenv.chosen_impl(buildenv.interface)
189 write_sample_interface(buildenv, src_iface, src_impl)
191 # Check 0compile is new enough
192 min_version = model.parse_version(src_impl.attrs.get(XMLNS_0COMPILE + ' min-version', None))
193 if min_version and min_version > model.parse_version(__main__.version):
194 raise SafeException("%s-%s requires 0compile >= %s, but we are only version %s" %
195 (src_iface.get_name(), src_impl.version, model.format_version(min_version), __main__.version))
197 # Create the patch
198 patch_file = join(buildenv.metadir, 'from-%s.patch' % src_impl.version)
199 if buildenv.user_srcdir:
200 # (ignore errors; will already be shown on stderr)
201 os.system("diff -urN '%s' src > %s" %
202 (buildenv.orig_srcdir.replace('\\', '\\\\').replace("'", "\\'"),
203 patch_file))
204 if os.path.getsize(patch_file) == 0:
205 os.unlink(patch_file)
206 elif os.path.exists(patch_file):
207 os.unlink(patch_file)
209 env('BUILDDIR', builddir)
210 env('DISTDIR', buildenv.distdir)
211 env('SRCDIR', buildenv.user_srcdir or buildenv.orig_srcdir)
212 env('BINARYFEED', buildenv.local_iface_file)
213 os.chdir(builddir)
214 print "cd", builddir
216 for needed_iface in sels.selections:
217 impl = buildenv.chosen_impl(needed_iface)
218 assert impl
220 def process_bindings(bindings, dep_impl):
221 if dep_impl.id.startswith('package:'):
222 return
223 for b in bindings:
224 if isinstance(b, model.EnvironmentBinding):
225 if b.name == 'PKG_CONFIG_PATH':
226 do_pkg_config_binding(b, dep_impl)
227 else:
228 do_env_binding(b, lookup(dep_impl))
230 # Bindings that tell this component how to find itself...
231 process_bindings(impl.bindings, impl)
233 # Bindings that tell this component how to find its dependencies...
234 for dep in impl.dependencies:
235 dep_impl = buildenv.chosen_impl(dep.interface)
236 process_bindings(dep.bindings, dep_impl)
238 # These mappings are needed when mixing Zero Install -dev packages with
239 # native package binaries.
240 mappings = {}
241 for impl in sels.selections.values():
242 # Add mappings that have been set explicitly...
243 new_mappings = impl.attrs.get(XMLNS_0COMPILE + ' lib-mappings', '')
244 if new_mappings:
245 new_mappings = new_mappings.split(' ')
246 for mapping in new_mappings:
247 assert ':' in mapping, "lib-mappings missing ':' in '%s' from '%s'" % (mapping, impl.feed)
248 name, major_version = mapping.split(':', 1)
249 assert '/' not in mapping, "lib-mappings '%s' contains a / in the version number (from '%s')!" % (mapping, impl.feed)
250 if sys.platform == 'darwin':
251 mappings[name] = 'lib%s.%s.dylib' % (name, major_version)
252 else:
253 mappings[name] = 'lib%s.so.%s' % (name, major_version)
254 # Auto-detect required mappings where possible...
255 # (if the -dev package is native, the symlinks will be OK)
256 if not is_package_impl(impl):
257 impl_path = lookup(impl)
258 for libdirname in ['lib', 'usr/lib', 'lib64', 'usr/lib64']:
259 libdir = os.path.join(impl_path, libdirname)
260 if os.path.isdir(libdir):
261 find_broken_version_symlinks(libdir, mappings)
263 if mappings:
264 set_up_mappings(mappings)
266 overrides_dir = os.path.join(os.environ['TMPDIR'], PKG_CONFIG_OVERRIDES)
267 if os.path.isdir(overrides_dir):
268 add_overrides = model.EnvironmentBinding('PKG_CONFIG_PATH', PKG_CONFIG_OVERRIDES)
269 do_env_binding(add_overrides, os.environ['TMPDIR'])
271 # Some programs want to put temporary build files in the source directory.
272 # Make a copy of the source if needed.
273 dup_src_type = src_impl.attrs.get(XMLNS_0COMPILE + ' dup-src', None)
274 if dup_src_type == 'true':
275 dup_src(shutil.copy2)
276 env('SRCDIR', builddir)
277 elif dup_src_type:
278 raise Exception("Unknown dup-src value '%s'" % dup_src_type)
280 if options.shell:
281 spawn_and_check(find_in_path('sh'), [])
282 else:
283 compile_command = sels.commands[0]
284 if compile_command:
285 command = compile_command.qdom.attrs['shell-command']
286 else:
287 command = src_impl.attrs[XMLNS_0COMPILE + ' command']
289 # Remove any existing log files
290 for log in ['build.log', 'build-success.log', 'build-failure.log']:
291 if os.path.exists(log):
292 os.unlink(log)
294 # Run the command, copying output to a new log
295 log = file('build.log', 'w')
296 try:
297 print >>log, "Build log for %s-%s" % (src_iface.get_name(),
298 src_impl.version)
299 print >>log, "\nBuilt using 0compile-%s" % __main__.version
300 print >>log, "\nBuild system: " + ', '.join(uname)
301 print >>log, "\n%s:\n" % ENV_FILE
302 shutil.copyfileobj(file("../" + ENV_FILE), log)
304 log.write('\n')
306 if os.path.exists(patch_file):
307 print >>log, "\nPatched with:\n"
308 shutil.copyfileobj(file(patch_file), log)
309 log.write('\n')
311 print "Executing: " + command, args
312 print >>log, "Executing: " + command, args
314 # Tee the output to the console and to the log
315 child = subprocess.Popen(['/bin/sh', '-c', command + ' "$@"', '-'] + args, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
316 while True:
317 data = os.read(child.stdout.fileno(), 100)
318 if not data: break
319 sys.stdout.write(data)
320 log.write(data)
321 status = child.wait()
322 failure = None
323 if status == 0:
324 print >>log, "Build successful"
325 fixup_generated_pkgconfig_files()
326 remove_la_files()
327 elif status > 0:
328 failure = "Build failed with exit code %d" % status
329 else:
330 failure = "Build failure: exited due to signal %d" % (-status)
331 if failure:
332 print >>log, failure
333 os.rename('build.log', 'build-failure.log')
334 raise SafeException("Command '%s': %s" % (command, failure))
335 else:
336 os.rename('build.log', 'build-success.log')
337 finally:
338 log.close()
340 def do_build(args):
341 """build [ --no-sandbox ] [ --shell | --force | --clean ]"""
342 buildenv = BuildEnv()
343 sels = buildenv.get_selections()
345 parser = OptionParser(usage="usage: %prog build [options]")
347 parser.add_option('', "--no-sandbox", help="disable use of sandboxing", action='store_true')
348 parser.add_option("-s", "--shell", help="run a shell instead of building", action='store_true')
349 parser.add_option("-c", "--clean", help="remove the build directories", action='store_true')
350 parser.add_option("-f", "--force", help="build even if dependencies have changed", action='store_true')
352 parser.disable_interspersed_args()
354 (options, args2) = parser.parse_args(args)
356 builddir = os.path.realpath('build')
358 changes = buildenv.get_build_changes()
359 if changes:
360 if not (options.force or options.clean):
361 raise SafeException("Build dependencies have changed:\n" +
362 '\n'.join(changes) + "\n\n" +
363 "To build anyway, use: 0compile build --force\n" +
364 "To do a clean build: 0compile build --clean")
365 if not options.no_sandbox:
366 print "Build dependencies have changed:\n" + '\n'.join(changes)
368 ensure_dir(builddir, options.clean)
369 ensure_dir(buildenv.distdir, options.clean)
371 if options.no_sandbox:
372 return do_build_internal(options, args2)
374 tmpdir = tempfile.mkdtemp(prefix = '0compile-')
375 try:
376 my_dir = os.path.dirname(__file__)
377 readable = ['.', my_dir]
378 writable = ['build', buildenv.distdir, tmpdir]
379 env('TMPDIR', tmpdir)
381 for selection in sels.selections.values():
382 if not is_package_impl(selection):
383 readable.append(lookup(selection))
385 options = []
386 if __main__.options.verbose:
387 options.append('--verbose')
389 readable.append('/etc') # /etc/ld.*
391 spawn_maybe_sandboxed(readable, writable, tmpdir, sys.executable, ['-u', sys.argv[0]] + options + ['build', '--no-sandbox'] + args)
392 finally:
393 info("Deleting temporary directory '%s'" % tmpdir)
394 shutil.rmtree(tmpdir)
396 def write_sample_interface(buildenv, iface, src_impl):
397 path = buildenv.local_iface_file
399 impl = minidom.getDOMImplementation()
401 XMLNS_IFACE = namespaces.XMLNS_IFACE
403 doc = impl.createDocument(XMLNS_IFACE, "interface", None)
405 root = doc.documentElement
406 root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns', XMLNS_IFACE)
407 prefixes = Prefixes(XMLNS_IFACE)
409 def addSimple(parent, name, text = None):
410 elem = doc.createElementNS(XMLNS_IFACE, name)
412 parent.appendChild(doc.createTextNode('\n' + ' ' * (1 + depth(parent))))
413 parent.appendChild(elem)
414 if text:
415 elem.appendChild(doc.createTextNode(text))
416 return elem
418 def close(element):
419 element.appendChild(doc.createTextNode('\n' + ' ' * depth(element)))
421 addSimple(root, 'name', iface.name)
422 addSimple(root, 'summary', iface.summary)
423 addSimple(root, 'description', iface.description)
424 feed_for = addSimple(root, 'feed-for')
426 uri = iface.uri
427 if uri.startswith('/'):
428 print "Note: source %s is a local feed" % iface.uri
429 for feed_uri in iface.feed_for or []:
430 uri = feed_uri
431 print "Will use <feed-for interface='%s'> instead..." % uri
432 break
433 else:
434 master_feed = minidom.parse(uri).documentElement
435 if master_feed.hasAttribute('uri'):
436 uri = master_feed.getAttribute('uri')
437 print "Will use <feed-for interface='%s'> instead..." % uri
439 feed_for.setAttributeNS(None, 'interface', uri)
441 group = addSimple(root, 'group')
442 main = src_impl.attrs.get(XMLNS_0COMPILE + ' binary-main', None)
443 if main:
444 group.setAttributeNS(None, 'main', main)
446 lib_mappings = src_impl.attrs.get(XMLNS_0COMPILE + ' binary-lib-mappings', None)
447 if lib_mappings:
448 prefixes.setAttributeNS(group, XMLNS_0COMPILE, 'lib-mappings', lib_mappings)
450 for d in src_impl.dependencies:
451 # 0launch < 0.32 messed up the namespace...
452 if parse_bool(d.metadata.get('include-binary', 'false')) or \
453 parse_bool(d.metadata.get(XMLNS_0COMPILE + ' include-binary', 'false')):
454 requires = addSimple(group, 'requires')
455 requires.setAttributeNS(None, 'interface', d.interface)
456 for b in d.bindings:
457 if isinstance(b, model.EnvironmentBinding):
458 env_elem = addSimple(requires, 'environment')
459 env_elem.setAttributeNS(None, 'name', b.name)
460 env_elem.setAttributeNS(None, 'insert', b.insert)
461 if b.default:
462 env_elem.setAttributeNS(None, 'default', b.default)
463 else:
464 raise Exception('Unknown binding type ' + b)
465 close(requires)
467 set_arch = True
469 impl_elem = addSimple(group, 'implementation')
470 impl_template = buildenv.get_binary_template()
471 if impl_template:
472 # Copy attributes from template
473 for fullname, value in impl_template.attrs.iteritems():
474 if fullname == 'arch':
475 set_arch = False
476 if value == '*-*':
477 continue
478 if ' ' in fullname:
479 ns, localName = fullname.split(' ', 1)
480 else:
481 ns, localName = None, fullname
482 prefixes.setAttributeNS(impl_elem, ns, localName, value)
483 # Copy child nodes
484 for child in impl_template.childNodes:
485 impl_elem.appendChild(child.toDOM(doc, prefixes))
486 if impl_template.content:
487 impl_elem.appendChild(doc.createTextNode(impl_template.content))
489 if set_arch:
490 group.setAttributeNS(None, 'arch', buildenv.target_arch)
492 impl_elem.setAttributeNS(None, 'version', src_impl.version)
494 version_modifier = buildenv.version_modifier
495 if version_modifier:
496 impl_elem.setAttributeNS(None, 'version-modifier', version_modifier)
498 impl_elem.setAttributeNS(None, 'id', '..')
499 impl_elem.setAttributeNS(None, 'released', time.strftime('%Y-%m-%d'))
500 close(group)
501 close(root)
503 for ns, prefix in prefixes.prefixes.items():
504 root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns:' + prefix, ns)
506 stream = codecs.open(path, 'w', encoding = 'utf-8')
507 try:
508 doc.writexml(stream)
509 finally:
510 stream.close()
512 def find_broken_version_symlinks(libdir, mappings):
513 """libdir may be a legacy -devel package containing lib* symlinks whose
514 targets would be provided by the corresponding runtime package. If so,
515 create fixed symlinks under $TMPDIR with the real location."""
516 prefix = 'lib'
517 if sys.platform == 'darwin':
518 extension = '.dylib'
519 else:
520 extension = '.so'
522 for x in os.listdir(libdir):
523 if x.startswith(prefix) and x.endswith(extension):
524 path = os.path.join(libdir, x)
525 if os.path.islink(path):
526 target = os.readlink(path)
527 if '/' not in target and not os.path.exists(os.path.join(libdir, target)):
528 print "Broken link %s -> %s; will relocate..." % (x, target)
529 mappings[x[len(prefix):-len(extension)]] = target
531 def set_up_mappings(mappings):
532 """Create a temporary directory with symlinks for each of the library mappings."""
533 libdirs = []
534 if sys.platform == 'darwin':
535 LD_LIBRARY_PATH='DYLD_LIBRARY_PATH'
536 else:
537 LD_LIBRARY_PATH='LD_LIBRARY_PATH'
538 for d in os.environ.get(LD_LIBRARY_PATH, '').split(':'):
539 if d: libdirs.append(d)
540 libdirs += ['/lib', '/usr/lib']
542 def add_ldconf(config_file):
543 if not os.path.isfile(config_file):
544 return
545 for line in file(config_file):
546 d = line.strip()
547 if d.startswith('include '):
548 glob_pattern = d.split(' ', 1)[1]
549 for conf in glob.glob(glob_pattern):
550 add_ldconf(conf)
551 elif d and not d.startswith('#'):
552 libdirs.append(d)
553 add_ldconf('/etc/ld.so.conf')
555 def find_library(name, wanted):
556 # Takes a short-name and target name of a library and returns
557 # the full path of the library.
558 for d in libdirs:
559 path = os.path.join(d, wanted)
560 if os.path.exists(path):
561 return path
562 print "WARNING: library '%s' not found (searched '%s')!" % (wanted, libdirs)
563 return None
565 mappings_dir = os.path.join(os.environ['TMPDIR'], 'lib-mappings')
566 os.mkdir(mappings_dir)
568 old_path = os.environ.get('LIBRARY_PATH', '')
569 if old_path: old_path = ':' + old_path
570 os.environ['LIBRARY_PATH'] = mappings_dir + old_path
572 if sys.platform == 'darwin':
573 soext='.dylib'
574 else:
575 soext='.so'
576 for name, wanted in mappings.items():
577 target = find_library(name, wanted)
578 if target:
579 print "Adding mapping lib%s%s -> %s" % (name, soext, target)
580 os.symlink(target, os.path.join(mappings_dir, 'lib' + name + soext))
582 def dup_src(fn):
583 srcdir = os.environ['SRCDIR'] + '/'
584 for root, dirs, files in os.walk(srcdir):
585 assert root.startswith(srcdir)
586 reldir = root[len(srcdir):]
587 for f in files:
588 target = os.path.join(reldir, f)
589 #print "Copy %s -> %s" % (os.path.join(root, f), target)
590 if os.path.exists(target):
591 os.unlink(target)
592 fn(os.path.join(root, f), target)
593 for d in dirs:
594 target = os.path.join(reldir, d)
595 if not os.path.isdir(target):
596 os.mkdir(target)
598 __main__.commands.append(do_build)