1 # Copyright (C) 2006, Thomas Leonard
2 # See http://0install.net/0compile.html
4 import sys
, os
, __main__
, time
, shutil
, glob
, codecs
, subprocess
5 from os
.path
import join
6 from logging
import info
, warn
7 from xml
.dom
import minidom
, XMLNS_NAMESPACE
8 from optparse
import OptionParser
11 from zeroinstall
import SafeException
12 from zeroinstall
.injector
import model
, namespaces
13 from zeroinstall
.injector
.iface_cache
import iface_cache
15 from support
import BuildEnv
, ensure_dir
, XMLNS_0COMPILE
, is_package_impl
, parse_bool
, depth
16 from support
import spawn_and_check
, find_in_path
, ENV_FILE
, lookup
, spawn_maybe_sandboxed
, Prefixes
18 if hasattr(os
.path
, 'relpath'):
19 relpath
= os
.path
.relpath
21 # Copied from Python 2.6 (GPL compatible license)
22 # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation; All Rights Reserved
24 # Return the longest prefix of all list elements.
26 "Given a list of pathnames, returns the longest common leading component"
30 for i
, c
in enumerate(s1
):
35 def relpath(path
, start
):
36 """Return a relative version of a path"""
39 raise ValueError("no path specified")
41 start_list
= os
.path
.abspath(start
).split('/')
42 path_list
= os
.path
.abspath(path
).split('/')
44 # Work out how much of the filepath is shared by start and path.
45 i
= len(commonprefix([start_list
, path_list
]))
47 rel_list
= ['..'] * (len(start_list
)-i
) + path_list
[i
:]
50 return join(*rel_list
)
52 # If we have to modify any pkg-config files, we put the new versions in $TMPDIR/PKG_CONFIG_OVERRIDES
53 PKG_CONFIG_OVERRIDES
= 'pkg-config-overrides'
56 os
.environ
[name
] = value
57 print "%s=%s" % (name
, value
)
59 def do_env_binding(binding
, path
):
60 env(binding
.name
, binding
.get_value(path
, os
.environ
.get(binding
.name
, None)))
62 def correct_for_64bit(base
, rel_path
):
63 """If rel_path starts lib or usr/lib and doesn't exist, try with lib64 instead."""
64 if os
.path
.exists(os
.path
.join(base
, rel_path
)):
67 if rel_path
.startswith('lib/') or rel_path
.startswith('usr/lib/'):
68 new_rel_path
= rel_path
.replace('lib/', 'lib64/', 1)
69 if os
.path
.exists(os
.path
.join(base
, new_rel_path
)):
74 def write_pc(name
, lines
):
75 overrides_dir
= os
.path
.join(os
.environ
['TMPDIR'], PKG_CONFIG_OVERRIDES
)
76 if not os
.path
.isdir(overrides_dir
):
77 os
.mkdir(overrides_dir
)
78 stream
= open(os
.path
.join(overrides_dir
, name
), 'w')
79 stream
.write(''.join(lines
))
82 def do_pkg_config_binding(binding
, impl
):
83 feed_name
= impl
.feed
.split('/')[-1]
85 new_insert
= correct_for_64bit(path
, binding
.insert
)
86 if new_insert
!= binding
.insert
:
87 print "PKG_CONFIG_PATH dir <%s>/%s not found; using %s instead" % (feed_name
, binding
.insert
, new_insert
)
88 binding
= model
.EnvironmentBinding(binding
.name
,
93 orig_path
= os
.path
.join(path
, binding
.insert
)
94 if os
.path
.isdir(orig_path
):
95 for pc
in os
.listdir(orig_path
):
96 stream
= open(os
.path
.join(orig_path
, pc
))
97 lines
= stream
.readlines()
99 for i
, line
in enumerate(lines
):
100 if '=' not in line
: continue
101 name
, value
= [x
.strip() for x
in line
.split('=', 1)]
102 if name
== 'prefix' and value
.startswith('/'):
103 print "Absolute prefix=%s in %s; overriding..." % (value
, feed_name
)
104 lines
[i
] = 'prefix=%s/%s\n' % (path
, value
[1:])
107 do_env_binding(binding
, path
)
109 def fixup_generated_pkgconfig_file(pc_file
):
110 stream
= open(pc_file
)
111 lines
= stream
.readlines()
113 for i
, line
in enumerate(lines
):
114 if '=' not in line
: continue
115 name
, value
= [x
.strip() for x
in line
.split('=', 1)]
116 if name
== 'prefix' and value
.startswith('/'):
117 print "Absolute prefix=%s in %s; fixing..." % (value
, pc_file
)
118 rel_path
= relpath(value
, os
.path
.dirname(pc_file
)) # Requires Python 2.6
119 lines
[i
] = 'prefix=${pcfiledir}/%s\n' % rel_path
120 write_pc(pc_file
, lines
)
123 # After doing a build, check that we didn't generate pkgconfig files with absolute paths
125 def fixup_generated_pkgconfig_files():
126 for root
, dirs
, files
in os
.walk(os
.environ
['DISTDIR']):
127 if os
.path
.basename(root
) == 'pkgconfig':
129 if f
.endswith('.pc'):
130 info("Checking generated pkgconfig file '%s'", f
)
131 fixup_generated_pkgconfig_file(os
.path
.join(root
, f
))
133 def remove_la_file(path
):
134 # Read the contents...
139 # Check it really is a libtool archive...
140 if 'Please DO NOT delete this file' not in data
:
141 warn("Ignoring %s; doesn't look like a libtool archive", path
)
145 print "Removed %s (.la files contain absolute paths)" % path
147 # libtool archives contain hard-coded paths. Lucky, modern systems don't need them, so remove
149 def remove_la_files():
150 for root
, dirs
, files
in os
.walk(os
.environ
['DISTDIR']):
151 if os
.path
.basename(root
) == 'lib':
153 if f
.endswith('.la'):
154 remove_la_file(os
.path
.join(root
, f
))
156 warn("Found static archive '%s'; maybe build with --disable-static?", f
)
158 def do_build_internal(options
, args
):
160 # If a sandbox is being used, we're in it now.
161 import getpass
, socket
163 buildenv
= BuildEnv()
164 sels
= buildenv
.get_selections()
166 builddir
= os
.path
.realpath('build')
167 ensure_dir(buildenv
.metadir
)
169 build_env_xml
= join(buildenv
.metadir
, 'build-environment.xml')
171 buildenv_doc
= sels
.toDOM()
173 # Create build-environment.xml file
174 root
= buildenv_doc
.documentElement
175 info
= buildenv_doc
.createElementNS(XMLNS_0COMPILE
, 'build-info')
176 root
.appendChild(info
)
177 info
.setAttributeNS(None, 'time', time
.strftime('%Y-%m-%d %H:%M').strip())
178 info
.setAttributeNS(None, 'host', socket
.getfqdn())
179 info
.setAttributeNS(None, 'user', getpass
.getuser())
181 info
.setAttributeNS(None, 'arch', '%s-%s' % (uname
[0], uname
[4]))
182 stream
= file(build_env_xml
, 'w')
183 buildenv_doc
.writexml(stream
, addindent
=" ", newl
="\n")
186 # Create local binary interface file
187 src_iface
= iface_cache
.get_interface(buildenv
.interface
)
188 src_impl
= buildenv
.chosen_impl(buildenv
.interface
)
189 write_sample_interface(buildenv
, src_iface
, src_impl
)
191 # Check 0compile is new enough
192 min_version
= model
.parse_version(src_impl
.attrs
.get(XMLNS_0COMPILE
+ ' min-version', None))
193 if min_version
and min_version
> model
.parse_version(__main__
.version
):
194 raise SafeException("%s-%s requires 0compile >= %s, but we are only version %s" %
195 (src_iface
.get_name(), src_impl
.version
, model
.format_version(min_version
), __main__
.version
))
198 patch_file
= join(buildenv
.metadir
, 'from-%s.patch' % src_impl
.version
)
199 if buildenv
.user_srcdir
:
200 # (ignore errors; will already be shown on stderr)
201 os
.system("diff -urN '%s' src > %s" %
202 (buildenv
.orig_srcdir
.replace('\\', '\\\\').replace("'", "\\'"),
204 if os
.path
.getsize(patch_file
) == 0:
205 os
.unlink(patch_file
)
206 elif os
.path
.exists(patch_file
):
207 os
.unlink(patch_file
)
209 env('BUILDDIR', builddir
)
210 env('DISTDIR', buildenv
.distdir
)
211 env('SRCDIR', buildenv
.user_srcdir
or buildenv
.orig_srcdir
)
212 env('BINARYFEED', buildenv
.local_iface_file
)
216 for needed_iface
in sels
.selections
:
217 impl
= buildenv
.chosen_impl(needed_iface
)
220 def process_bindings(bindings
, dep_impl
):
221 if dep_impl
.id.startswith('package:'):
224 if isinstance(b
, model
.EnvironmentBinding
):
225 if b
.name
== 'PKG_CONFIG_PATH':
226 do_pkg_config_binding(b
, dep_impl
)
228 do_env_binding(b
, lookup(dep_impl
))
230 # Bindings that tell this component how to find itself...
231 process_bindings(impl
.bindings
, impl
)
233 # Bindings that tell this component how to find its dependencies...
234 for dep
in impl
.dependencies
:
235 dep_impl
= buildenv
.chosen_impl(dep
.interface
)
236 process_bindings(dep
.bindings
, dep_impl
)
238 # These mappings are needed when mixing Zero Install -dev packages with
239 # native package binaries.
241 for impl
in sels
.selections
.values():
242 # Add mappings that have been set explicitly...
243 new_mappings
= impl
.attrs
.get(XMLNS_0COMPILE
+ ' lib-mappings', '')
245 new_mappings
= new_mappings
.split(' ')
246 for mapping
in new_mappings
:
247 assert ':' in mapping
, "lib-mappings missing ':' in '%s' from '%s'" % (mapping
, impl
.feed
)
248 name
, major_version
= mapping
.split(':', 1)
249 assert '/' not in mapping
, "lib-mappings '%s' contains a / in the version number (from '%s')!" % (mapping
, impl
.feed
)
250 if sys
.platform
== 'darwin':
251 mappings
[name
] = 'lib%s.%s.dylib' % (name
, major_version
)
253 mappings
[name
] = 'lib%s.so.%s' % (name
, major_version
)
254 # Auto-detect required mappings where possible...
255 # (if the -dev package is native, the symlinks will be OK)
256 if not is_package_impl(impl
):
257 impl_path
= lookup(impl
)
258 for libdirname
in ['lib', 'usr/lib', 'lib64', 'usr/lib64']:
259 libdir
= os
.path
.join(impl_path
, libdirname
)
260 if os
.path
.isdir(libdir
):
261 find_broken_version_symlinks(libdir
, mappings
)
264 set_up_mappings(mappings
)
266 overrides_dir
= os
.path
.join(os
.environ
['TMPDIR'], PKG_CONFIG_OVERRIDES
)
267 if os
.path
.isdir(overrides_dir
):
268 add_overrides
= model
.EnvironmentBinding('PKG_CONFIG_PATH', PKG_CONFIG_OVERRIDES
)
269 do_env_binding(add_overrides
, os
.environ
['TMPDIR'])
271 # Some programs want to put temporary build files in the source directory.
272 # Make a copy of the source if needed.
273 dup_src_type
= src_impl
.attrs
.get(XMLNS_0COMPILE
+ ' dup-src', None)
274 if dup_src_type
== 'true':
275 dup_src(shutil
.copy2
)
276 env('SRCDIR', builddir
)
278 raise Exception("Unknown dup-src value '%s'" % dup_src_type
)
281 spawn_and_check(find_in_path('sh'), [])
283 compile_command
= sels
.commands
[0]
285 command
= compile_command
.qdom
.attrs
['shell-command']
287 command
= src_impl
.attrs
[XMLNS_0COMPILE
+ ' command']
289 # Remove any existing log files
290 for log
in ['build.log', 'build-success.log', 'build-failure.log']:
291 if os
.path
.exists(log
):
294 # Run the command, copying output to a new log
295 log
= file('build.log', 'w')
297 print >>log
, "Build log for %s-%s" % (src_iface
.get_name(),
299 print >>log
, "\nBuilt using 0compile-%s" % __main__
.version
300 print >>log
, "\nBuild system: " + ', '.join(uname
)
301 print >>log
, "\n%s:\n" % ENV_FILE
302 shutil
.copyfileobj(file("../" + ENV_FILE
), log
)
306 if os
.path
.exists(patch_file
):
307 print >>log
, "\nPatched with:\n"
308 shutil
.copyfileobj(file(patch_file
), log
)
311 print "Executing: " + command
, args
312 print >>log
, "Executing: " + command
, args
314 # Tee the output to the console and to the log
315 child
= subprocess
.Popen(['/bin/sh', '-c', command
+ ' "$@"', '-'] + args
, stdout
= subprocess
.PIPE
, stderr
= subprocess
.STDOUT
)
317 data
= os
.read(child
.stdout
.fileno(), 100)
319 sys
.stdout
.write(data
)
321 status
= child
.wait()
324 print >>log
, "Build successful"
325 fixup_generated_pkgconfig_files()
328 failure
= "Build failed with exit code %d" % status
330 failure
= "Build failure: exited due to signal %d" % (-status
)
333 os
.rename('build.log', 'build-failure.log')
334 raise SafeException("Command '%s': %s" % (command
, failure
))
336 os
.rename('build.log', 'build-success.log')
341 """build [ --no-sandbox ] [ --shell | --force | --clean ]"""
342 buildenv
= BuildEnv()
343 sels
= buildenv
.get_selections()
345 parser
= OptionParser(usage
="usage: %prog build [options]")
347 parser
.add_option('', "--no-sandbox", help="disable use of sandboxing", action
='store_true')
348 parser
.add_option("-s", "--shell", help="run a shell instead of building", action
='store_true')
349 parser
.add_option("-c", "--clean", help="remove the build directories", action
='store_true')
350 parser
.add_option("-f", "--force", help="build even if dependencies have changed", action
='store_true')
352 parser
.disable_interspersed_args()
354 (options
, args2
) = parser
.parse_args(args
)
356 builddir
= os
.path
.realpath('build')
358 changes
= buildenv
.get_build_changes()
360 if not (options
.force
or options
.clean
):
361 raise SafeException("Build dependencies have changed:\n" +
362 '\n'.join(changes
) + "\n\n" +
363 "To build anyway, use: 0compile build --force\n" +
364 "To do a clean build: 0compile build --clean")
365 if not options
.no_sandbox
:
366 print "Build dependencies have changed:\n" + '\n'.join(changes
)
368 ensure_dir(builddir
, options
.clean
)
369 ensure_dir(buildenv
.distdir
, options
.clean
)
371 if options
.no_sandbox
:
372 return do_build_internal(options
, args2
)
374 tmpdir
= tempfile
.mkdtemp(prefix
= '0compile-')
376 my_dir
= os
.path
.dirname(__file__
)
377 readable
= ['.', my_dir
]
378 writable
= ['build', buildenv
.distdir
, tmpdir
]
379 env('TMPDIR', tmpdir
)
381 for selection
in sels
.selections
.values():
382 if not is_package_impl(selection
):
383 readable
.append(lookup(selection
))
386 if __main__
.options
.verbose
:
387 options
.append('--verbose')
389 readable
.append('/etc') # /etc/ld.*
391 spawn_maybe_sandboxed(readable
, writable
, tmpdir
, sys
.executable
, ['-u', sys
.argv
[0]] + options
+ ['build', '--no-sandbox'] + args
)
393 info("Deleting temporary directory '%s'" % tmpdir
)
394 shutil
.rmtree(tmpdir
)
396 def write_sample_interface(buildenv
, iface
, src_impl
):
397 path
= buildenv
.local_iface_file
399 impl
= minidom
.getDOMImplementation()
401 XMLNS_IFACE
= namespaces
.XMLNS_IFACE
403 doc
= impl
.createDocument(XMLNS_IFACE
, "interface", None)
405 root
= doc
.documentElement
406 root
.setAttributeNS(XMLNS_NAMESPACE
, 'xmlns', XMLNS_IFACE
)
407 prefixes
= Prefixes(XMLNS_IFACE
)
409 def addSimple(parent
, name
, text
= None):
410 elem
= doc
.createElementNS(XMLNS_IFACE
, name
)
412 parent
.appendChild(doc
.createTextNode('\n' + ' ' * (1 + depth(parent
))))
413 parent
.appendChild(elem
)
415 elem
.appendChild(doc
.createTextNode(text
))
419 element
.appendChild(doc
.createTextNode('\n' + ' ' * depth(element
)))
421 addSimple(root
, 'name', iface
.name
)
422 addSimple(root
, 'summary', iface
.summary
)
423 addSimple(root
, 'description', iface
.description
)
424 feed_for
= addSimple(root
, 'feed-for')
427 if uri
.startswith('/'):
428 print "Note: source %s is a local feed" % iface
.uri
429 for feed_uri
in iface
.feed_for
or []:
431 print "Will use <feed-for interface='%s'> instead..." % uri
434 master_feed
= minidom
.parse(uri
).documentElement
435 if master_feed
.hasAttribute('uri'):
436 uri
= master_feed
.getAttribute('uri')
437 print "Will use <feed-for interface='%s'> instead..." % uri
439 feed_for
.setAttributeNS(None, 'interface', uri
)
441 group
= addSimple(root
, 'group')
442 main
= src_impl
.attrs
.get(XMLNS_0COMPILE
+ ' binary-main', None)
444 group
.setAttributeNS(None, 'main', main
)
446 lib_mappings
= src_impl
.attrs
.get(XMLNS_0COMPILE
+ ' binary-lib-mappings', None)
448 prefixes
.setAttributeNS(group
, XMLNS_0COMPILE
, 'lib-mappings', lib_mappings
)
450 for d
in src_impl
.dependencies
:
451 # 0launch < 0.32 messed up the namespace...
452 if parse_bool(d
.metadata
.get('include-binary', 'false')) or \
453 parse_bool(d
.metadata
.get(XMLNS_0COMPILE
+ ' include-binary', 'false')):
454 requires
= addSimple(group
, 'requires')
455 requires
.setAttributeNS(None, 'interface', d
.interface
)
457 if isinstance(b
, model
.EnvironmentBinding
):
458 env_elem
= addSimple(requires
, 'environment')
459 env_elem
.setAttributeNS(None, 'name', b
.name
)
460 env_elem
.setAttributeNS(None, 'insert', b
.insert
)
462 env_elem
.setAttributeNS(None, 'default', b
.default
)
464 raise Exception('Unknown binding type ' + b
)
469 impl_elem
= addSimple(group
, 'implementation')
470 impl_template
= buildenv
.get_binary_template()
472 # Copy attributes from template
473 for fullname
, value
in impl_template
.attrs
.iteritems():
474 if fullname
== 'arch':
479 ns
, localName
= fullname
.split(' ', 1)
481 ns
, localName
= None, fullname
482 prefixes
.setAttributeNS(impl_elem
, ns
, localName
, value
)
484 for child
in impl_template
.childNodes
:
485 impl_elem
.appendChild(child
.toDOM(doc
, prefixes
))
486 if impl_template
.content
:
487 impl_elem
.appendChild(doc
.createTextNode(impl_template
.content
))
490 group
.setAttributeNS(None, 'arch', buildenv
.target_arch
)
492 impl_elem
.setAttributeNS(None, 'version', src_impl
.version
)
494 version_modifier
= buildenv
.version_modifier
496 impl_elem
.setAttributeNS(None, 'version-modifier', version_modifier
)
498 impl_elem
.setAttributeNS(None, 'id', '..')
499 impl_elem
.setAttributeNS(None, 'released', time
.strftime('%Y-%m-%d'))
503 for ns
, prefix
in prefixes
.prefixes
.items():
504 root
.setAttributeNS(XMLNS_NAMESPACE
, 'xmlns:' + prefix
, ns
)
506 stream
= codecs
.open(path
, 'w', encoding
= 'utf-8')
512 def find_broken_version_symlinks(libdir
, mappings
):
513 """libdir may be a legacy -devel package containing lib* symlinks whose
514 targets would be provided by the corresponding runtime package. If so,
515 create fixed symlinks under $TMPDIR with the real location."""
517 if sys
.platform
== 'darwin':
522 for x
in os
.listdir(libdir
):
523 if x
.startswith(prefix
) and x
.endswith(extension
):
524 path
= os
.path
.join(libdir
, x
)
525 if os
.path
.islink(path
):
526 target
= os
.readlink(path
)
527 if '/' not in target
and not os
.path
.exists(os
.path
.join(libdir
, target
)):
528 print "Broken link %s -> %s; will relocate..." % (x
, target
)
529 mappings
[x
[len(prefix
):-len(extension
)]] = target
531 def set_up_mappings(mappings
):
532 """Create a temporary directory with symlinks for each of the library mappings."""
534 if sys
.platform
== 'darwin':
535 LD_LIBRARY_PATH
='DYLD_LIBRARY_PATH'
537 LD_LIBRARY_PATH
='LD_LIBRARY_PATH'
538 for d
in os
.environ
.get(LD_LIBRARY_PATH
, '').split(':'):
539 if d
: libdirs
.append(d
)
540 libdirs
+= ['/lib', '/usr/lib']
542 def add_ldconf(config_file
):
543 if not os
.path
.isfile(config_file
):
545 for line
in file(config_file
):
547 if d
.startswith('include '):
548 glob_pattern
= d
.split(' ', 1)[1]
549 for conf
in glob
.glob(glob_pattern
):
551 elif d
and not d
.startswith('#'):
553 add_ldconf('/etc/ld.so.conf')
555 def find_library(name
, wanted
):
556 # Takes a short-name and target name of a library and returns
557 # the full path of the library.
559 path
= os
.path
.join(d
, wanted
)
560 if os
.path
.exists(path
):
562 print "WARNING: library '%s' not found (searched '%s')!" % (wanted
, libdirs
)
565 mappings_dir
= os
.path
.join(os
.environ
['TMPDIR'], 'lib-mappings')
566 os
.mkdir(mappings_dir
)
568 old_path
= os
.environ
.get('LIBRARY_PATH', '')
569 if old_path
: old_path
= ':' + old_path
570 os
.environ
['LIBRARY_PATH'] = mappings_dir
+ old_path
572 if sys
.platform
== 'darwin':
576 for name
, wanted
in mappings
.items():
577 target
= find_library(name
, wanted
)
579 print "Adding mapping lib%s%s -> %s" % (name
, soext
, target
)
580 os
.symlink(target
, os
.path
.join(mappings_dir
, 'lib' + name
+ soext
))
583 srcdir
= os
.environ
['SRCDIR'] + '/'
584 for root
, dirs
, files
in os
.walk(srcdir
):
585 assert root
.startswith(srcdir
)
586 reldir
= root
[len(srcdir
):]
588 target
= os
.path
.join(reldir
, f
)
589 #print "Copy %s -> %s" % (os.path.join(root, f), target)
590 if os
.path
.exists(target
):
592 fn(os
.path
.join(root
, f
), target
)
594 target
= os
.path
.join(reldir
, d
)
595 if not os
.path
.isdir(target
):
598 __main__
.commands
.append(do_build
)