build.py

   1 # Copyright (C) 2006, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import sys, os, __main__, time, shutil, glob, codecs, subprocess
   5 from os.path import join
   6 from logging import info, warn
   7 from xml.dom import minidom, XMLNS_NAMESPACE
   8 from optparse import OptionParser
   9
  10 from support import *
  11
  12 if hasattr(os.path, 'relpath'):
  13         relpath = os.path.relpath
  14 else:
  15         # Copied from Python 2.6 (GPL compatible license)
  16         # Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006 Python Software Foundation; All Rights Reserved
  17
  18         # Return the longest prefix of all list elements.
  19         def commonprefix(m):
  20                 "Given a list of pathnames, returns the longest common leading component"
  21                 if not m: return ''
  22                 s1 = min(m)
  23                 s2 = max(m)
  24                 for i, c in enumerate(s1):
  25                         if c != s2[i]:
  26                                 return s1[:i]
  27                 return s1
  28
  29         def relpath(path, start):
  30                 """Return a relative version of a path"""
  31
  32                 if not path:
  33                         raise ValueError("no path specified")
  34
  35                 start_list = os.path.abspath(start).split('/')
  36                 path_list = os.path.abspath(path).split('/')
  37
  38                 # Work out how much of the filepath is shared by start and path.
  39                 i = len(commonprefix([start_list, path_list]))
  40
  41                 rel_list = ['..'] * (len(start_list)-i) + path_list[i:]
  42                 if not rel_list:
  43                         return '.'
  44                 return join(*rel_list)
  45
  46 # If we have to modify any pkg-config files, we put the new versions in $TMPDIR/PKG_CONFIG_OVERRIDES
  47 PKG_CONFIG_OVERRIDES = 'pkg-config-overrides'
  48
  49 def env(name, value):
  50         os.environ[name] = value
  51         print "%s=%s" % (name, value)
  52
  53 def do_env_binding(binding, path):
  54         env(binding.name, binding.get_value(path, os.environ.get(binding.name, None)))
  55
  56 def correct_for_64bit(base, rel_path):
  57         """If rel_path starts lib or usr/lib and doesn't exist, try with lib64 instead."""
  58         if os.path.exists(os.path.join(base, rel_path)):
  59                 return rel_path
  60
  61         if rel_path.startswith('lib/') or rel_path.startswith('usr/lib/'):
  62                 new_rel_path = rel_path.replace('lib/', 'lib64/', 1)
  63                 if os.path.exists(os.path.join(base, new_rel_path)):
  64                         return new_rel_path
  65
  66         return rel_path
  67
  68 def write_pc(name, lines):
  69         overrides_dir = os.path.join(os.environ['TMPDIR'], PKG_CONFIG_OVERRIDES)
  70         if not os.path.isdir(overrides_dir):
  71                 os.mkdir(overrides_dir)
  72         stream = open(os.path.join(overrides_dir, name), 'w')
  73         stream.write(''.join(lines))
  74         stream.close()
  75
  76 def do_pkg_config_binding(binding, impl):
  77         feed_name = impl.feed.split('/')[-1]
  78         path = lookup(impl)
  79         new_insert = correct_for_64bit(path, binding.insert)
  80         if new_insert != binding.insert:
  81                 print "PKG_CONFIG_PATH dir <%s>/%s not found; using %s instead" % (feed_name, binding.insert, new_insert)
  82                 binding = model.EnvironmentBinding(binding.name,
  83                                         new_insert,
  84                                         binding.default,
  85                                         binding.mode)
  86
  87         orig_path = os.path.join(path, binding.insert)
  88         if os.path.isdir(orig_path):
  89                 for pc in os.listdir(orig_path):
  90                         stream = open(os.path.join(orig_path, pc))
  91                         lines = stream.readlines()
  92                         stream.close()
  93                         for i, line in enumerate(lines):
  94                                 if '=' not in line: continue
  95                                 name, value = [x.strip() for x in line.split('=', 1)]
  96                                 if name == 'prefix' and value.startswith('/'):
  97                                         print "Absolute prefix=%s in %s; overriding..." % (value, feed_name)
  98                                         lines[i] = 'prefix=%s/%s\n' % (path, value[1:])
  99                                         write_pc(pc, lines)
 100                                         break
 101         do_env_binding(binding, path)
 102
 103 def fixup_generated_pkgconfig_file(pc_file):
 104         stream = open(pc_file)
 105         lines = stream.readlines()
 106         stream.close()
 107         for i, line in enumerate(lines):
 108                 if '=' not in line: continue
 109                 name, value = [x.strip() for x in line.split('=', 1)]
 110                 if name == 'prefix' and value.startswith('/'):
 111                         print "Absolute prefix=%s in %s; fixing..." % (value, pc_file)
 112                         rel_path = relpath(value, os.path.dirname(pc_file))     # Requires Python 2.6
 113                         lines[i] = 'prefix=${pcfiledir}/%s\n' % rel_path
 114                         write_pc(pc_file, lines)
 115                         break
 116
 117 # After doing a build, check that we didn't generate pkgconfig files with absolute paths
 118 # Rewrite if so
 119 def fixup_generated_pkgconfig_files():
 120         for root, dirs, files in os.walk(os.environ['DISTDIR']):
 121                 if os.path.basename(root) == 'pkgconfig':
 122                         for f in files:
 123                                 if f.endswith('.pc'):
 124                                         info("Checking generated pkgconfig file '%s'", f)
 125                                         fixup_generated_pkgconfig_file(os.path.join(root, f))
 126
 127 def remove_la_file(path):
 128         # Read the contents...
 129         stream = open(path)
 130         data = stream.read()
 131         stream.close()
 132
 133         # Check it really is a libtool archive...
 134         if 'Please DO NOT delete this file' not in data:
 135                 warn("Ignoring %s; doesn't look like a libtool archive", path)
 136                 return
 137
 138         os.unlink(path)
 139         print "Removed %s (.la files contain absolute paths)" % path
 140
 141 # libtool archives contain hard-coded paths. Lucky, modern systems don't need them, so remove
 142 # them.
 143 def remove_la_files():
 144         for root, dirs, files in os.walk(os.environ['DISTDIR']):
 145                 if os.path.basename(root) == 'lib':
 146                         for f in files:
 147                                 if f.endswith('.la'):
 148                                         remove_la_file(os.path.join(root, f))
 149                                 if f.endswith('.a'):
 150                                         warn("Found static archive '%s'; maybe build with --disable-static?", f)
 151
 152 def do_build_internal(options, args):
 153         """build-internal"""
 154         # If a sandbox is being used, we're in it now.
 155         import getpass, socket, time
 156
 157         buildenv = BuildEnv()
 158         sels = buildenv.get_selections()
 159
 160         builddir = os.path.realpath('build')
 161         ensure_dir(buildenv.metadir)
 162
 163         build_env_xml = join(buildenv.metadir, 'build-environment.xml')
 164
 165         buildenv_doc = buildenv.get_selections().toDOM()
 166
 167         # Create build-environment.xml file
 168         root = buildenv_doc.documentElement
 169         info = buildenv_doc.createElementNS(XMLNS_0COMPILE, 'build-info')
 170         root.appendChild(info)
 171         info.setAttributeNS(None, 'time', time.strftime('%Y-%m-%d %H:%M').strip())
 172         info.setAttributeNS(None, 'host', socket.getfqdn())
 173         info.setAttributeNS(None, 'user', getpass.getuser())
 174         uname = os.uname()
 175         info.setAttributeNS(None, 'arch', '%s-%s' % (uname[0], uname[4]))
 176         stream = file(build_env_xml, 'w')
 177         buildenv_doc.writexml(stream, addindent="  ", newl="\n")
 178         stream.close()
 179
 180         # Create local binary interface file
 181         src_iface = iface_cache.get_interface(buildenv.interface)
 182         src_impl = buildenv.chosen_impl(buildenv.interface)
 183         write_sample_interface(buildenv, src_iface, src_impl)
 184
 185         # Check 0compile is new enough
 186         min_version = parse_version(src_impl.attrs.get(XMLNS_0COMPILE + ' min-version', None))
 187         if min_version and min_version > parse_version(__main__.version):
 188                 raise SafeException("%s-%s requires 0compile >= %s, but we are only version %s" %
 189                                 (src_iface.get_name(), src_impl.version, format_version(min_version), __main__.version))
 190
 191         # Create the patch
 192         patch_file = join(buildenv.metadir, 'from-%s.patch' % src_impl.version)
 193         if buildenv.user_srcdir:
 194                 # (ignore errors; will already be shown on stderr)
 195                 os.system("diff -urN '%s' src > %s" %
 196                         (buildenv.orig_srcdir.replace('\\', '\\\\').replace("'", "\\'"),
 197                          patch_file))
 198                 if os.path.getsize(patch_file) == 0:
 199                         os.unlink(patch_file)
 200         elif os.path.exists(patch_file):
 201                 os.unlink(patch_file)
 202
 203         env('BUILDDIR', builddir)
 204         env('DISTDIR', buildenv.distdir)
 205         env('SRCDIR', buildenv.user_srcdir or buildenv.orig_srcdir)
 206         os.chdir(builddir)
 207         print "cd", builddir
 208
 209         for needed_iface in sels.selections:
 210                 impl = buildenv.chosen_impl(needed_iface)
 211                 assert impl
 212
 213                 def process_bindings(bindings, dep_impl):
 214                         if dep_impl.id.startswith('package:'):
 215                                 return
 216                         for b in bindings:
 217                                 if isinstance(b, EnvironmentBinding):
 218                                         if b.name == 'PKG_CONFIG_PATH':
 219                                                 do_pkg_config_binding(b, dep_impl)
 220                                         else:
 221                                                 do_env_binding(b, lookup(dep_impl))
 222
 223                 # Bindings that tell this component how to find itself...
 224                 process_bindings(impl.bindings, impl)
 225
 226                 # Bindings that tell this component how to find its dependencies...
 227                 for dep in impl.dependencies:
 228                         dep_iface = sels.selections[dep.interface]
 229                         dep_impl = buildenv.chosen_impl(dep.interface)
 230                         process_bindings(dep.bindings, dep_impl)
 231
 232         # These mappings are needed when mixing Zero Install -dev packages with
 233         # native package binaries.
 234         mappings = {}
 235         for impl in sels.selections.values():
 236                 # Add mappings that have been set explicitly...
 237                 new_mappings = impl.attrs.get(XMLNS_0COMPILE + ' lib-mappings', '')
 238                 if new_mappings:
 239                         new_mappings = new_mappings.split(' ')
 240                         for mapping in new_mappings:
 241                                 assert ':' in mapping, "lib-mappings missing ':' in '%s' from '%s'" % (mapping, impl.feed)
 242                                 name, major_version = mapping.split(':', 1)
 243                                 assert '/' not in mapping, "lib-mappings '%s' contains a / in the version number (from '%s')!" % (mapping, impl.feed)
 244                                 mappings[name] = 'lib%s.so.%s' % (name, major_version)
 245                 # Auto-detect required mappings where possible...
 246                 # (if the -dev package is native, the symlinks will be OK)
 247                 if not is_package_impl(impl):
 248                         impl_path = lookup(impl)
 249                         for libdirname in ['lib', 'usr/lib', 'lib64', 'usr/lib64']:
 250                                 libdir = os.path.join(impl_path, libdirname)
 251                                 if os.path.isdir(libdir):
 252                                         find_broken_version_symlinks(libdir, mappings)
 253
 254         if mappings:
 255                 set_up_mappings(mappings)
 256
 257         overrides_dir = os.path.join(os.environ['TMPDIR'], PKG_CONFIG_OVERRIDES)
 258         if os.path.isdir(overrides_dir):
 259                 add_overrides = model.EnvironmentBinding('PKG_CONFIG_PATH', PKG_CONFIG_OVERRIDES)
 260                 do_env_binding(add_overrides, os.environ['TMPDIR'])
 261
 262         # Some programs want to put temporary build files in the source directory.
 263         # Make a copy of the source if needed.
 264         dup_src_type = src_impl.attrs.get(XMLNS_0COMPILE + ' dup-src', None)
 265         if dup_src_type == 'true':
 266                 dup_src(shutil.copy2)
 267                 env('SRCDIR', builddir)
 268         elif dup_src_type:
 269                 raise Exception("Unknown dup-src value '%s'" % dup_src_type)
 270
 271         if options.shell:
 272                 spawn_and_check(find_in_path('sh'), [])
 273         else:
 274                 command = src_impl.attrs[XMLNS_0COMPILE + ' command']
 275
 276                 # Remove any existing log files
 277                 for log in ['build.log', 'build-success.log', 'build-failure.log']:
 278                         if os.path.exists(log):
 279                                 os.unlink(log)
 280
 281                 # Run the command, copying output to a new log
 282                 log = file('build.log', 'w')
 283                 try:
 284                         print >>log, "Build log for %s-%s" % (src_iface.get_name(),
 285                                                               src_impl.version)
 286                         print >>log, "\nBuilt using 0compile-%s" % __main__.version
 287                         print >>log, "\nBuild system: " + ', '.join(uname)
 288                         print >>log, "\n%s:\n" % ENV_FILE
 289                         shutil.copyfileobj(file("../" + ENV_FILE), log)
 290
 291                         log.write('\n')
 292
 293                         if os.path.exists(patch_file):
 294                                 print >>log, "\nPatched with:\n"
 295                                 shutil.copyfileobj(file(patch_file), log)
 296                                 log.write('\n')
 297
 298                         print "Executing: " + command
 299                         print >>log, "Executing: " + command
 300
 301                         # Tee the output to the console and to the log
 302                         child = subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
 303                         while True:
 304                                 data = os.read(child.stdout.fileno(), 100)
 305                                 if not data: break
 306                                 sys.stdout.write(data)
 307                                 log.write(data)
 308                         status = child.wait()
 309                         failure = None
 310                         if status == 0:
 311                                 print >>log, "Build successful"
 312                                 fixup_generated_pkgconfig_files()
 313                                 remove_la_files()
 314                         elif status > 0:
 315                                 failure = "Build failed with exit code %d" % status
 316                         else:
 317                                 failure = "Build failure: exited due to signal %d" % (-status)
 318                         if failure:
 319                                 print >>log, failure
 320                                 os.rename('build.log', 'build-failure.log')
 321                                 raise SafeException("Command '%s': %s" % (command, failure))
 322                         else:
 323                                 os.rename('build.log', 'build-success.log')
 324                 finally:
 325                         log.close()
 326
 327 def do_build(args):
 328         """build [ --no-sandbox ] [ --shell | --force | --clean ]"""
 329         buildenv = BuildEnv()
 330         sels = buildenv.get_selections()
 331
 332         parser = OptionParser(usage="usage: %prog build [options]")
 333
 334         parser.add_option('', "--no-sandbox", help="disable use of sandboxing", action='store_true')
 335         parser.add_option("-s", "--shell", help="run a shell instead of building", action='store_true')
 336         parser.add_option("-c", "--clean", help="remove the build directories", action='store_true')
 337         parser.add_option("-f", "--force", help="build even if dependencies have changed", action='store_true')
 338
 339         parser.disable_interspersed_args()
 340
 341         (options, args2) = parser.parse_args(args)
 342
 343         builddir = os.path.realpath('build')
 344
 345         changes = buildenv.get_build_changes()
 346         if changes:
 347                 if not (options.force or options.clean):
 348                         raise SafeException("Build dependencies have changed:\n" +
 349                                         '\n'.join(changes) + "\n\n" +
 350                                         "To build anyway, use: 0compile build --force\n" +
 351                                         "To do a clean build:  0compile build --clean")
 352                 if not options.no_sandbox:
 353                         print "Build dependencies have changed:\n" + '\n'.join(changes)
 354
 355         ensure_dir(builddir, options.clean)
 356         ensure_dir(buildenv.distdir, options.clean)
 357
 358         if options.no_sandbox:
 359                 return do_build_internal(options, args2)
 360
 361         tmpdir = tempfile.mkdtemp(prefix = '0compile-')
 362         try:
 363                 my_dir = os.path.dirname(__file__)
 364                 readable = ['.', my_dir]
 365                 writable = ['build', buildenv.distdir, tmpdir]
 366                 env('TMPDIR', tmpdir)
 367
 368                 for selection in sels.selections.values():
 369                         if not is_package_impl(selection):
 370                                 readable.append(lookup(selection))
 371
 372                 options = []
 373                 if __main__.options.verbose:
 374                         options.append('--verbose')
 375
 376                 readable.append('/etc') # /etc/ld.*
 377
 378                 spawn_maybe_sandboxed(readable, writable, tmpdir, sys.executable, ['-u', sys.argv[0]] + options + ['build', '--no-sandbox'] + args)
 379         finally:
 380                 info("Deleting temporary directory '%s'" % tmpdir)
 381                 shutil.rmtree(tmpdir)
 382
 383 def write_sample_interface(buildenv, iface, src_impl):
 384         path = buildenv.local_iface_file
 385         target_arch = buildenv.target_arch
 386
 387         impl = minidom.getDOMImplementation()
 388
 389         XMLNS_IFACE = namespaces.XMLNS_IFACE
 390
 391         doc = impl.createDocument(XMLNS_IFACE, "interface", None)
 392
 393         root = doc.documentElement
 394         root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns', XMLNS_IFACE)
 395
 396         def addSimple(parent, name, text = None):
 397                 elem = doc.createElementNS(XMLNS_IFACE, name)
 398
 399                 parent.appendChild(doc.createTextNode('\n' + '  ' * (1 + depth(parent))))
 400                 parent.appendChild(elem)
 401                 if text:
 402                         elem.appendChild(doc.createTextNode(text))
 403                 return elem
 404
 405         def close(element):
 406                 element.appendChild(doc.createTextNode('\n' + '  ' * depth(element)))
 407
 408         addSimple(root, 'name', iface.name)
 409         addSimple(root, 'summary', iface.summary)
 410         addSimple(root, 'description', iface.description)
 411         feed_for = addSimple(root, 'feed-for')
 412
 413         uri = iface.uri
 414         if uri.startswith('/'):
 415                 print "Note: source %s is a local feed" % iface.uri
 416                 for feed_uri in iface.feed_for or []:
 417                         uri = feed_uri
 418                         print "Will use <feed-for interface='%s'> instead..." % uri
 419                         break
 420                 else:
 421                         master_feed = minidom.parse(uri).documentElement
 422                         if master_feed.hasAttribute('uri'):
 423                                 uri = master_feed.getAttribute('uri')
 424                                 print "Will use <feed-for interface='%s'> instead..." % uri
 425
 426         feed_for.setAttributeNS(None, 'interface', uri)
 427
 428         group = addSimple(root, 'group')
 429         main = src_impl.attrs.get(XMLNS_0COMPILE + ' binary-main', None)
 430         if main:
 431                 group.setAttributeNS(None, 'main', main)
 432
 433         lib_mappings = src_impl.attrs.get(XMLNS_0COMPILE + ' binary-lib-mappings', None)
 434         if lib_mappings:
 435                 root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns:compile', XMLNS_0COMPILE)
 436                 group.setAttributeNS(XMLNS_0COMPILE, 'compile:lib-mappings', lib_mappings)
 437
 438         for d in src_impl.dependencies:
 439                 # 0launch < 0.32 messed up the namespace...
 440                 if parse_bool(d.metadata.get('include-binary', 'false')) or \
 441                    parse_bool(d.metadata.get(XMLNS_0COMPILE + ' include-binary', 'false')):
 442                         requires = addSimple(group, 'requires')
 443                         requires.setAttributeNS(None, 'interface', d.interface)
 444                         for b in d.bindings:
 445                                 if isinstance(b, model.EnvironmentBinding):
 446                                         env_elem = addSimple(requires, 'environment')
 447                                         env_elem.setAttributeNS(None, 'name', b.name)
 448                                         env_elem.setAttributeNS(None, 'insert', b.insert)
 449                                         if b.default:
 450                                                 env_elem.setAttributeNS(None, 'default', b.default)
 451                                 else:
 452                                         raise Exception('Unknown binding type ' + b)
 453                         close(requires)
 454
 455         group.setAttributeNS(None, 'arch', target_arch)
 456         impl_elem = addSimple(group, 'implementation')
 457         impl_elem.setAttributeNS(None, 'version', src_impl.version)
 458
 459         version_modifier = buildenv.version_modifier
 460         if version_modifier:
 461                 impl_elem.setAttributeNS(None, 'version-modifier', version_modifier)
 462
 463         impl_elem.setAttributeNS(None, 'id', '..')
 464         impl_elem.setAttributeNS(None, 'released', time.strftime('%Y-%m-%d'))
 465         close(group)
 466         close(root)
 467
 468         stream = codecs.open(path, 'w', encoding = 'utf-8')
 469         try:
 470                 doc.writexml(stream)
 471         finally:
 472                 stream.close()
 473
 474 def find_broken_version_symlinks(libdir, mappings):
 475         """libdir may be a legacy -devel package containing lib* symlinks whose
 476         targets would be provided by the corresponding runtime package. If so,
 477         create fixed symlinks under $TMPDIR with the real location."""
 478         for x in os.listdir(libdir):
 479                 if x.startswith('lib') and x.endswith('.so'):
 480                         path = os.path.join(libdir, x)
 481                         if os.path.islink(path):
 482                                 target = os.readlink(path)
 483                                 if '/' not in target and not os.path.exists(os.path.join(libdir, target)):
 484                                         print "Broken link %s -> %s; will relocate..." % (x, target)
 485                                         mappings[x[3:-3]] = target
 486
 487 def set_up_mappings(mappings):
 488         """Create a temporary directory with symlinks for each of the library mappings."""
 489         libdirs = []
 490         for d in os.environ.get('LD_LIBRARY_PATH', '').split(':'):
 491                 if d: libdirs.append(d)
 492         libdirs += ['/lib', '/usr/lib']
 493
 494         def add_ldconf(config_file):
 495                 if not os.path.isfile(config_file):
 496                         return
 497                 for line in file(config_file):
 498                         d = line.strip()
 499                         if d.startswith('include '):
 500                                 glob_pattern = d.split(' ', 1)[1]
 501                                 for conf in glob.glob(glob_pattern):
 502                                         add_ldconf(conf)
 503                         elif d and not d.startswith('#'):
 504                                 libdirs.append(d)
 505         add_ldconf('/etc/ld.so.conf')
 506
 507         def find_library(name, wanted):
 508                 # Takes a short-name and target name of a library and returns
 509                 # the full path of the library.
 510                 for d in libdirs:
 511                         path = os.path.join(d, wanted)
 512                         if os.path.exists(path):
 513                                 return path
 514                 print "WARNING: library '%s' not found (searched '%s')!" % (wanted, libdirs)
 515                 return None
 516
 517         mappings_dir = os.path.join(os.environ['TMPDIR'], 'lib-mappings')
 518         os.mkdir(mappings_dir)
 519
 520         old_path = os.environ.get('LIBRARY_PATH', '')
 521         if old_path: old_path = ':' + old_path
 522         os.environ['LIBRARY_PATH'] = mappings_dir + old_path
 523
 524         for name, wanted in mappings.items():
 525                 target = find_library(name, wanted)
 526                 if target:
 527                         print "Adding mapping lib%s.so -> %s" % (name, target)
 528                         os.symlink(target, os.path.join(mappings_dir, 'lib' + name + '.so'))
 529
 530 def dup_src(fn):
 531         srcdir = os.environ['SRCDIR'] + '/'
 532         for root, dirs, files in os.walk(srcdir):
 533                 assert root.startswith(srcdir)
 534                 reldir = root[len(srcdir):]
 535                 for f in files:
 536                         target = os.path.join(reldir, f)
 537                         #print "Copy %s -> %s" % (os.path.join(root, f), target)
 538                         if os.path.exists(target):
 539                                 os.unlink(target)
 540                         fn(os.path.join(root, f), target)
 541                 for d in dirs:
 542                         target = os.path.join(reldir, d)
 543                         if not os.path.isdir(target):
 544                                 os.mkdir(target)
 545
 546 __main__.commands.append(do_build)