build.py

   1 # Copyright (C) 2006, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import sys, os, __main__, time, shutil, glob, codecs, subprocess
   5 from os.path import join
   6 from logging import info
   7 from xml.dom import minidom, XMLNS_NAMESPACE
   8 from optparse import OptionParser
   9
  10 from support import *
  11
  12 def env(name, value):
  13         os.environ[name] = value
  14         print "%s=%s" % (name, value)
  15
  16 def do_env_binding(binding, path):
  17         env(binding.name, binding.get_value(path, os.environ.get(binding.name, None)))
  18
  19 def do_build_internal(options, args):
  20         """build-internal"""
  21         # If a sandbox is being used, we're in it now.
  22         import getpass, socket, time
  23
  24         buildenv = BuildEnv()
  25         sels = buildenv.get_selections()
  26
  27         builddir = os.path.realpath('build')
  28         ensure_dir(buildenv.metadir)
  29
  30         build_env_xml = join(buildenv.metadir, 'build-environment.xml')
  31
  32         buildenv_doc = buildenv.get_selections().toDOM()
  33
  34         # Create build-environment.xml file
  35         root = buildenv_doc.documentElement
  36         info = buildenv_doc.createElementNS(XMLNS_0COMPILE, 'build-info')
  37         root.appendChild(info)
  38         info.setAttributeNS(None, 'time', time.strftime('%Y-%m-%d %H:%M').strip())
  39         info.setAttributeNS(None, 'host', socket.getfqdn())
  40         info.setAttributeNS(None, 'user', getpass.getuser())
  41         uname = os.uname()
  42         info.setAttributeNS(None, 'arch', '%s-%s' % (uname[0], uname[4]))
  43         stream = file(build_env_xml, 'w')
  44         buildenv_doc.writexml(stream, addindent="  ", newl="\n")
  45         stream.close()
  46
  47         # Create local binary interface file
  48         src_iface = iface_cache.get_interface(buildenv.interface)
  49         src_impl = buildenv.chosen_impl(buildenv.interface)
  50         write_sample_interface(buildenv, src_iface, src_impl)
  51
  52         # Check 0compile is new enough
  53         min_version = parse_version(src_impl.attrs.get(XMLNS_0COMPILE + ' min-version', None))
  54         if min_version and min_version > parse_version(__main__.version):
  55                 raise SafeException("%s-%s requires 0compile >= %s, but we are only version %s" %
  56                                 (src_iface.get_name(), src_impl.version, format_version(min_version), __main__.version))
  57
  58         # Create the patch
  59         patch_file = join(buildenv.metadir, 'from-%s.patch' % src_impl.version)
  60         if buildenv.user_srcdir:
  61                 # (ignore errors; will already be shown on stderr)
  62                 os.system("diff -urN '%s' src > %s" %
  63                         (buildenv.orig_srcdir.replace('\\', '\\\\').replace("'", "\\'"),
  64                          patch_file))
  65                 if os.path.getsize(patch_file) == 0:
  66                         os.unlink(patch_file)
  67         elif os.path.exists(patch_file):
  68                 os.unlink(patch_file)
  69
  70         env('BUILDDIR', builddir)
  71         env('DISTDIR', buildenv.distdir)
  72         env('SRCDIR', buildenv.user_srcdir or buildenv.orig_srcdir)
  73         os.chdir(builddir)
  74         print "cd", builddir
  75
  76         for needed_iface in sels.selections:
  77                 impl = buildenv.chosen_impl(needed_iface)
  78                 assert impl
  79                 for dep in impl.dependencies:
  80                         dep_iface = sels.selections[dep.interface]
  81                         for b in dep.bindings:
  82                                 if isinstance(b, EnvironmentBinding):
  83                                         dep_impl = buildenv.chosen_impl(dep.interface)
  84                                         do_env_binding(b, lookup(dep_impl.id))
  85
  86         mappings = []
  87         for impl in sels.selections.values():
  88                 new_mappings = impl.attrs.get(XMLNS_0COMPILE + ' lib-mappings', '')
  89                 if new_mappings:
  90                         new_mappings = new_mappings.split(' ')
  91                         for mapping in new_mappings:
  92                                 assert ':' in mapping, "lib-mappings missing ':' in '%s' from '%s'" % (mapping, impl.feed)
  93                                 name, major_version = mapping.split(':', 1)
  94                                 assert '/' not in mapping, "lib-mappings '%s' contains a / in the version number (from '%s')!" % (mapping, impl.feed)
  95                                 mappings.append((name, major_version))
  96
  97         if mappings:
  98                 set_up_mappings(mappings)
  99
 100         # Some programs want to put temporary build files in the source directory.
 101         # Make a copy of the source if needed.
 102         dup_src_type = src_impl.attrs.get(XMLNS_0COMPILE + ' dup-src', None)
 103         if dup_src_type == 'true':
 104                 dup_src(shutil.copy2)
 105                 env('SRCDIR', builddir)
 106         elif dup_src_type:
 107                 raise Exception("Unknown dup-src value '%s'" % dup_src_type)
 108
 109         if options.shell:
 110                 spawn_and_check(find_in_path('sh'), [])
 111         else:
 112                 command = src_impl.attrs[XMLNS_0COMPILE + ' command']
 113
 114                 # Remove any existing log files
 115                 for log in ['build.log', 'build-success.log', 'build-failure.log']:
 116                         if os.path.exists(log):
 117                                 os.unlink(log)
 118
 119                 # Run the command, copying output to a new log
 120                 log = file('build.log', 'w')
 121                 try:
 122                         print >>log, "Build log for %s-%s" % (src_iface.get_name(),
 123                                                               src_impl.version)
 124                         print >>log, "\nBuilt using 0compile-%s" % __main__.version
 125                         print >>log, "\nBuild system: " + ', '.join(uname)
 126                         print >>log, "\n%s:\n" % ENV_FILE
 127                         shutil.copyfileobj(file("../" + ENV_FILE), log)
 128
 129                         log.write('\n')
 130
 131                         if os.path.exists(patch_file):
 132                                 print >>log, "\nPatched with:\n"
 133                                 shutil.copyfileobj(file(patch_file), log)
 134                                 log.write('\n')
 135
 136                         print "Executing: " + command
 137                         print >>log, "Executing: " + command
 138
 139                         # Tee the output to the console and to the log
 140                         child = subprocess.Popen(command, shell = True, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
 141                         while True:
 142                                 data = os.read(child.stdout.fileno(), 100)
 143                                 if not data: break
 144                                 sys.stdout.write(data)
 145                                 log.write(data)
 146                         status = child.wait()
 147                         failure = None
 148                         if status == 0:
 149                                 print >>log, "Build successful"
 150                         elif status > 0:
 151                                 failure = "Build failed with exit code %d" % status
 152                         else:
 153                                 failure = "Build failure: exited due to signal %d" % (-status)
 154                         if failure:
 155                                 print >>log, failure
 156                                 os.rename('build.log', 'build-failure.log')
 157                                 raise SafeException("Command '%s': %s" % (command, failure))
 158                         else:
 159                                 os.rename('build.log', 'build-success.log')
 160                 finally:
 161                         log.close()
 162
 163 def do_build(args):
 164         """build [ --no-sandbox ] [ --shell | --force | --clean ]"""
 165         buildenv = BuildEnv()
 166         sels = buildenv.get_selections()
 167
 168         parser = OptionParser(usage="usage: %prog build [options]")
 169
 170         parser.add_option('', "--no-sandbox", help="disable use of sandboxing", action='store_true')
 171         parser.add_option("-s", "--shell", help="run a shell instead of building", action='store_true')
 172         parser.add_option("-c", "--clean", help="remove the build directories", action='store_true')
 173         parser.add_option("-f", "--force", help="build even if dependencies have changed", action='store_true')
 174
 175         parser.disable_interspersed_args()
 176
 177         (options, args2) = parser.parse_args(args)
 178
 179         builddir = os.path.realpath('build')
 180
 181         changes = buildenv.get_build_changes()
 182         if changes:
 183                 if not (options.force or options.clean):
 184                         raise SafeException("Build dependencies have changed:\n" +
 185                                         '\n'.join(changes) + "\n\n" +
 186                                         "To build anyway, use: 0compile build --force\n" +
 187                                         "To do a clean build:  0compile build --clean")
 188                 if not options.no_sandbox:
 189                         print "Build dependencies have changed:\n" + '\n'.join(changes)
 190
 191         ensure_dir(builddir, options.clean)
 192         ensure_dir(buildenv.distdir, options.clean)
 193
 194         if options.no_sandbox:
 195                 return do_build_internal(options, args2)
 196
 197         tmpdir = tempfile.mkdtemp(prefix = '0compile-')
 198         try:
 199                 my_dir = os.path.dirname(__file__)
 200                 readable = ['.', my_dir]
 201                 writable = ['build', buildenv.distdir, tmpdir]
 202                 env('TMPDIR', tmpdir)
 203
 204                 for selection in sels.selections.values():
 205                         readable.append(lookup(selection.id))
 206
 207                 options = []
 208                 if __main__.options.verbose:
 209                         options.append('--verbose')
 210
 211                 readable.append('/etc') # /etc/ld.*
 212
 213                 spawn_maybe_sandboxed(readable, writable, tmpdir, sys.executable, [sys.argv[0]] + options + ['build', '--no-sandbox'] + args)
 214         finally:
 215                 info("Deleting temporary directory '%s'" % tmpdir)
 216                 shutil.rmtree(tmpdir)
 217
 218 def write_sample_interface(buildenv, iface, src_impl):
 219         path = buildenv.local_iface_file
 220         target_arch = buildenv.target_arch
 221
 222         impl = minidom.getDOMImplementation()
 223
 224         XMLNS_IFACE = namespaces.XMLNS_IFACE
 225
 226         doc = impl.createDocument(XMLNS_IFACE, "interface", None)
 227
 228         root = doc.documentElement
 229         root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns', XMLNS_IFACE)
 230
 231         def addSimple(parent, name, text = None):
 232                 elem = doc.createElementNS(XMLNS_IFACE, name)
 233
 234                 parent.appendChild(doc.createTextNode('\n' + '  ' * (1 + depth(parent))))
 235                 parent.appendChild(elem)
 236                 if text:
 237                         elem.appendChild(doc.createTextNode(text))
 238                 return elem
 239
 240         def close(element):
 241                 element.appendChild(doc.createTextNode('\n' + '  ' * depth(element)))
 242
 243         addSimple(root, 'name', iface.name)
 244         addSimple(root, 'summary', iface.summary)
 245         addSimple(root, 'description', iface.description)
 246         feed_for = addSimple(root, 'feed-for')
 247
 248         uri = iface.uri
 249         if uri.startswith('/') and iface.feed_for:
 250                 for uri in iface.feed_for:
 251                         print "Note: source %s is a local feed" % iface.uri
 252                         print "Will use <feed-for interface='%s'> instead..." % uri
 253                         break
 254
 255         feed_for.setAttributeNS(None, 'interface', uri)
 256
 257         group = addSimple(root, 'group')
 258         main = src_impl.attrs.get(XMLNS_0COMPILE + ' binary-main', None)
 259         if main:
 260                 group.setAttributeNS(None, 'main', main)
 261
 262         lib_mappings = src_impl.attrs.get(XMLNS_0COMPILE + ' binary-lib-mappings', None)
 263         if lib_mappings:
 264                 root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns:compile', XMLNS_0COMPILE)
 265                 group.setAttributeNS(XMLNS_0COMPILE, 'compile:lib-mappings', lib_mappings)
 266
 267         for d in src_impl.dependencies:
 268                 # 0launch < 0.32 messed up the namespace...
 269                 if parse_bool(d.metadata.get('include-binary', 'false')) or \
 270                    parse_bool(d.metadata.get(XMLNS_0COMPILE + ' include-binary', 'false')):
 271                         requires = addSimple(group, 'requires')
 272                         requires.setAttributeNS(None, 'interface', d.interface)
 273                         for b in d.bindings:
 274                                 if isinstance(b, model.EnvironmentBinding):
 275                                         env_elem = addSimple(requires, 'environment')
 276                                         env_elem.setAttributeNS(None, 'name', b.name)
 277                                         env_elem.setAttributeNS(None, 'insert', b.insert)
 278                                         if b.default:
 279                                                 env_elem.setAttributeNS(None, 'default', b.default)
 280                                 else:
 281                                         raise Exception('Unknown binding type ' + b)
 282                         close(requires)
 283
 284         group.setAttributeNS(None, 'arch', target_arch)
 285         impl_elem = addSimple(group, 'implementation')
 286         impl_elem.setAttributeNS(None, 'version', src_impl.version)
 287
 288         version_modifier = buildenv.version_modifier
 289         if version_modifier:
 290                 impl_elem.setAttributeNS(None, 'version-modifier', version_modifier)
 291
 292         impl_elem.setAttributeNS(None, 'id', '..')
 293         impl_elem.setAttributeNS(None, 'released', time.strftime('%Y-%m-%d'))
 294         close(group)
 295         close(root)
 296
 297         stream = codecs.open(path, 'w', encoding = 'utf-8')
 298         try:
 299                 doc.writexml(stream)
 300         finally:
 301                 stream.close()
 302
 303 def set_up_mappings(mappings):
 304         """Create a temporary directory with symlinks for each of the library mappings."""
 305         # The find_library function takes a short-name and major version of a library and
 306         # returns the full path of the library.
 307         libdirs = ['/lib', '/usr/lib']
 308         for d in os.environ.get('LD_LIBRARY_PATH', '').split(':'):
 309                 if d: libdirs.append(d)
 310
 311         def add_ldconf(config_file):
 312                 if not os.path.isfile(config_file):
 313                         return
 314                 for line in file(config_file):
 315                         d = line.strip()
 316                         if d.startswith('include '):
 317                                 glob_pattern = d.split(' ', 1)[1]
 318                                 for conf in glob.glob(glob_pattern):
 319                                         add_ldconf(conf)
 320                         elif d and not d.startswith('#'):
 321                                 libdirs.append(d)
 322         add_ldconf('/etc/ld.so.conf')
 323
 324         def find_library(name, major):
 325                 wanted = 'lib%s.so.%s' % (name, major)
 326                 for d in libdirs:
 327                         path = os.path.join(d, wanted)
 328                         if os.path.exists(path):
 329                                 return path
 330                 print "WARNING: library '%s' not found (searched '%s')!" % (wanted, libdirs)
 331                 return None
 332
 333         mappings_dir = os.path.join(os.environ['TMPDIR'], 'lib-mappings')
 334         os.mkdir(mappings_dir)
 335
 336         old_path = os.environ.get('LIBRARY_PATH', '')
 337         if old_path: old_path = ':' + old_path
 338         os.environ['LIBRARY_PATH'] = mappings_dir + old_path
 339
 340         for name, major_version in mappings:
 341                 target = find_library(name, major_version)
 342                 if target:
 343                         print "Adding mapping lib%s.so -> %s" % (name, target)
 344                         os.symlink(target, os.path.join(mappings_dir, 'lib' + name + '.so'))
 345
 346 def dup_src(fn):
 347         srcdir = os.environ['SRCDIR'] + '/'
 348         for root, dirs, files in os.walk(srcdir):
 349                 assert root.startswith(srcdir)
 350                 reldir = root[len(srcdir):]
 351                 for f in files:
 352                         target = os.path.join(reldir, f)
 353                         #print "Copy %s -> %s" % (os.path.join(root, f), target)
 354                         if os.path.exists(target):
 355                                 os.unlink(target)
 356                         fn(os.path.join(root, f), target)
 357                 for d in dirs:
 358                         target = os.path.join(reldir, d)
 359                         if not os.path.isdir(target):
 360                                 os.mkdir(target)
 361
 362 __main__.commands.append(do_build)