build.py

   1 # Copyright (C) 2006, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import sys, os, __main__, time, shutil, glob
   5 from os.path import join
   6 from logging import info
   7 from xml.dom import minidom, XMLNS_NAMESPACE
   8
   9 from support import *
  10
  11 def env(name, value):
  12         info('Setting %s="%s"', name, value)
  13         os.environ[name] = value
  14
  15 def do_env_binding(binding, path):
  16         os.environ[binding.name] = binding.get_value(path,
  17                                         os.environ.get(binding.name, None))
  18         info("%s=%s", binding.name, os.environ[binding.name])
  19
  20 def do_build_internal(args):
  21         """build-internal"""
  22         import getpass, socket, time
  23
  24         buildenv = BuildEnv()
  25
  26         builddir = os.path.realpath('build')
  27         ensure_dir(buildenv.metadir)
  28
  29         build_env_xml = join(buildenv.metadir, 'build-environment.xml')
  30
  31         buildenv_doc = buildenv.selections.toDOM()
  32
  33         # Create build-environment.xml file
  34         root = buildenv_doc.documentElement
  35         info = buildenv_doc.createElementNS(XMLNS_0COMPILE, 'build-info')
  36         root.appendChild(info)
  37         info.setAttributeNS(None, 'time', time.strftime('%Y-%m-%d %H:%M').strip())
  38         info.setAttributeNS(None, 'host', socket.getfqdn())
  39         info.setAttributeNS(None, 'user', getpass.getuser())
  40         uname = os.uname()
  41         info.setAttributeNS(None, 'arch', '%s-%s' % (uname[0], uname[4]))
  42         stream = file(build_env_xml, 'w')
  43         buildenv_doc.writexml(stream, addindent="  ", newl="\n")
  44         stream.close()
  45
  46         # Create local binary interface file
  47         src_iface = iface_cache.get_interface(buildenv.interface)
  48         src_impl = buildenv.chosen_impl(buildenv.interface)
  49         write_sample_interface(buildenv, src_iface, src_impl)
  50
  51         # Create the patch
  52         orig_impl = buildenv.chosen_impl(buildenv.interface)
  53         patch_file = join(buildenv.metadir, 'from-%s.patch' % orig_impl.version)
  54         if os.path.isdir('src'):
  55                 orig_src = lookup(orig_impl.id)
  56                 # (ignore errors; will already be shown on stderr)
  57                 os.system("diff -urN '%s' src > %s" %
  58                         (orig_src.replace('\\', '\\\\').replace("'", "\\'"),
  59                          patch_file))
  60                 if os.path.getsize(patch_file) == 0:
  61                         os.unlink(patch_file)
  62         elif os.path.exists(patch_file):
  63                 os.unlink(patch_file)
  64
  65         env('BUILDDIR', builddir)
  66         env('DISTDIR', buildenv.distdir)
  67         env('SRCDIR', buildenv.srcdir)
  68         os.chdir(builddir)
  69
  70         for needed_iface in buildenv.selections.selections:
  71                 impl = buildenv.chosen_impl(needed_iface)
  72                 assert impl
  73                 for dep in impl.dependencies:
  74                         dep_iface = buildenv.selections.selections[dep.interface]
  75                         for b in dep.bindings:
  76                                 if isinstance(b, EnvironmentBinding):
  77                                         dep_impl = buildenv.chosen_impl(dep.interface)
  78                                         do_env_binding(b, lookup(dep_impl.id))
  79
  80         mappings = []
  81         for impl in buildenv.selections.selections.values():
  82                 new_mappings = impl.attrs.get(XMLNS_0COMPILE + ' lib-mappings', '')
  83                 if new_mappings:
  84                         new_mappings = new_mappings.split(' ')
  85                         for mapping in new_mappings:
  86                                 assert ':' in mapping, "lib-mappings missing ':' in '%s' from '%s'" % (mapping, impl.feed)
  87                                 name, major_version = mapping.split(':', 1)
  88                                 assert '/' not in mapping, "lib-mappings '%s' contains a / in the version number (from '%s')!" % (mapping, impl.feed)
  89                                 mappings.append((name, major_version))
  90
  91         if mappings:
  92                 set_up_mappings(mappings)
  93
  94         if args == ['--shell']:
  95                 spawn_and_check(find_in_path('sh'), [])
  96         else:
  97                 command = buildenv.doc.getAttribute(XMLNS_0COMPILE + ' command')
  98
  99                 # Remove any existing log files
 100                 for log in ['build.log', 'build-success.log', 'build-failure.log']:
 101                         if os.path.exists(log):
 102                                 os.unlink(log)
 103
 104                 # Run the command, copying output to a new log
 105                 log = file('build.log', 'w')
 106                 try:
 107                         print >>log, "Build log for %s-%s" % (src_iface.get_name(),
 108                                                               src_impl.version)
 109                         print >>log, "\nBuilt using 0compile-%s" % __main__.version
 110                         print >>log, "\nBuild system: " + ', '.join(uname)
 111                         print >>log, "\n%s:\n" % ENV_FILE
 112                         shutil.copyfileobj(file("../" + ENV_FILE), log)
 113
 114                         log.write('\n')
 115
 116                         if os.path.exists(patch_file):
 117                                 print >>log, "\nPatched with:\n"
 118                                 shutil.copyfileobj(file(patch_file), log)
 119                                 log.write('\n')
 120
 121                         print "Executing: " + command
 122                         print >>log, "Executing: " + command
 123
 124                         # Tee the output to the console and to the log
 125                         from popen2 import Popen4
 126                         child = Popen4(command)
 127                         child.tochild.close()
 128                         while True:
 129                                 data = os.read(child.fromchild.fileno(), 100)
 130                                 if not data: break
 131                                 sys.stdout.write(data)
 132                                 log.write(data)
 133                         status = child.wait()
 134                         failure = None
 135                         if os.WIFEXITED(status):
 136                                 exit_code = os.WEXITSTATUS(status)
 137                                 if exit_code == 0:
 138                                         print >>log, "Build successful"
 139                                 else:
 140                                         failure = "Build failed with exit code %d" % exit_code
 141                         else:
 142                                 failure = "Build failure: exited due to signal %d" % os.WTERMSIG(status)
 143                         if failure:
 144                                 print >>log, failure
 145                                 os.rename('build.log', 'build-failure.log')
 146                                 raise SafeException("Command '%s': %s" % (command, failure))
 147                         else:
 148                                 os.rename('build.log', 'build-success.log')
 149                 finally:
 150                         log.close()
 151
 152 def do_build(args):
 153         """build [ --nosandbox ] [ --shell ]"""
 154         buildenv = BuildEnv()
 155
 156         builddir = os.path.realpath('build')
 157
 158         ensure_dir(builddir)
 159         ensure_dir(buildenv.distdir)
 160
 161         if args[:1] == ['--nosandbox']:
 162                 return do_build_internal(args[1:])
 163
 164         tmpdir = tempfile.mkdtemp(prefix = '0compile-')
 165         try:
 166                 my_dir = os.path.dirname(__file__)
 167                 readable = ['.', my_dir]
 168                 writable = ['build', buildenv.distdir, tmpdir]
 169                 env('TMPDIR', tmpdir)
 170
 171                 # Why did we need this?
 172                 #readable.append(get_cached_iface_path(buildenv.interface))
 173
 174                 for selection in buildenv.selections.selections.values():
 175                         readable.append(lookup(selection.id))
 176
 177                 options = []
 178                 if __main__.options.verbose:
 179                         options.append('--verbose')
 180
 181                 readable.append('/etc') # /etc/ld.*
 182
 183                 spawn_maybe_sandboxed(readable, writable, tmpdir, sys.executable, [sys.argv[0]] + options + ['build', '--nosandbox'] + args)
 184         finally:
 185                 info("Deleting temporary directory '%s'" % tmpdir)
 186                 shutil.rmtree(tmpdir)
 187
 188 def write_sample_interface(buildenv, iface, src_impl):
 189         path = buildenv.local_iface_file
 190         target_arch = buildenv.target_arch
 191
 192         impl = minidom.getDOMImplementation()
 193
 194         XMLNS_IFACE = namespaces.XMLNS_IFACE
 195
 196         doc = impl.createDocument(XMLNS_IFACE, "interface", None)
 197
 198         root = doc.documentElement
 199         root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns', XMLNS_IFACE)
 200
 201         def addSimple(parent, name, text = None):
 202                 elem = doc.createElementNS(XMLNS_IFACE, name)
 203
 204                 parent.appendChild(doc.createTextNode('\n' + '  ' * (1 + depth(parent))))
 205                 parent.appendChild(elem)
 206                 if text:
 207                         elem.appendChild(doc.createTextNode(text))
 208                 return elem
 209
 210         def close(element):
 211                 element.appendChild(doc.createTextNode('\n' + '  ' * depth(element)))
 212
 213         addSimple(root, 'name', iface.name)
 214         addSimple(root, 'summary', iface.summary)
 215         addSimple(root, 'description', iface.description)
 216         feed_for = addSimple(root, 'feed-for')
 217         feed_for.setAttributeNS(None, 'interface', iface.uri)
 218
 219         group = addSimple(root, 'group')
 220         main = buildenv.doc.getAttribute(XMLNS_0COMPILE + ' binary-main')
 221         if main:
 222                 group.setAttributeNS(None, 'main', main)
 223
 224         lib_mappings = buildenv.doc.getAttribute(XMLNS_0COMPILE + ' binary-lib-mappings')
 225         if lib_mappings:
 226                 root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns:compile', XMLNS_0COMPILE)
 227                 group.setAttributeNS(XMLNS_0COMPILE, 'compile:lib-mappings', lib_mappings)
 228
 229         for d in src_impl.dependencies:
 230                 # 0launch < 0.32 messed up the namespace...
 231                 if parse_bool(d.metadata.get('include-binary', 'false')) or \
 232                    parse_bool(d.metadata.get(XMLNS_0COMPILE + ' include-binary', 'false')):
 233                         requires = addSimple(group, 'requires')
 234                         requires.setAttributeNS(None, 'interface', d.interface)
 235                         for b in d.bindings:
 236                                 if isinstance(b, model.EnvironmentBinding):
 237                                         env_elem = addSimple(requires, 'environment')
 238                                         env_elem.setAttributeNS(None, 'name', b.name)
 239                                         env_elem.setAttributeNS(None, 'insert', b.insert)
 240                                         if b.default:
 241                                                 env_elem.setAttributeNS(None, 'default', b.default)
 242                                 else:
 243                                         raise Exception('Unknown binding type ' + b)
 244                         close(requires)
 245
 246         group.setAttributeNS(None, 'arch', target_arch)
 247         impl_elem = addSimple(group, 'implementation')
 248         impl_elem.setAttributeNS(None, 'version', src_impl.version)
 249
 250         if buildenv.version_modifier:
 251                 impl_elem.setAttributeNS(None, 'version-modifier', buildenv.version_modifier)
 252
 253         impl_elem.setAttributeNS(None, 'id', '..')
 254         impl_elem.setAttributeNS(None, 'released', time.strftime('%Y-%m-%d'))
 255         close(group)
 256         close(root)
 257
 258         doc.writexml(file(path, 'w'))
 259
 260 def set_up_mappings(mappings):
 261         """Create a temporary directory with symlinks for each of the library mappings."""
 262         # The find_library function takes a short-name and major version of a library and
 263         # returns the full path of the library.
 264         libdirs = ['/lib', '/usr/lib']
 265         for d in os.environ.get('LD_LIBRARY_PATH', '').split(':'):
 266                 if d: libdirs.append(d)
 267
 268         def add_ldconf(config_file):
 269                 if not os.path.isfile(config_file):
 270                         return
 271                 for line in file(config_file):
 272                         d = line.strip()
 273                         if d.startswith('include '):
 274                                 glob_pattern = d.split(' ', 1)[1]
 275                                 for conf in glob.glob(glob_pattern):
 276                                         add_ldconf(conf)
 277                         elif d and not d.startswith('#'):
 278                                 libdirs.append(d)
 279         add_ldconf('/etc/ld.so.conf')
 280
 281         def find_library(name, major):
 282                 wanted = 'lib%s.so.%s' % (name, major)
 283                 for d in libdirs:
 284                         path = os.path.join(d, wanted)
 285                         if os.path.exists(path):
 286                                 return path
 287                 print "WARNING: library '%s' not found (searched '%s')!" % (wanted, libdirs)
 288                 return None
 289
 290         mappings_dir = os.path.join(os.environ['TMPDIR'], 'lib-mappings')
 291         os.mkdir(mappings_dir)
 292
 293         old_path = os.environ.get('LIBRARY_PATH', '')
 294         if old_path: old_path = ':' + old_path
 295         os.environ['LIBRARY_PATH'] = mappings_dir + old_path
 296
 297         for name, major_version in mappings:
 298                 target = find_library(name, major_version)
 299                 if target:
 300                         print "Adding mapping lib%s.so -> %s" % (name, target)
 301                         os.symlink(target, os.path.join(mappings_dir, 'lib' + name + '.so'))
 302
 303 __main__.commands.append(do_build)