build.py

   1 # Copyright (C) 2006, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import sys, os, __main__, time, shutil, glob
   5 from os.path import join
   6 from logging import info
   7 from xml.dom import minidom, XMLNS_NAMESPACE
   8
   9 from support import *
  10
  11 def env(name, value):
  12         info('Setting %s="%s"', name, value)
  13         os.environ[name] = value
  14
  15 def do_env_binding(binding, path):
  16         os.environ[binding.name] = binding.get_value(path,
  17                                         os.environ.get(binding.name, None))
  18         info("%s=%s", binding.name, os.environ[binding.name])
  19
  20 def do_build_internal(args):
  21         """build-internal"""
  22         # If a sandbox is being used, we're in it now.
  23         import getpass, socket, time
  24
  25         buildenv = BuildEnv()
  26
  27         builddir = os.path.realpath('build')
  28         ensure_dir(buildenv.metadir)
  29
  30         build_env_xml = join(buildenv.metadir, 'build-environment.xml')
  31
  32         buildenv_doc = buildenv.selections.toDOM()
  33
  34         # Create build-environment.xml file
  35         root = buildenv_doc.documentElement
  36         info = buildenv_doc.createElementNS(XMLNS_0COMPILE, 'build-info')
  37         root.appendChild(info)
  38         info.setAttributeNS(None, 'time', time.strftime('%Y-%m-%d %H:%M').strip())
  39         info.setAttributeNS(None, 'host', socket.getfqdn())
  40         info.setAttributeNS(None, 'user', getpass.getuser())
  41         uname = os.uname()
  42         info.setAttributeNS(None, 'arch', '%s-%s' % (uname[0], uname[4]))
  43         stream = file(build_env_xml, 'w')
  44         buildenv_doc.writexml(stream, addindent="  ", newl="\n")
  45         stream.close()
  46
  47         # Create local binary interface file
  48         src_iface = iface_cache.get_interface(buildenv.interface)
  49         src_impl = buildenv.chosen_impl(buildenv.interface)
  50         write_sample_interface(buildenv, src_iface, src_impl)
  51
  52         # Create the patch
  53         orig_impl = buildenv.chosen_impl(buildenv.interface)
  54         patch_file = join(buildenv.metadir, 'from-%s.patch' % orig_impl.version)
  55         if os.path.isdir('src'):
  56                 orig_src = lookup(orig_impl.id)
  57                 # (ignore errors; will already be shown on stderr)
  58                 os.system("diff -urN '%s' src > %s" %
  59                         (orig_src.replace('\\', '\\\\').replace("'", "\\'"),
  60                          patch_file))
  61                 if os.path.getsize(patch_file) == 0:
  62                         os.unlink(patch_file)
  63         elif os.path.exists(patch_file):
  64                 os.unlink(patch_file)
  65
  66         env('BUILDDIR', builddir)
  67         env('DISTDIR', buildenv.distdir)
  68         env('SRCDIR', buildenv.srcdir)
  69         os.chdir(builddir)
  70
  71         for needed_iface in buildenv.selections.selections:
  72                 impl = buildenv.chosen_impl(needed_iface)
  73                 assert impl
  74                 for dep in impl.dependencies:
  75                         dep_iface = buildenv.selections.selections[dep.interface]
  76                         for b in dep.bindings:
  77                                 if isinstance(b, EnvironmentBinding):
  78                                         dep_impl = buildenv.chosen_impl(dep.interface)
  79                                         do_env_binding(b, lookup(dep_impl.id))
  80
  81         mappings = []
  82         for impl in buildenv.selections.selections.values():
  83                 new_mappings = impl.attrs.get(XMLNS_0COMPILE + ' lib-mappings', '')
  84                 if new_mappings:
  85                         new_mappings = new_mappings.split(' ')
  86                         for mapping in new_mappings:
  87                                 assert ':' in mapping, "lib-mappings missing ':' in '%s' from '%s'" % (mapping, impl.feed)
  88                                 name, major_version = mapping.split(':', 1)
  89                                 assert '/' not in mapping, "lib-mappings '%s' contains a / in the version number (from '%s')!" % (mapping, impl.feed)
  90                                 mappings.append((name, major_version))
  91
  92         if mappings:
  93                 set_up_mappings(mappings)
  94
  95         # Some programs want to put temporary build files in the source directory.
  96         # Make a copy of the source if needed.
  97         dup_src_type = buildenv.doc.getAttribute(XMLNS_0COMPILE + ' dup-src')
  98         if dup_src_type == 'true':
  99                 dup_src(shutil.copyfile)
 100         elif dup_src_type:
 101                 raise Exception("Unknown dup-src value '%s'" % dup_src_type)
 102
 103         if args == ['--shell']:
 104                 spawn_and_check(find_in_path('sh'), [])
 105         else:
 106                 command = buildenv.doc.getAttribute(XMLNS_0COMPILE + ' command')
 107
 108                 # Remove any existing log files
 109                 for log in ['build.log', 'build-success.log', 'build-failure.log']:
 110                         if os.path.exists(log):
 111                                 os.unlink(log)
 112
 113                 # Run the command, copying output to a new log
 114                 log = file('build.log', 'w')
 115                 try:
 116                         print >>log, "Build log for %s-%s" % (src_iface.get_name(),
 117                                                               src_impl.version)
 118                         print >>log, "\nBuilt using 0compile-%s" % __main__.version
 119                         print >>log, "\nBuild system: " + ', '.join(uname)
 120                         print >>log, "\n%s:\n" % ENV_FILE
 121                         shutil.copyfileobj(file("../" + ENV_FILE), log)
 122
 123                         log.write('\n')
 124
 125                         if os.path.exists(patch_file):
 126                                 print >>log, "\nPatched with:\n"
 127                                 shutil.copyfileobj(file(patch_file), log)
 128                                 log.write('\n')
 129
 130                         print "Executing: " + command
 131                         print >>log, "Executing: " + command
 132
 133                         # Tee the output to the console and to the log
 134                         from popen2 import Popen4
 135                         child = Popen4(command)
 136                         child.tochild.close()
 137                         while True:
 138                                 data = os.read(child.fromchild.fileno(), 100)
 139                                 if not data: break
 140                                 sys.stdout.write(data)
 141                                 log.write(data)
 142                         status = child.wait()
 143                         failure = None
 144                         if os.WIFEXITED(status):
 145                                 exit_code = os.WEXITSTATUS(status)
 146                                 if exit_code == 0:
 147                                         print >>log, "Build successful"
 148                                 else:
 149                                         failure = "Build failed with exit code %d" % exit_code
 150                         else:
 151                                 failure = "Build failure: exited due to signal %d" % os.WTERMSIG(status)
 152                         if failure:
 153                                 print >>log, failure
 154                                 os.rename('build.log', 'build-failure.log')
 155                                 raise SafeException("Command '%s': %s" % (command, failure))
 156                         else:
 157                                 os.rename('build.log', 'build-success.log')
 158                 finally:
 159                         log.close()
 160
 161 def do_build(args):
 162         """build [ --nosandbox ] [ --shell ]"""
 163         buildenv = BuildEnv()
 164
 165         builddir = os.path.realpath('build')
 166
 167         ensure_dir(builddir)
 168         ensure_dir(buildenv.distdir)
 169
 170         if args[:1] == ['--nosandbox']:
 171                 return do_build_internal(args[1:])
 172
 173         tmpdir = tempfile.mkdtemp(prefix = '0compile-')
 174         try:
 175                 my_dir = os.path.dirname(__file__)
 176                 readable = ['.', my_dir]
 177                 writable = ['build', buildenv.distdir, tmpdir]
 178                 env('TMPDIR', tmpdir)
 179
 180                 # Why did we need this?
 181                 #readable.append(get_cached_iface_path(buildenv.interface))
 182
 183                 for selection in buildenv.selections.selections.values():
 184                         readable.append(lookup(selection.id))
 185
 186                 options = []
 187                 if __main__.options.verbose:
 188                         options.append('--verbose')
 189
 190                 readable.append('/etc') # /etc/ld.*
 191
 192                 spawn_maybe_sandboxed(readable, writable, tmpdir, sys.executable, [sys.argv[0]] + options + ['build', '--nosandbox'] + args)
 193         finally:
 194                 info("Deleting temporary directory '%s'" % tmpdir)
 195                 shutil.rmtree(tmpdir)
 196
 197 def write_sample_interface(buildenv, iface, src_impl):
 198         path = buildenv.local_iface_file
 199         target_arch = buildenv.target_arch
 200
 201         impl = minidom.getDOMImplementation()
 202
 203         XMLNS_IFACE = namespaces.XMLNS_IFACE
 204
 205         doc = impl.createDocument(XMLNS_IFACE, "interface", None)
 206
 207         root = doc.documentElement
 208         root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns', XMLNS_IFACE)
 209
 210         def addSimple(parent, name, text = None):
 211                 elem = doc.createElementNS(XMLNS_IFACE, name)
 212
 213                 parent.appendChild(doc.createTextNode('\n' + '  ' * (1 + depth(parent))))
 214                 parent.appendChild(elem)
 215                 if text:
 216                         elem.appendChild(doc.createTextNode(text))
 217                 return elem
 218
 219         def close(element):
 220                 element.appendChild(doc.createTextNode('\n' + '  ' * depth(element)))
 221
 222         addSimple(root, 'name', iface.name)
 223         addSimple(root, 'summary', iface.summary)
 224         addSimple(root, 'description', iface.description)
 225         feed_for = addSimple(root, 'feed-for')
 226
 227         uri = iface.uri
 228         if uri.startswith('/') and iface.feed_for:
 229                 for uri in iface.feed_for:
 230                         print "Note: source %s is a local feed" % iface.uri
 231                         print "Will use <feed-for interface='%s'> instead..." % uri
 232                         break
 233
 234         feed_for.setAttributeNS(None, 'interface', uri)
 235
 236         group = addSimple(root, 'group')
 237         main = buildenv.doc.getAttribute(XMLNS_0COMPILE + ' binary-main')
 238         if main:
 239                 group.setAttributeNS(None, 'main', main)
 240
 241         lib_mappings = buildenv.doc.getAttribute(XMLNS_0COMPILE + ' binary-lib-mappings')
 242         if lib_mappings:
 243                 root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns:compile', XMLNS_0COMPILE)
 244                 group.setAttributeNS(XMLNS_0COMPILE, 'compile:lib-mappings', lib_mappings)
 245
 246         for d in src_impl.dependencies:
 247                 # 0launch < 0.32 messed up the namespace...
 248                 if parse_bool(d.metadata.get('include-binary', 'false')) or \
 249                    parse_bool(d.metadata.get(XMLNS_0COMPILE + ' include-binary', 'false')):
 250                         requires = addSimple(group, 'requires')
 251                         requires.setAttributeNS(None, 'interface', d.interface)
 252                         for b in d.bindings:
 253                                 if isinstance(b, model.EnvironmentBinding):
 254                                         env_elem = addSimple(requires, 'environment')
 255                                         env_elem.setAttributeNS(None, 'name', b.name)
 256                                         env_elem.setAttributeNS(None, 'insert', b.insert)
 257                                         if b.default:
 258                                                 env_elem.setAttributeNS(None, 'default', b.default)
 259                                 else:
 260                                         raise Exception('Unknown binding type ' + b)
 261                         close(requires)
 262
 263         group.setAttributeNS(None, 'arch', target_arch)
 264         impl_elem = addSimple(group, 'implementation')
 265         impl_elem.setAttributeNS(None, 'version', src_impl.version)
 266
 267         if buildenv.version_modifier:
 268                 impl_elem.setAttributeNS(None, 'version-modifier', buildenv.version_modifier)
 269
 270         impl_elem.setAttributeNS(None, 'id', '..')
 271         impl_elem.setAttributeNS(None, 'released', time.strftime('%Y-%m-%d'))
 272         close(group)
 273         close(root)
 274
 275         doc.writexml(file(path, 'w'))
 276
 277 def set_up_mappings(mappings):
 278         """Create a temporary directory with symlinks for each of the library mappings."""
 279         # The find_library function takes a short-name and major version of a library and
 280         # returns the full path of the library.
 281         libdirs = ['/lib', '/usr/lib']
 282         for d in os.environ.get('LD_LIBRARY_PATH', '').split(':'):
 283                 if d: libdirs.append(d)
 284
 285         def add_ldconf(config_file):
 286                 if not os.path.isfile(config_file):
 287                         return
 288                 for line in file(config_file):
 289                         d = line.strip()
 290                         if d.startswith('include '):
 291                                 glob_pattern = d.split(' ', 1)[1]
 292                                 for conf in glob.glob(glob_pattern):
 293                                         add_ldconf(conf)
 294                         elif d and not d.startswith('#'):
 295                                 libdirs.append(d)
 296         add_ldconf('/etc/ld.so.conf')
 297
 298         def find_library(name, major):
 299                 wanted = 'lib%s.so.%s' % (name, major)
 300                 for d in libdirs:
 301                         path = os.path.join(d, wanted)
 302                         if os.path.exists(path):
 303                                 return path
 304                 print "WARNING: library '%s' not found (searched '%s')!" % (wanted, libdirs)
 305                 return None
 306
 307         mappings_dir = os.path.join(os.environ['TMPDIR'], 'lib-mappings')
 308         os.mkdir(mappings_dir)
 309
 310         old_path = os.environ.get('LIBRARY_PATH', '')
 311         if old_path: old_path = ':' + old_path
 312         os.environ['LIBRARY_PATH'] = mappings_dir + old_path
 313
 314         for name, major_version in mappings:
 315                 target = find_library(name, major_version)
 316                 if target:
 317                         print "Adding mapping lib%s.so -> %s" % (name, target)
 318                         os.symlink(target, os.path.join(mappings_dir, 'lib' + name + '.so'))
 319
 320 def dup_src(fn):
 321         srcdir = os.environ['SRCDIR'] + '/'
 322         for root, dirs, files in os.walk(srcdir):
 323                 assert root.startswith(srcdir)
 324                 reldir = root[len(srcdir):]
 325                 for f in files:
 326                         target = os.path.join(reldir, f)
 327                         #print "Copy %s -> %s" % (os.path.join(root, f), target)
 328                         if os.path.exists(target):
 329                                 os.unlink(target)
 330                         fn(os.path.join(root, f), target)
 331                 for d in dirs:
 332                         target = os.path.join(reldir, d)
 333                         if not os.path.isdir(target):
 334                                 os.mkdir(target)
 335
 336 __main__.commands.append(do_build)