config/JarMaker.py

   1 # ***** BEGIN LICENSE BLOCK *****
   2 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   3 #
   4 # The contents of this file are subject to the Mozilla Public License Version
   5 # 1.1 (the "License"); you may not use this file except in compliance with
   6 # the License. You may obtain a copy of the License at
   7 # http://www.mozilla.org/MPL/
   8 #
   9 # Software distributed under the License is distributed on an "AS IS" basis,
  10 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  11 # for the specific language governing rights and limitations under the
  12 # License.
  13 #
  14 # The Original Code is Mozilla build system.
  15 #
  16 # The Initial Developer of the Original Code is
  17 # Mozilla Foundation.
  18 # Portions created by the Initial Developer are Copyright (C) 2008
  19 # the Initial Developer. All Rights Reserved.
  20 #
  21 # Contributor(s):
  22 #  Axel Hecht <l10n@mozilla.com>
  23 #
  24 # Alternatively, the contents of this file may be used under the terms of
  25 # either the GNU General Public License Version 2 or later (the "GPL"), or
  26 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  27 # in which case the provisions of the GPL or the LGPL are applicable instead
  28 # of those above. If you wish to allow use of your version of this file only
  29 # under the terms of either the GPL or the LGPL, and not to allow others to
  30 # use your version of this file under the terms of the MPL, indicate your
  31 # decision by deleting the provisions above and replace them with the notice
  32 # and other provisions required by the GPL or the LGPL. If you do not delete
  33 # the provisions above, a recipient may use your version of this file under
  34 # the terms of any one of the MPL, the GPL or the LGPL.
  35 #
  36 # ***** END LICENSE BLOCK *****
  37
  38 '''jarmaker.py provides a python class to package up chrome content by
  39 processing jar.mn files.
  40
  41 See the documentation for jar.mn on MDC for further details on the format.
  42 '''
  43
  44 import sys
  45 import os
  46 import os.path
  47 import errno
  48 import re
  49 import logging
  50 from time import localtime
  51 from optparse import OptionParser
  52 from MozZipFile import ZipFile
  53 from cStringIO import StringIO
  54 from datetime import datetime
  55
  56 from utils import pushback_iter, lockFile
  57 from Preprocessor import Preprocessor
  58 from buildlist import addEntriesToListFile
  59
  60 __all__ = ['JarMaker']
  61
  62 class ZipEntry:
  63   '''Helper class for jar output.
  64
  65   This class defines a simple file-like object for a zipfile.ZipEntry
  66   so that we can consecutively write to it and then close it.
  67   This methods hooks into ZipFile.writestr on close().
  68   '''
  69   def __init__(self, name, zipfile):
  70     self._zipfile = zipfile
  71     self._name = name
  72     self._inner = StringIO()
  73
  74   def write(self, content):
  75     'Append the given content to this zip entry'
  76     self._inner.write(content)
  77     return
  78
  79   def close(self):
  80     'The close method writes the content back to the zip file.'
  81     self._zipfile.writestr(self._name, self._inner.getvalue())
  82
  83 def getModTime(aPath):
  84   if not os.path.isfile(aPath):
  85     return 0
  86   mtime = os.stat(aPath).st_mtime
  87   return localtime(mtime)
  88
  89
  90 class JarMaker(object):
  91   '''JarMaker reads jar.mn files and process those into jar files or
  92   flat directories, along with chrome.manifest files.
  93   '''
  94
  95   ignore = re.compile('\s*(\#.*)?$')
  96   jarline = re.compile('(?:(?P<jarfile>[\w\d.\-\_\\\/]+).jar\:)|(?:\s*(\#.*)?)\s*$')
  97   regline = re.compile('\%\s+(.*)$')
  98   entryre = '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+'
  99   entryline = re.compile(entryre + '(?P<output>[\w\d.\-\_\\\/\+]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/]+)\))?\s*$')
 100
 101   def __init__(self, outputFormat = 'flat', useJarfileManifest = True,
 102                useChromeManifest = False):
 103     self.outputFormat = outputFormat
 104     self.useJarfileManifest = useJarfileManifest
 105     self.useChromeManifest = useChromeManifest
 106     self.pp = Preprocessor()
 107
 108   def getCommandLineParser(self):
 109     '''Get a optparse.OptionParser for jarmaker.
 110
 111     This OptionParser has the options for jarmaker as well as
 112     the options for the inner PreProcessor.
 113     '''
 114     # HACK, we need to unescape the string variables we get,
 115     # the perl versions didn't grok strings right
 116     p = self.pp.getCommandLineParser(unescapeDefines = True)
 117     p.add_option('-f', type="choice", default="jar",
 118                  choices=('jar', 'flat', 'symlink'),
 119                  help="fileformat used for output", metavar="[jar, flat, symlink]")
 120     p.add_option('-v', action="store_true", dest="verbose",
 121                  help="verbose output")
 122     p.add_option('-q', action="store_false", dest="verbose",
 123                  help="verbose output")
 124     p.add_option('-e', action="store_true",
 125                  help="create chrome.manifest instead of jarfile.manifest")
 126     p.add_option('--both-manifests', action="store_true",
 127                  dest="bothManifests",
 128                  help="create chrome.manifest and jarfile.manifest")
 129     p.add_option('-s', type="string", action="append", default=[],
 130                  help="source directory")
 131     p.add_option('-t', type="string",
 132                  help="top source directory")
 133     p.add_option('-c', '--l10n-src', type="string", action="append",
 134                  help="localization directory")
 135     p.add_option('--l10n-base', type="string", action="append", default=[],
 136                  help="base directory to be used for localization (multiple)")
 137     p.add_option('-j', type="string",
 138                  help="jarfile directory")
 139     # backwards compat, not needed
 140     p.add_option('-a', action="store_false", default=True,
 141                  help="NOT SUPPORTED, turn auto-registration of chrome off (installed-chrome.txt)")
 142     p.add_option('-d', type="string",
 143                  help="UNUSED, chrome directory")
 144     p.add_option('-o', help="cross compile for auto-registration, ignored")
 145     p.add_option('-l', action="store_true",
 146                  help="ignored (used to switch off locks)")
 147     p.add_option('-x', action="store_true",
 148                  help="force Unix")
 149     p.add_option('-z', help="backwards compat, ignored")
 150     p.add_option('-p', help="backwards compat, ignored")
 151     return p
 152
 153   def processIncludes(self, includes):
 154     '''Process given includes with the inner PreProcessor.
 155
 156     Only use this for #defines, the includes shouldn't generate
 157     content.
 158     '''
 159     self.pp.out = StringIO()
 160     for inc in includes:
 161       self.pp.do_include(inc)
 162     includesvalue = self.pp.out.getvalue()
 163     if includesvalue:
 164       logging.info("WARNING: Includes produce non-empty output")
 165     self.pp.out = None
 166     pass
 167
 168   def finalizeJar(self, jarPath, chromebasepath, register,
 169                   doZip=True):
 170     '''Helper method to write out the chrome registration entries to
 171     jarfile.manifest or chrome.manifest, or both.
 172
 173     The actual file processing is done in updateManifest.
 174     '''
 175     # rewrite the manifest, if entries given
 176     if not register:
 177       return
 178
 179     chromeManifest = os.path.join(os.path.dirname(jarPath),
 180                                   '..', 'chrome.manifest')
 181
 182     if self.useJarfileManifest:
 183       self.updateManifest(jarPath + '.manifest', chromebasepath % '',
 184                           register)
 185       addEntriesToListFile(chromeManifest, ['manifest chrome/%s.manifest' % (os.path.basename(jarPath),)])
 186     if self.useChromeManifest:
 187       self.updateManifest(chromeManifest, chromebasepath % 'chrome/',
 188                           register)
 189
 190   def updateManifest(self, manifestPath, chromebasepath, register):
 191     '''updateManifest replaces the % in the chrome registration entries
 192     with the given chrome base path, and updates the given manifest file.
 193     '''
 194     lock = lockFile(manifestPath + '.lck')
 195     try:
 196       myregister = dict.fromkeys(map(lambda s: s.replace('%', chromebasepath),
 197                                      register.iterkeys()))
 198       manifestExists = os.path.isfile(manifestPath)
 199       mode = (manifestExists and 'r+b') or 'wb'
 200       mf = open(manifestPath, mode)
 201       if manifestExists:
 202         # import previous content into hash, ignoring empty ones and comments
 203         imf = re.compile('(#.*)?$')
 204         for l in re.split('[\r\n]+', mf.read()):
 205           if imf.match(l):
 206             continue
 207           myregister[l] = None
 208         mf.seek(0)
 209       for k in myregister.iterkeys():
 210         mf.write(k + os.linesep)
 211       mf.close()
 212     finally:
 213       lock = None
 214
 215   def makeJar(self, infile=None,
 216                jardir='',
 217                sourcedirs=[], topsourcedir='', localedirs=None):
 218     '''makeJar is the main entry point to JarMaker.
 219
 220     It takes the input file, the output directory, the source dirs and the
 221     top source dir as argument, and optionally the l10n dirs.
 222     '''
 223     if isinstance(infile, basestring):
 224       logging.info("processing " + infile)
 225     pp = self.pp.clone()
 226     pp.out = StringIO()
 227     pp.do_include(infile)
 228     lines = pushback_iter(pp.out.getvalue().splitlines())
 229     try:
 230       while True:
 231         l = lines.next()
 232         m = self.jarline.match(l)
 233         if not m:
 234           raise RuntimeError(l)
 235         if m.group('jarfile') is None:
 236           # comment
 237           continue
 238         self.processJarSection(m.group('jarfile'), lines,
 239                                jardir, sourcedirs, topsourcedir,
 240                                localedirs)
 241     except StopIteration:
 242       # we read the file
 243       pass
 244     return
 245
 246   def makeJars(self, infiles, l10nbases,
 247                jardir='',
 248                sourcedirs=[], topsourcedir='', localedirs=None):
 249     '''makeJars is the second main entry point to JarMaker.
 250
 251     It takes an iterable sequence of input file names, the l10nbases,
 252     the output directory, the source dirs and the
 253     top source dir as argument, and optionally the l10n dirs.
 254
 255     It iterates over all inputs, guesses srcdir and l10ndir from the
 256     path and topsourcedir and calls into makeJar.
 257
 258     The l10ndirs are created by guessing the relativesrcdir, and resolving
 259     that against the l10nbases. l10nbases can either be path strings, or
 260     callables. In the latter case, that will be called with the
 261     relativesrcdir as argument, and is expected to return a path string.
 262     This logic is disabled if the jar.mn path is not inside the topsrcdir.
 263     '''
 264     topsourcedir = os.path.normpath(os.path.abspath(topsourcedir))
 265     def resolveL10nBase(relpath):
 266       def _resolve(base):
 267         if isinstance(base, basestring):
 268           return os.path.join(base, relpath)
 269         if callable(base):
 270           return base(relpath)
 271         return base
 272       return _resolve
 273     for infile in infiles:
 274       srcdir = os.path.normpath(os.path.abspath(os.path.dirname(infile)))
 275       l10ndir = srcdir
 276       if os.path.basename(srcdir) == 'locales':
 277         l10ndir = os.path.dirname(l10ndir)
 278
 279       l10ndirs = None
 280       # srcdir may not be a child of topsourcedir, in which case
 281       # we assume that the caller passed in suitable sourcedirs,
 282       # and just skip passing in localedirs
 283       if srcdir.startswith(topsourcedir):
 284         rell10ndir = l10ndir[len(topsourcedir):].lstrip(os.sep)
 285
 286         l10ndirs = map(resolveL10nBase(rell10ndir), l10nbases)
 287         if localedirs is not None:
 288           l10ndirs += [os.path.normpath(os.path.abspath(s))
 289                        for s in localedirs]
 290       srcdirs = [os.path.normpath(os.path.abspath(s))
 291                  for s in sourcedirs] + [srcdir]
 292       self.makeJar(infile=infile,
 293                    sourcedirs=srcdirs, topsourcedir=topsourcedir,
 294                    localedirs=l10ndirs,
 295                    jardir=jardir)
 296
 297
 298   def processJarSection(self, jarfile, lines,
 299                         jardir, sourcedirs, topsourcedir, localedirs):
 300     '''Internal method called by makeJar to actually process a section
 301     of a jar.mn file.
 302
 303     jarfile is the basename of the jarfile or the directory name for
 304     flat output, lines is a pushback_iterator of the lines of jar.mn,
 305     the remaining options are carried over from makeJar.
 306     '''
 307
 308     # chromebasepath is used for chrome registration manifests
 309     # %s is getting replaced with chrome/ for chrome.manifest, and with
 310     # an empty string for jarfile.manifest
 311     chromebasepath = '%s' + jarfile
 312     if self.outputFormat == 'jar':
 313       chromebasepath = 'jar:' + chromebasepath + '.jar!'
 314     chromebasepath += '/'
 315
 316     jarfile = os.path.join(jardir, jarfile)
 317     jf = None
 318     if self.outputFormat == 'jar':
 319       #jar
 320       jarfilepath = jarfile + '.jar'
 321       try:
 322         os.makedirs(os.path.dirname(jarfilepath))
 323       except OSError:
 324         pass
 325       jf = ZipFile(jarfilepath, 'a', lock = True)
 326       outHelper = self.OutputHelper_jar(jf)
 327     else:
 328       outHelper = getattr(self, 'OutputHelper_' + self.outputFormat)(jarfile)
 329     register = {}
 330     # This loop exits on either
 331     # - the end of the jar.mn file
 332     # - an line in the jar.mn file that's not part of a jar section
 333     # - on an exception raised, close the jf in that case in a finally
 334     try:
 335       while True:
 336         try:
 337           l = lines.next()
 338         except StopIteration:
 339           # we're done with this jar.mn, and this jar section
 340           self.finalizeJar(jarfile, chromebasepath, register)
 341           if jf is not None:
 342             jf.close()
 343           # reraise the StopIteration for makeJar
 344           raise
 345         if self.ignore.match(l):
 346           continue
 347         m = self.regline.match(l)
 348         if  m:
 349           rline = m.group(1)
 350           register[rline] = 1
 351           continue
 352         m = self.entryline.match(l)
 353         if not m:
 354           # neither an entry line nor chrome reg, this jar section is done
 355           self.finalizeJar(jarfile, chromebasepath, register)
 356           if jf is not None:
 357             jf.close()
 358           lines.pushback(l)
 359           return
 360         self._processEntryLine(m, sourcedirs, topsourcedir, localedirs,
 361                               outHelper, jf)
 362     finally:
 363       if jf is not None:
 364         jf.close()
 365     return
 366
 367   def _processEntryLine(self, m,
 368                         sourcedirs, topsourcedir, localedirs,
 369                         outHelper, jf):
 370       out = m.group('output')
 371       src = m.group('source') or os.path.basename(out)
 372       # pick the right sourcedir -- l10n, topsrc or src
 373       if m.group('locale'):
 374         src_base = localedirs
 375       elif src.startswith('/'):
 376         # path/in/jar/file_name.xul     (/path/in/sourcetree/file_name.xul)
 377         # refers to a path relative to topsourcedir, use that as base
 378         # and strip the leading '/'
 379         src_base = [topsourcedir]
 380         src = src[1:]
 381       else:
 382         # use srcdirs and the objdir (current working dir) for relative paths
 383         src_base = sourcedirs + ['.']
 384       # check if the source file exists
 385       realsrc = None
 386       for _srcdir in src_base:
 387         if os.path.isfile(os.path.join(_srcdir, src)):
 388           realsrc = os.path.join(_srcdir, src)
 389           break
 390       if realsrc is None:
 391         if jf is not None:
 392           jf.close()
 393         raise RuntimeError('File "%s" not found in %s' % (src, ', '.join(src_base)))
 394       if m.group('optPreprocess'):
 395         outf = outHelper.getOutput(out)
 396         inf = open(realsrc)
 397         pp = self.pp.clone()
 398         if src[-4:] == '.css':
 399           pp.setMarker('%')
 400         pp.out = outf
 401         pp.do_include(inf)
 402         outf.close()
 403         inf.close()
 404         return
 405       # copy or symlink if newer or overwrite
 406       if (m.group('optOverwrite')
 407           or (getModTime(realsrc) >
 408               outHelper.getDestModTime(m.group('output')))):
 409         if self.outputFormat == 'symlink' and hasattr(os, 'symlink'):
 410           outHelper.symlink(realsrc, out)
 411           return
 412         outf = outHelper.getOutput(out)
 413         # open in binary mode, this can be images etc
 414         inf = open(realsrc, 'rb')
 415         outf.write(inf.read())
 416         outf.close()
 417         inf.close()
 418
 419
 420   class OutputHelper_jar(object):
 421     '''Provide getDestModTime and getOutput for a given jarfile.
 422     '''
 423     def __init__(self, jarfile):
 424       self.jarfile = jarfile
 425     def getDestModTime(self, aPath):
 426       try :
 427         info = self.jarfile.getinfo(aPath)
 428         return info.date_time
 429       except:
 430         return 0
 431     def getOutput(self, name):
 432       return ZipEntry(name, self.jarfile)
 433
 434   class OutputHelper_flat(object):
 435     '''Provide getDestModTime and getOutput for a given flat
 436     output directory. The helper method ensureDirFor is used by
 437     the symlink subclass.
 438     '''
 439     def __init__(self, basepath):
 440       self.basepath = basepath
 441     def getDestModTime(self, aPath):
 442       return getModTime(os.path.join(self.basepath, aPath))
 443     def getOutput(self, name):
 444       out = self.ensureDirFor(name)
 445       # remove previous link or file
 446       try:
 447         os.remove(out)
 448       except OSError, e:
 449         if e.errno != errno.ENOENT:
 450           raise
 451       return open(out, 'wb')
 452     def ensureDirFor(self, name):
 453       out = os.path.join(self.basepath, name)
 454       outdir = os.path.dirname(out)
 455       if not os.path.isdir(outdir):
 456         os.makedirs(outdir)
 457       return out
 458
 459   class OutputHelper_symlink(OutputHelper_flat):
 460     '''Subclass of OutputHelper_flat that provides a helper for
 461     creating a symlink including creating the parent directories.
 462     '''
 463     def symlink(self, src, dest):
 464       out = self.ensureDirFor(dest)
 465       # remove previous link or file
 466       try:
 467         os.remove(out)
 468       except OSError, e:
 469         if e.errno != errno.ENOENT:
 470           raise
 471       os.symlink(src, out)
 472
 473 def main():
 474   jm = JarMaker()
 475   p = jm.getCommandLineParser()
 476   (options, args) = p.parse_args()
 477   jm.processIncludes(options.I)
 478   jm.outputFormat = options.f
 479   if options.e:
 480     jm.useChromeManifest = True
 481     jm.useJarfileManifest = False
 482   if options.bothManifests:
 483     jm.useChromeManifest = True
 484     jm.useJarfileManifest = True
 485   noise = logging.INFO
 486   if options.verbose is not None:
 487     noise = (options.verbose and logging.DEBUG) or logging.WARN
 488   if sys.version_info[:2] > (2,3):
 489     logging.basicConfig(format = "%(message)s")
 490   else:
 491     logging.basicConfig()
 492   logging.getLogger().setLevel(noise)
 493   topsrc = options.t
 494   topsrc = os.path.normpath(os.path.abspath(topsrc))
 495   if not args:
 496     jm.makeJar(infile=sys.stdin,
 497                sourcedirs=options.s, topsourcedir=topsrc,
 498                localedirs=options.l10n_src,
 499                jardir=options.j)
 500   else:
 501     jm.makeJars(args, options.l10n_base,
 502                 jardir=options.j,
 503                 sourcedirs=options.s, topsourcedir=topsrc,
 504                 localedirs=options.l10n_src)
 505
 506 if __name__ == "__main__":
 507   main()