config/JarMaker.py

   1 # ***** BEGIN LICENSE BLOCK *****
   2 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   3 #
   4 # The contents of this file are subject to the Mozilla Public License Version
   5 # 1.1 (the "License"); you may not use this file except in compliance with
   6 # the License. You may obtain a copy of the License at
   7 # http://www.mozilla.org/MPL/
   8 #
   9 # Software distributed under the License is distributed on an "AS IS" basis,
  10 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  11 # for the specific language governing rights and limitations under the
  12 # License.
  13 #
  14 # The Original Code is Mozilla build system.
  15 #
  16 # The Initial Developer of the Original Code is
  17 # Mozilla Foundation.
  18 # Portions created by the Initial Developer are Copyright (C) 2008
  19 # the Initial Developer. All Rights Reserved.
  20 #
  21 # Contributor(s):
  22 #  Axel Hecht <l10n@mozilla.com>
  23 #
  24 # Alternatively, the contents of this file may be used under the terms of
  25 # either the GNU General Public License Version 2 or later (the "GPL"), or
  26 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  27 # in which case the provisions of the GPL or the LGPL are applicable instead
  28 # of those above. If you wish to allow use of your version of this file only
  29 # under the terms of either the GPL or the LGPL, and not to allow others to
  30 # use your version of this file under the terms of the MPL, indicate your
  31 # decision by deleting the provisions above and replace them with the notice
  32 # and other provisions required by the GPL or the LGPL. If you do not delete
  33 # the provisions above, a recipient may use your version of this file under
  34 # the terms of any one of the MPL, the GPL or the LGPL.
  35 #
  36 # ***** END LICENSE BLOCK *****
  37
  38 '''jarmaker.py provides a python class to package up chrome content by
  39 processing jar.mn files.
  40
  41 See the documentation for jar.mn on MDC for further details on the format.
  42 '''
  43
  44 import sys
  45 import os
  46 import os.path
  47 import re
  48 import logging
  49 from time import localtime
  50 from optparse import OptionParser
  51 from MozZipFile import ZipFile
  52 from cStringIO import StringIO
  53 from datetime import datetime
  54
  55 from utils import pushback_iter, lockFile
  56 from Preprocessor import Preprocessor
  57 from buildlist import addEntriesToListFile
  58
  59 __all__ = ['JarMaker']
  60
  61 class ZipEntry:
  62   '''Helper class for jar output.
  63
  64   This class defines a simple file-like object for a zipfile.ZipEntry
  65   so that we can consecutively write to it and then close it.
  66   This methods hooks into ZipFile.writestr on close().
  67   '''
  68   def __init__(self, name, zipfile):
  69     self._zipfile = zipfile
  70     self._name = name
  71     self._inner = StringIO()
  72
  73   def write(self, content):
  74     'Append the given content to this zip entry'
  75     self._inner.write(content)
  76     return
  77
  78   def close(self):
  79     'The close method writes the content back to the zip file.'
  80     self._zipfile.writestr(self._name, self._inner.getvalue())
  81
  82 def getModTime(aPath):
  83   if not os.path.isfile(aPath):
  84     return 0
  85   mtime = os.stat(aPath).st_mtime
  86   return localtime(mtime)
  87
  88
  89 class JarMaker(object):
  90   '''JarMaker reads jar.mn files and process those into jar files or
  91   flat directories, along with chrome.manifest files.
  92   '''
  93
  94   ignore = re.compile('\s*(\#.*)?$')
  95   jarline = re.compile('(?:(?P<jarfile>[\w\d.\-\_\\\/]+).jar\:)|(?:\s*(\#.*)?)\s*$')
  96   regline = re.compile('\%\s+(.*)$')
  97   entryre = '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+'
  98   entryline = re.compile(entryre + '(?P<output>[\w\d.\-\_\\\/\+]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/]+)\))?\s*$')
  99
 100   def __init__(self, outputFormat = 'flat', useJarfileManifest = True,
 101                useChromeManifest = False):
 102     self.outputFormat = outputFormat
 103     self.useJarfileManifest = useJarfileManifest
 104     self.useChromeManifest = useChromeManifest
 105     self.pp = Preprocessor()
 106
 107   def getCommandLineParser(self):
 108     '''Get a optparse.OptionParser for jarmaker.
 109
 110     This OptionParser has the options for jarmaker as well as
 111     the options for the inner PreProcessor.
 112     '''
 113     # HACK, we need to unescape the string variables we get,
 114     # the perl versions didn't grok strings right
 115     p = self.pp.getCommandLineParser(unescapeDefines = True)
 116     p.add_option('-f', type="choice", default="jar",
 117                  choices=('jar', 'flat', 'symlink'),
 118                  help="fileformat used for output", metavar="[jar, flat, symlink]")
 119     p.add_option('-v', action="store_true", dest="verbose",
 120                  help="verbose output")
 121     p.add_option('-q', action="store_false", dest="verbose",
 122                  help="verbose output")
 123     p.add_option('-e', action="store_true",
 124                  help="create chrome.manifest instead of jarfile.manifest")
 125     p.add_option('--both-manifests', action="store_true",
 126                  dest="bothManifests",
 127                  help="create chrome.manifest and jarfile.manifest")
 128     p.add_option('-s', type="string", action="append", default=[],
 129                  help="source directory")
 130     p.add_option('-t', type="string",
 131                  help="top source directory")
 132     p.add_option('-c', '--l10n-src', type="string", action="append",
 133                  help="localization directory")
 134     p.add_option('--l10n-base', type="string", action="append", default=[],
 135                  help="base directory to be used for localization (multiple)")
 136     p.add_option('-j', type="string",
 137                  help="jarfile directory")
 138     # backwards compat, not needed
 139     p.add_option('-a', action="store_false", default=True,
 140                  help="NOT SUPPORTED, turn auto-registration of chrome off (installed-chrome.txt)")
 141     p.add_option('-d', type="string",
 142                  help="UNUSED, chrome directory")
 143     p.add_option('-o', help="cross compile for auto-registration, ignored")
 144     p.add_option('-l', action="store_true",
 145                  help="ignored (used to switch off locks)")
 146     p.add_option('-x', action="store_true",
 147                  help="force Unix")
 148     p.add_option('-z', help="backwards compat, ignored")
 149     p.add_option('-p', help="backwards compat, ignored")
 150     return p
 151
 152   def processIncludes(self, includes):
 153     '''Process given includes with the inner PreProcessor.
 154
 155     Only use this for #defines, the includes shouldn't generate
 156     content.
 157     '''
 158     self.pp.out = StringIO()
 159     for inc in includes:
 160       self.pp.do_include(inc)
 161     includesvalue = self.pp.out.getvalue()
 162     if includesvalue:
 163       logging.info("WARNING: Includes produce non-empty output")
 164     self.pp.out = None
 165     pass
 166
 167   def finalizeJar(self, jarPath, chromebasepath, register,
 168                   doZip=True):
 169     '''Helper method to write out the chrome registration entries to
 170     jarfile.manifest or chrome.manifest, or both.
 171
 172     The actual file processing is done in updateManifest.
 173     '''
 174     # rewrite the manifest, if entries given
 175     if not register:
 176       return
 177
 178     chromeManifest = os.path.join(os.path.dirname(jarPath),
 179                                   '..', 'chrome.manifest')
 180
 181     if self.useJarfileManifest:
 182       self.updateManifest(jarPath + '.manifest', chromebasepath % '',
 183                           register)
 184       addEntriesToListFile(chromeManifest, ['manifest chrome/%s.manifest' % (os.path.basename(jarPath),)])
 185     if self.useChromeManifest:
 186       self.updateManifest(chromeManifest, chromebasepath % 'chrome/',
 187                           register)
 188
 189   def updateManifest(self, manifestPath, chromebasepath, register):
 190     '''updateManifest replaces the % in the chrome registration entries
 191     with the given chrome base path, and updates the given manifest file.
 192     '''
 193     lock = lockFile(manifestPath + '.lck')
 194     try:
 195       myregister = dict.fromkeys(map(lambda s: s.replace('%', chromebasepath),
 196                                      register.iterkeys()))
 197       manifestExists = os.path.isfile(manifestPath)
 198       mode = (manifestExists and 'r+b') or 'wb'
 199       mf = open(manifestPath, mode)
 200       if manifestExists:
 201         # import previous content into hash, ignoring empty ones and comments
 202         imf = re.compile('(#.*)?$')
 203         for l in re.split('[\r\n]+', mf.read()):
 204           if imf.match(l):
 205             continue
 206           myregister[l] = None
 207         mf.seek(0)
 208       for k in myregister.iterkeys():
 209         mf.write(k + os.linesep)
 210       mf.close()
 211     finally:
 212       lock = None
 213
 214   def makeJar(self, infile=None,
 215                jardir='',
 216                sourcedirs=[], topsourcedir='', localedirs=None):
 217     '''makeJar is the main entry point to JarMaker.
 218
 219     It takes the input file, the output directory, the source dirs and the
 220     top source dir as argument, and optionally the l10n dirs.
 221     '''
 222     if isinstance(infile, basestring):
 223       logging.info("processing " + infile)
 224     pp = self.pp.clone()
 225     pp.out = StringIO()
 226     pp.do_include(infile)
 227     lines = pushback_iter(pp.out.getvalue().splitlines())
 228     try:
 229       while True:
 230         l = lines.next()
 231         m = self.jarline.match(l)
 232         if not m:
 233           raise RuntimeError(l)
 234         if m.group('jarfile') is None:
 235           # comment
 236           continue
 237         self.processJarSection(m.group('jarfile'), lines,
 238                                jardir, sourcedirs, topsourcedir,
 239                                localedirs)
 240     except StopIteration:
 241       # we read the file
 242       pass
 243     return
 244
 245   def makeJars(self, infiles, l10nbases,
 246                jardir='',
 247                sourcedirs=[], topsourcedir='', localedirs=None):
 248     '''makeJars is the second main entry point to JarMaker.
 249
 250     It takes an iterable sequence of input file names, the l10nbases,
 251     the output directory, the source dirs and the
 252     top source dir as argument, and optionally the l10n dirs.
 253
 254     It iterates over all inputs, guesses srcdir and l10ndir from the
 255     path and topsourcedir and calls into makeJar.
 256
 257     The l10ndirs are created by guessing the relativesrcdir, and resolving
 258     that against the l10nbases. l10nbases can either be path strings, or
 259     callables. In the latter case, that will be called with the
 260     relativesrcdir as argument, and is expected to return a path string.
 261     This logic is disabled if the jar.mn path is not inside the topsrcdir.
 262     '''
 263     topsourcedir = os.path.normpath(os.path.abspath(topsourcedir))
 264     def resolveL10nBase(relpath):
 265       def _resolve(base):
 266         if isinstance(base, basestring):
 267           return os.path.join(base, relpath)
 268         if callable(base):
 269           return base(relpath)
 270         return base
 271       return _resolve
 272     for infile in infiles:
 273       srcdir = os.path.normpath(os.path.abspath(os.path.dirname(infile)))
 274       l10ndir = srcdir
 275       if os.path.basename(srcdir) == 'locales':
 276         l10ndir = os.path.dirname(l10ndir)
 277
 278       l10ndirs = None
 279       # srcdir may not be a child of topsourcedir, in which case
 280       # we assume that the caller passed in suitable sourcedirs,
 281       # and just skip passing in localedirs
 282       if srcdir.startswith(topsourcedir):
 283         rell10ndir = l10ndir[len(topsourcedir):].lstrip(os.sep)
 284
 285         l10ndirs = map(resolveL10nBase(rell10ndir), l10nbases)
 286         if localedirs is not None:
 287           l10ndirs += [os.path.normpath(os.path.abspath(s))
 288                        for s in localedirs]
 289       srcdirs = [os.path.normpath(os.path.abspath(s))
 290                  for s in sourcedirs] + [srcdir]
 291       self.makeJar(infile=infile,
 292                    sourcedirs=srcdirs, topsourcedir=topsourcedir,
 293                    localedirs=l10ndirs,
 294                    jardir=jardir)
 295
 296
 297   def processJarSection(self, jarfile, lines,
 298                         jardir, sourcedirs, topsourcedir, localedirs):
 299     '''Internal method called by makeJar to actually process a section
 300     of a jar.mn file.
 301
 302     jarfile is the basename of the jarfile or the directory name for
 303     flat output, lines is a pushback_iterator of the lines of jar.mn,
 304     the remaining options are carried over from makeJar.
 305     '''
 306
 307     # chromebasepath is used for chrome registration manifests
 308     # %s is getting replaced with chrome/ for chrome.manifest, and with
 309     # an empty string for jarfile.manifest
 310     chromebasepath = '%s' + jarfile
 311     if self.outputFormat == 'jar':
 312       chromebasepath = 'jar:' + chromebasepath + '.jar!'
 313     chromebasepath += '/'
 314
 315     jarfile = os.path.join(jardir, jarfile)
 316     jf = None
 317     if self.outputFormat == 'jar':
 318       #jar
 319       jarfilepath = jarfile + '.jar'
 320       try:
 321         os.makedirs(os.path.dirname(jarfilepath))
 322       except OSError:
 323         pass
 324       jf = ZipFile(jarfilepath, 'a', lock = True)
 325       outHelper = self.OutputHelper_jar(jf)
 326     else:
 327       outHelper = getattr(self, 'OutputHelper_' + self.outputFormat)(jarfile)
 328     register = {}
 329     # This loop exits on either
 330     # - the end of the jar.mn file
 331     # - an line in the jar.mn file that's not part of a jar section
 332     # - on an exception raised, close the jf in that case in a finally
 333     try:
 334       while True:
 335         try:
 336           l = lines.next()
 337         except StopIteration:
 338           # we're done with this jar.mn, and this jar section
 339           self.finalizeJar(jarfile, chromebasepath, register)
 340           if jf is not None:
 341             jf.close()
 342           # reraise the StopIteration for makeJar
 343           raise
 344         if self.ignore.match(l):
 345           continue
 346         m = self.regline.match(l)
 347         if  m:
 348           rline = m.group(1)
 349           register[rline] = 1
 350           continue
 351         m = self.entryline.match(l)
 352         if not m:
 353           # neither an entry line nor chrome reg, this jar section is done
 354           self.finalizeJar(jarfile, chromebasepath, register)
 355           if jf is not None:
 356             jf.close()
 357           lines.pushback(l)
 358           return
 359         self._processEntryLine(m, sourcedirs, topsourcedir, localedirs,
 360                               outHelper, jf)
 361     finally:
 362       if jf is not None:
 363         jf.close()
 364     return
 365
 366   def _processEntryLine(self, m,
 367                         sourcedirs, topsourcedir, localedirs,
 368                         outHelper, jf):
 369       out = m.group('output')
 370       src = m.group('source') or os.path.basename(out)
 371       # pick the right sourcedir -- l10n, topsrc or src
 372       if m.group('locale'):
 373         src_base = localedirs
 374       elif src.startswith('/'):
 375         # path/in/jar/file_name.xul     (/path/in/sourcetree/file_name.xul)
 376         # refers to a path relative to topsourcedir, use that as base
 377         # and strip the leading '/'
 378         src_base = [topsourcedir]
 379         src = src[1:]
 380       else:
 381         # use srcdirs and the objdir (current working dir) for relative paths
 382         src_base = sourcedirs + ['.']
 383       # check if the source file exists
 384       realsrc = None
 385       for _srcdir in src_base:
 386         if os.path.isfile(os.path.join(_srcdir, src)):
 387           realsrc = os.path.join(_srcdir, src)
 388           break
 389       if realsrc is None:
 390         if jf is not None:
 391           jf.close()
 392         raise RuntimeError('File "%s" not found in %s' % (src, ', '.join(src_base)))
 393       if m.group('optPreprocess'):
 394         outf = outHelper.getOutput(out)
 395         inf = open(realsrc)
 396         pp = self.pp.clone()
 397         if src[-4:] == '.css':
 398           pp.setMarker('%')
 399         pp.out = outf
 400         pp.do_include(inf)
 401         outf.close()
 402         inf.close()
 403         return
 404       # copy or symlink if newer or overwrite
 405       if (m.group('optOverwrite')
 406           or (getModTime(realsrc) >
 407               outHelper.getDestModTime(m.group('output')))):
 408         if self.outputFormat == 'symlink' and hasattr(os, 'symlink'):
 409           outHelper.symlink(realsrc, out)
 410           return
 411         outf = outHelper.getOutput(out)
 412         # open in binary mode, this can be images etc
 413         inf = open(realsrc, 'rb')
 414         outf.write(inf.read())
 415         outf.close()
 416         inf.close()
 417
 418
 419   class OutputHelper_jar(object):
 420     '''Provide getDestModTime and getOutput for a given jarfile.
 421     '''
 422     def __init__(self, jarfile):
 423       self.jarfile = jarfile
 424     def getDestModTime(self, aPath):
 425       try :
 426         info = self.jarfile.getinfo(aPath)
 427         return info.date_time
 428       except:
 429         return 0
 430     def getOutput(self, name):
 431       return ZipEntry(name, self.jarfile)
 432
 433   class OutputHelper_flat(object):
 434     '''Provide getDestModTime and getOutput for a given flat
 435     output directory. The helper method ensureDirFor is used by
 436     the symlink subclass.
 437     '''
 438     def __init__(self, basepath):
 439       self.basepath = basepath
 440     def getDestModTime(self, aPath):
 441       return getModTime(os.path.join(self.basepath, aPath))
 442     def getOutput(self, name):
 443       out = self.ensureDirFor(name)
 444       # remove previous link or file
 445       try:
 446         os.remove(out)
 447       except OSError, e:
 448         if e.errno != 2:
 449           raise
 450       return open(out, 'wb')
 451     def ensureDirFor(self, name):
 452       out = os.path.join(self.basepath, name)
 453       outdir = os.path.dirname(out)
 454       if not os.path.isdir(outdir):
 455         os.makedirs(outdir)
 456       return out
 457
 458   class OutputHelper_symlink(OutputHelper_flat):
 459     '''Subclass of OutputHelper_flat that provides a helper for
 460     creating a symlink including creating the parent directories.
 461     '''
 462     def symlink(self, src, dest):
 463       out = self.ensureDirFor(dest)
 464       # remove previous link or file
 465       try:
 466         os.remove(out)
 467       except OSError, e:
 468         if e.errno != 2:
 469           raise
 470       os.symlink(src, out)
 471
 472 def main():
 473   jm = JarMaker()
 474   p = jm.getCommandLineParser()
 475   (options, args) = p.parse_args()
 476   jm.processIncludes(options.I)
 477   jm.outputFormat = options.f
 478   if options.e:
 479     jm.useChromeManifest = True
 480     jm.useJarfileManifest = False
 481   if options.bothManifests:
 482     jm.useChromeManifest = True
 483     jm.useJarfileManifest = True
 484   noise = logging.INFO
 485   if options.verbose is not None:
 486     noise = (options.verbose and logging.DEBUG) or logging.WARN
 487   if sys.version_info[:2] > (2,3):
 488     logging.basicConfig(format = "%(message)s")
 489   else:
 490     logging.basicConfig()
 491   logging.getLogger().setLevel(noise)
 492   topsrc = options.t
 493   topsrc = os.path.normpath(os.path.abspath(topsrc))
 494   if not args:
 495     jm.makeJar(infile=sys.stdin,
 496                sourcedirs=options.s, topsourcedir=topsrc,
 497                localedirs=options.l10n_src,
 498                jardir=options.j)
 499   else:
 500     jm.makeJars(args, options.l10n_base,
 501                 jardir=options.j,
 502                 sourcedirs=options.s, topsourcedir=topsrc,
 503                 localedirs=options.l10n_src)
 504
 505 if __name__ == "__main__":
 506   main()