config/JarMaker.py

   1 # ***** BEGIN LICENSE BLOCK *****
   2 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   3 #
   4 # The contents of this file are subject to the Mozilla Public License Version
   5 # 1.1 (the "License"); you may not use this file except in compliance with
   6 # the License. You may obtain a copy of the License at
   7 # http://www.mozilla.org/MPL/
   8 #
   9 # Software distributed under the License is distributed on an "AS IS" basis,
  10 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  11 # for the specific language governing rights and limitations under the
  12 # License.
  13 #
  14 # The Original Code is Mozilla build system.
  15 #
  16 # The Initial Developer of the Original Code is
  17 # Mozilla Foundation.
  18 # Portions created by the Initial Developer are Copyright (C) 2008
  19 # the Initial Developer. All Rights Reserved.
  20 #
  21 # Contributor(s):
  22 #  Axel Hecht <l10n@mozilla.com>
  23 #
  24 # Alternatively, the contents of this file may be used under the terms of
  25 # either the GNU General Public License Version 2 or later (the "GPL"), or
  26 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  27 # in which case the provisions of the GPL or the LGPL are applicable instead
  28 # of those above. If you wish to allow use of your version of this file only
  29 # under the terms of either the GPL or the LGPL, and not to allow others to
  30 # use your version of this file under the terms of the MPL, indicate your
  31 # decision by deleting the provisions above and replace them with the notice
  32 # and other provisions required by the GPL or the LGPL. If you do not delete
  33 # the provisions above, a recipient may use your version of this file under
  34 # the terms of any one of the MPL, the GPL or the LGPL.
  35 #
  36 # ***** END LICENSE BLOCK *****
  37
  38 '''jarmaker.py provides a python class to package up chrome content by
  39 processing jar.mn files.
  40
  41 See the documentation for jar.mn on MDC for further details on the format.
  42 '''
  43
  44 import sys
  45 import os
  46 import os.path
  47 import re
  48 import logging
  49 from time import localtime
  50 from optparse import OptionParser
  51 from MozZipFile import ZipFile
  52 from cStringIO import StringIO
  53 from datetime import datetime
  54
  55 from utils import pushback_iter
  56 from Preprocessor import Preprocessor
  57
  58 __all__ = ['JarMaker']
  59
  60 class ZipEntry:
  61   '''Helper class for jar output.
  62
  63   This class defines a simple file-like object for a zipfile.ZipEntry
  64   so that we can consecutively write to it and then close it.
  65   This methods hooks into ZipFile.writestr on close().
  66   '''
  67   def __init__(self, name, zipfile):
  68     self._zipfile = zipfile
  69     self._name = name
  70     self._inner = StringIO()
  71
  72   def write(self, content):
  73     'Append the given content to this zip entry'
  74     self._inner.write(content)
  75     return
  76
  77   def close(self):
  78     'The close method writes the content back to the zip file.'
  79     self._zipfile.writestr(self._name, self._inner.getvalue())
  80
  81 def getModTime(aPath):
  82   if not os.path.isfile(aPath):
  83     return 0
  84   mtime = os.stat(aPath).st_mtime
  85   return localtime(mtime)
  86
  87
  88 class JarMaker(object):
  89   '''JarMaker reads jar.mn files and process those into jar files or
  90   flat directories, along with chrome.manifest files.
  91   '''
  92
  93   ignore = re.compile('\s*(\#.*)?$')
  94   jarline = re.compile('(?:(?P<jarfile>[\w\d.\-\_\\\/]+).jar\:)|(?:\s*(\#.*)?)\s*$')
  95   regline = re.compile('\%\s+(.*)$')
  96   entryre = '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+'
  97   entryline = re.compile(entryre + '(?P<output>[\w\d.\-\_\\\/\+]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/]+)\))?\s*$')
  98
  99   def __init__(self, outputFormat = 'flat', useJarfileManifest = True,
 100                useChromeManifest = False):
 101     self.outputFormat = outputFormat
 102     self.useJarfileManifest = useJarfileManifest
 103     self.useChromeManifest = useChromeManifest
 104     self.pp = Preprocessor()
 105
 106   def getCommandLineParser(self):
 107     '''Get a optparse.OptionParser for jarmaker.
 108
 109     This OptionParser has the options for jarmaker as well as
 110     the options for the inner PreProcessor.
 111     '''
 112     # HACK, we need to unescape the string variables we get,
 113     # the perl versions didn't grok strings right
 114     p = self.pp.getCommandLineParser(unescapeDefines = True)
 115     p.add_option('-f', type="choice", default="jar",
 116                  choices=('jar', 'flat', 'symlink'),
 117                  help="fileformat used for output", metavar="[jar, flat, symlink]")
 118     p.add_option('-v', action="store_true", dest="verbose",
 119                  help="verbose output")
 120     p.add_option('-q', action="store_false", dest="verbose",
 121                  help="verbose output")
 122     p.add_option('-e', action="store_true",
 123                  help="create chrome.manifest instead of jarfile.manifest")
 124     p.add_option('--both-manifests', action="store_true",
 125                  dest="bothManifests",
 126                  help="create chrome.manifest and jarfile.manifest")
 127     p.add_option('-s', type="string", action="append", default=[],
 128                  help="source directory")
 129     p.add_option('-t', type="string",
 130                  help="top source directory")
 131     p.add_option('-c', '--l10n-src', type="string", action="append",
 132                  help="localization directory")
 133     p.add_option('--l10n-base', type="string", action="append", default=[],
 134                  help="base directory to be used for localization (multiple)")
 135     p.add_option('-j', type="string",
 136                  help="jarfile directory")
 137     # backwards compat, not needed
 138     p.add_option('-a', action="store_false", default=True,
 139                  help="NOT SUPPORTED, turn auto-registration of chrome off (installed-chrome.txt)")
 140     p.add_option('-d', type="string",
 141                  help="UNUSED, chrome directory")
 142     p.add_option('-o', help="cross compile for auto-registration, ignored")
 143     p.add_option('-l', action="store_true",
 144                  help="ignored (used to switch off locks)")
 145     p.add_option('-x', action="store_true",
 146                  help="force Unix")
 147     p.add_option('-z', help="backwards compat, ignored")
 148     p.add_option('-p', help="backwards compat, ignored")
 149     return p
 150
 151   def processIncludes(self, includes):
 152     '''Process given includes with the inner PreProcessor.
 153
 154     Only use this for #defines, the includes shouldn't generate
 155     content.
 156     '''
 157     self.pp.out = StringIO()
 158     for inc in includes:
 159       self.pp.do_include(inc)
 160     includesvalue = self.pp.out.getvalue()
 161     if includesvalue:
 162       logging.info("WARNING: Includes produce non-empty output")
 163     self.pp.out = None
 164     pass
 165
 166   def finalizeJar(self, jarPath, chromebasepath, register,
 167                    doZip=True):
 168     '''Helper method to write out the chrome registration entries to
 169     jarfile.manifest or chrome.manifest, or both.
 170
 171     The actual file processing is done in updateManifest.
 172     '''
 173     # rewrite the manifest, if entries given
 174     if not register:
 175       return
 176     if self.useJarfileManifest:
 177       self.updateManifest(jarPath + '.manifest', chromebasepath % '',
 178                           register)
 179     if self.useChromeManifest:
 180       manifestPath = os.path.join(os.path.dirname(jarPath),
 181                                   '..', 'chrome.manifest')
 182       self.updateManifest(manifestPath, chromebasepath % 'chrome/',
 183                           register)
 184
 185   def updateManifest(self, manifestPath, chromebasepath, register):
 186     '''updateManifest replaces the % in the chrome registration entries
 187     with the given chrome base path, and updates the given manifest file.
 188     '''
 189     myregister = dict.fromkeys(map(lambda s: s.replace('%', chromebasepath),
 190                                    register.iterkeys()))
 191     manifestExists = os.path.isfile(manifestPath)
 192     mode = (manifestExists and 'r+b') or 'wb'
 193     mf = open(manifestPath, mode)
 194     if manifestExists:
 195       # import previous content into hash, ignoring empty ones and comments
 196       imf = re.compile('(#.*)?$')
 197       for l in re.split('[\r\n]+', mf.read()):
 198         if imf.match(l):
 199           continue
 200         myregister[l] = None
 201       mf.seek(0)
 202     for k in myregister.iterkeys():
 203       mf.write(k + os.linesep)
 204     mf.close()
 205
 206   def makeJar(self, infile=None,
 207                jardir='',
 208                sourcedirs=[], topsourcedir='', localedirs=None):
 209     '''makeJar is the main entry point to JarMaker.
 210
 211     It takes the input file, the output directory, the source dirs and the
 212     top source dir as argument, and optionally the l10n dirs.
 213     '''
 214     if isinstance(infile, basestring):
 215       logging.info("processing " + infile)
 216     pp = self.pp.clone()
 217     pp.out = StringIO()
 218     pp.do_include(infile)
 219     lines = pushback_iter(pp.out.getvalue().splitlines())
 220     try:
 221       while True:
 222         l = lines.next()
 223         m = self.jarline.match(l)
 224         if not m:
 225           raise RuntimeError(l)
 226         if m.group('jarfile') is None:
 227           # comment
 228           continue
 229         self.processJarSection(m.group('jarfile'), lines,
 230                                jardir, sourcedirs, topsourcedir,
 231                                localedirs)
 232     except StopIteration:
 233       # we read the file
 234       pass
 235     return
 236
 237   def processJarSection(self, jarfile, lines,
 238                         jardir, sourcedirs, topsourcedir, localedirs):
 239     '''Internal method called by makeJar to actually process a section
 240     of a jar.mn file.
 241
 242     jarfile is the basename of the jarfile or the directory name for
 243     flat output, lines is a pushback_iterator of the lines of jar.mn,
 244     the remaining options are carried over from makeJar.
 245     '''
 246
 247     # chromebasepath is used for chrome registration manifests
 248     # %s is getting replaced with chrome/ for chrome.manifest, and with
 249     # an empty string for jarfile.manifest
 250     chromebasepath = '%s' + jarfile
 251     if self.outputFormat == 'jar':
 252       chromebasepath = 'jar:' + chromebasepath + '.jar!'
 253     chromebasepath += '/'
 254
 255     jarfile = os.path.join(jardir, jarfile)
 256     jf = None
 257     if self.outputFormat == 'jar':
 258       #jar
 259       jarfilepath = jarfile + '.jar'
 260       try:
 261         os.makedirs(os.path.dirname(jarfilepath))
 262       except OSError:
 263         pass
 264       jf = ZipFile(jarfilepath, 'a', lock = True)
 265       outHelper = self.OutputHelper_jar(jf)
 266     else:
 267       outHelper = getattr(self, 'OutputHelper_' + self.outputFormat)(jarfile)
 268     register = {}
 269     # This loop exits on either
 270     # - the end of the jar.mn file
 271     # - an line in the jar.mn file that's not part of a jar section
 272     # - on an exception raised, close the jf in that case in a finally
 273     try:
 274       while True:
 275         try:
 276           l = lines.next()
 277         except StopIteration:
 278           # we're done with this jar.mn, and this jar section
 279           self.finalizeJar(jarfile, chromebasepath, register)
 280           if jf is not None:
 281             jf.close()
 282           # reraise the StopIteration for makeJar
 283           raise
 284         if self.ignore.match(l):
 285           continue
 286         m = self.regline.match(l)
 287         if  m:
 288           rline = m.group(1)
 289           register[rline] = 1
 290           continue
 291         m = self.entryline.match(l)
 292         if not m:
 293           # neither an entry line nor chrome reg, this jar section is done
 294           self.finalizeJar(jarfile, chromebasepath, register)
 295           if jf is not None:
 296             jf.close()
 297           lines.pushback(l)
 298           return
 299         self._processEntryLine(m, sourcedirs, topsourcedir, localedirs,
 300                               outHelper, jf)
 301     finally:
 302       if jf is not None:
 303         jf.close()
 304     return
 305
 306   def _processEntryLine(self, m,
 307                         sourcedirs, topsourcedir, localedirs,
 308                         outHelper, jf):
 309       out = m.group('output')
 310       src = m.group('source') or os.path.basename(out)
 311       # pick the right sourcedir -- l10n, topsrc or src
 312       if m.group('locale'):
 313         src_base = localedirs
 314       elif src.startswith('/'):
 315         # path/in/jar/file_name.xul     (/path/in/sourcetree/file_name.xul)
 316         # refers to a path relative to topsourcedir, use that as base
 317         # and strip the leading '/'
 318         src_base = [topsourcedir]
 319         src = src[1:]
 320       else:
 321         # use srcdirs and the objdir (current working dir) for relative paths
 322         src_base = sourcedirs + ['.']
 323       # check if the source file exists
 324       realsrc = None
 325       for _srcdir in src_base:
 326         if os.path.isfile(os.path.join(_srcdir, src)):
 327           realsrc = os.path.join(_srcdir, src)
 328           break
 329       if realsrc is None:
 330         if jf is not None:
 331           jf.close()
 332         raise RuntimeError("file not found: " + src)
 333       if m.group('optPreprocess'):
 334         outf = outHelper.getOutput(out)
 335         inf = open(realsrc)
 336         pp = self.pp.clone()
 337         if src[-4:] == '.css':
 338           pp.setMarker('%')
 339         pp.out = outf
 340         pp.do_include(inf)
 341         outf.close()
 342         inf.close()
 343         return
 344       # copy or symlink if newer or overwrite
 345       if (m.group('optOverwrite')
 346           or (getModTime(realsrc) >
 347               outHelper.getDestModTime(m.group('output')))):
 348         if self.outputFormat == 'symlink' and hasattr(os, 'symlink'):
 349           outHelper.symlink(realsrc, out)
 350           return
 351         outf = outHelper.getOutput(out)
 352         # open in binary mode, this can be images etc
 353         inf = open(realsrc, 'rb')
 354         outf.write(inf.read())
 355         outf.close()
 356         inf.close()
 357
 358
 359   class OutputHelper_jar(object):
 360     '''Provide getDestModTime and getOutput for a given jarfile.
 361     '''
 362     def __init__(self, jarfile):
 363       self.jarfile = jarfile
 364     def getDestModTime(self, aPath):
 365       try :
 366         info = self.jarfile.getinfo(aPath)
 367         return info.date_time
 368       except:
 369         return 0
 370     def getOutput(self, name):
 371       return ZipEntry(name, self.jarfile)
 372
 373   class OutputHelper_flat(object):
 374     '''Provide getDestModTime and getOutput for a given flat
 375     output directory. The helper method ensureDirFor is used by
 376     the symlink subclass.
 377     '''
 378     def __init__(self, basepath):
 379       self.basepath = basepath
 380     def getDestModTime(self, aPath):
 381       return getModTime(os.path.join(self.basepath, aPath))
 382     def getOutput(self, name):
 383       out = self.ensureDirFor(name)
 384       # remove previous link or file
 385       try:
 386         os.remove(out)
 387       except OSError, e:
 388         if e.errno != 2:
 389           raise
 390       return open(out, 'wb')
 391     def ensureDirFor(self, name):
 392       out = os.path.join(self.basepath, name)
 393       outdir = os.path.dirname(out)
 394       if not os.path.isdir(outdir):
 395         os.makedirs(outdir)
 396       return out
 397
 398   class OutputHelper_symlink(OutputHelper_flat):
 399     '''Subclass of OutputHelper_flat that provides a helper for
 400     creating a symlink including creating the parent directories.
 401     '''
 402     def symlink(self, src, dest):
 403       out = self.ensureDirFor(dest)
 404       # remove previous link or file
 405       try:
 406         os.remove(out)
 407       except OSError, e:
 408         if e.errno != 2:
 409           raise
 410       os.symlink(src, out)
 411
 412 def main():
 413   jm = JarMaker()
 414   p = jm.getCommandLineParser()
 415   (options, args) = p.parse_args()
 416   jm.processIncludes(options.I)
 417   jm.outputFormat = options.f
 418   if options.e:
 419     jm.useChromeManifest = True
 420     jm.useJarfileManifest = False
 421   if options.bothManifests:
 422     jm.useChromeManifest = True
 423     jm.useJarfileManifest = True
 424   noise = logging.INFO
 425   if options.verbose is not None:
 426     noise = (options.verbose and logging.DEBUG) or logging.WARN
 427   if sys.version_info[:2] > (2,3):
 428     logging.basicConfig(format = "%(message)s")
 429   else:
 430     logging.basicConfig()
 431   logging.getLogger().setLevel(noise)
 432   if not args:
 433     jm.makeJar(infile=sys.stdin,
 434                sourcedirs=options.s, topsourcedir=options.t,
 435                localedirs=options.l10n_src,
 436                jardir=options.j)
 437     return
 438   topsrc = options.t
 439   topsrc = os.path.normpath(os.path.abspath(topsrc))
 440   for infile in args:
 441     # guess srcdir and l10n dirs from jar.mn path and topsrcdir
 442     # srcdir is the dir of the jar.mn and
 443     # the l10n dirs are the relative path of topsrcdir to srcdir
 444     # resolved against all l10n base dirs.
 445     srcdir = os.path.normpath(os.path.abspath(os.path.dirname(infile)))
 446     l10ndir = srcdir
 447     if os.path.basename(srcdir) == 'locales':
 448       l10ndir = os.path.dirname(l10ndir)
 449     assert srcdir.startswith(topsrc), "src dir %s not in topsrcdir %s" % (srcdir, topsrc)
 450     rell10ndir = l10ndir[len(topsrc):].lstrip(os.sep)
 451     l10ndirs = map(lambda d: os.path.join(d, rell10ndir), options.l10n_base)
 452     if options.l10n_src is not None:
 453       l10ndirs += options.l10n_src
 454     srcdirs = options.s + [srcdir]
 455     jm.makeJar(infile=infile,
 456                sourcedirs=srcdirs, topsourcedir=options.t,
 457                localedirs=l10ndirs,
 458                jardir=options.j)
 459
 460 if __name__ == "__main__":
 461   main()