config/JarMaker.py

   1 # ***** BEGIN LICENSE BLOCK *****
   2 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   3 #
   4 # The contents of this file are subject to the Mozilla Public License Version
   5 # 1.1 (the "License"); you may not use this file except in compliance with
   6 # the License. You may obtain a copy of the License at
   7 # http://www.mozilla.org/MPL/
   8 #
   9 # Software distributed under the License is distributed on an "AS IS" basis,
  10 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  11 # for the specific language governing rights and limitations under the
  12 # License.
  13 #
  14 # The Original Code is Mozilla build system.
  15 #
  16 # The Initial Developer of the Original Code is
  17 # Mozilla Foundation.
  18 # Portions created by the Initial Developer are Copyright (C) 2008
  19 # the Initial Developer. All Rights Reserved.
  20 #
  21 # Contributor(s):
  22 #  Axel Hecht <l10n@mozilla.com>
  23 #
  24 # Alternatively, the contents of this file may be used under the terms of
  25 # either the GNU General Public License Version 2 or later (the "GPL"), or
  26 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  27 # in which case the provisions of the GPL or the LGPL are applicable instead
  28 # of those above. If you wish to allow use of your version of this file only
  29 # under the terms of either the GPL or the LGPL, and not to allow others to
  30 # use your version of this file under the terms of the MPL, indicate your
  31 # decision by deleting the provisions above and replace them with the notice
  32 # and other provisions required by the GPL or the LGPL. If you do not delete
  33 # the provisions above, a recipient may use your version of this file under
  34 # the terms of any one of the MPL, the GPL or the LGPL.
  35 #
  36 # ***** END LICENSE BLOCK *****
  37
  38 '''jarmaker.py provides a python class to package up chrome content by
  39 processing jar.mn files.
  40
  41 See the documentation for jar.mn on MDC for further details on the format.
  42 '''
  43
  44 import sys
  45 import os
  46 import os.path
  47 import re
  48 import logging
  49 from time import localtime
  50 from optparse import OptionParser
  51 from MozZipFile import ZipFile
  52 from cStringIO import StringIO
  53 from datetime import datetime
  54
  55 from utils import pushback_iter
  56 from Preprocessor import Preprocessor
  57
  58 __all__ = ['JarMaker']
  59
  60 class ZipEntry:
  61   '''Helper class for jar output.
  62
  63   This class defines a simple file-like object for a zipfile.ZipEntry
  64   so that we can consecutively write to it and then close it.
  65   This methods hooks into ZipFile.writestr on close().
  66   '''
  67   def __init__(self, name, zipfile):
  68     self._zipfile = zipfile
  69     self._name = name
  70     self._inner = StringIO()
  71
  72   def write(self, content):
  73     'Append the given content to this zip entry'
  74     self._inner.write(content)
  75     return
  76
  77   def close(self):
  78     'The close method writes the content back to the zip file.'
  79     self._zipfile.writestr(self._name, self._inner.getvalue())
  80
  81 def getModTime(aPath):
  82   if not os.path.isfile(aPath):
  83     return 0
  84   mtime = os.stat(aPath).st_mtime
  85   return localtime(mtime)
  86
  87
  88 class JarMaker(object):
  89   '''JarMaker reads jar.mn files and process those into jar files or
  90   flat directories, along with chrome.manifest files.
  91   '''
  92
  93   ignore = re.compile('\s*(\#.*)?$')
  94   jarline = re.compile('(?:(?P<jarfile>[\w\d.\-\_\\\/]+).jar\:)|(?:\s*(\#.*)?)\s*$')
  95   regline = re.compile('\%\s+(.*)$')
  96   entryre = '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+'
  97   entryline = re.compile(entryre + '(?P<output>[\w\d.\-\_\\\/\+]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/]+)\))?\s*$')
  98
  99   def __init__(self, outputFormat = 'flat', useJarfileManifest = True,
 100                useChromeManifest = False):
 101     self.outputFormat = outputFormat
 102     self.useJarfileManifest = useJarfileManifest
 103     self.useChromeManifest = useChromeManifest
 104     self.pp = Preprocessor()
 105
 106   def getCommandLineParser(self):
 107     '''Get a optparse.OptionParser for jarmaker.
 108
 109     This OptionParser has the options for jarmaker as well as
 110     the options for the inner PreProcessor.
 111     '''
 112     # HACK, we need to unescape the string variables we get,
 113     # the perl versions didn't grok strings right
 114     p = self.pp.getCommandLineParser(unescapeDefines = True)
 115     p.add_option('-f', type="choice", default="jar",
 116                  choices=('jar', 'flat', 'symlink'),
 117                  help="fileformat used for output", metavar="[jar, flat, symlink]")
 118     p.add_option('-v', action="store_true", dest="verbose",
 119                  help="verbose output")
 120     p.add_option('-q', action="store_false", dest="verbose",
 121                  help="verbose output")
 122     p.add_option('-e', action="store_true",
 123                  help="create chrome.manifest instead of jarfile.manifest")
 124     p.add_option('--both-manifests', action="store_true",
 125                  dest="bothManifests",
 126                  help="create chrome.manifest and jarfile.manifest")
 127     p.add_option('-s', type="string", action="append", default=[],
 128                  help="source directory")
 129     p.add_option('-t', type="string",
 130                  help="top source directory")
 131     p.add_option('-c', '--l10n-src', type="string", action="append",
 132                  help="localization directory")
 133     p.add_option('--l10n-base', type="string", action="append", default=[],
 134                  help="base directory to be used for localization (multiple)")
 135     p.add_option('-j', type="string",
 136                  help="jarfile directory")
 137     # backwards compat, not needed
 138     p.add_option('-a', action="store_false", default=True,
 139                  help="NOT SUPPORTED, turn auto-registration of chrome off (installed-chrome.txt)")
 140     p.add_option('-d', type="string",
 141                  help="UNUSED, chrome directory")
 142     p.add_option('-o', help="cross compile for auto-registration, ignored")
 143     p.add_option('-l', action="store_true",
 144                  help="ignored (used to switch off locks)")
 145     p.add_option('-x', action="store_true",
 146                  help="force Unix")
 147     p.add_option('-z', help="backwards compat, ignored")
 148     p.add_option('-p', help="backwards compat, ignored")
 149     return p
 150
 151   def processIncludes(self, includes):
 152     '''Process given includes with the inner PreProcessor.
 153
 154     Only use this for #defines, the includes shouldn't generate
 155     content.
 156     '''
 157     self.pp.out = StringIO()
 158     for inc in includes:
 159       self.pp.do_include(inc)
 160     includesvalue = self.pp.out.getvalue()
 161     if includesvalue:
 162       logging.info("WARNING: Includes produce non-empty output")
 163     self.pp.out = None
 164     pass
 165
 166   def finalizeJar(self, jarPath, chromebasepath, register,
 167                    doZip=True):
 168     '''Helper method to write out the chrome registration entries to
 169     jarfile.manifest or chrome.manifest, or both.
 170
 171     The actual file processing is done in updateManifest.
 172     '''
 173     # rewrite the manifest, if entries given
 174     if not register:
 175       return
 176     if self.useJarfileManifest:
 177       self.updateManifest(jarPath + '.manifest', chromebasepath % '',
 178                           register)
 179     if self.useChromeManifest:
 180       manifestPath = os.path.join(os.path.dirname(jarPath),
 181                                   '..', 'chrome.manifest')
 182       self.updateManifest(manifestPath, chromebasepath % 'chrome/',
 183                           register)
 184
 185   def updateManifest(self, manifestPath, chromebasepath, register):
 186     '''updateManifest replaces the % in the chrome registration entries
 187     with the given chrome base path, and updates the given manifest file.
 188     '''
 189     myregister = dict.fromkeys(map(lambda s: s.replace('%', chromebasepath),
 190                                    register.iterkeys()))
 191     manifestExists = os.path.isfile(manifestPath)
 192     mode = (manifestExists and 'r+b') or 'wb'
 193     mf = open(manifestPath, mode)
 194     if manifestExists:
 195       # import previous content into hash, ignoring empty ones and comments
 196       imf = re.compile('(#.*)?$')
 197       for l in re.split('[\r\n]+', mf.read()):
 198         if imf.match(l):
 199           continue
 200         myregister[l] = None
 201       mf.seek(0)
 202     for k in myregister.iterkeys():
 203       mf.write(k + os.linesep)
 204     mf.close()
 205
 206   def makeJar(self, infile=None,
 207                jardir='',
 208                sourcedirs=[], topsourcedir='', localedirs=None):
 209     '''makeJar is the main entry point to JarMaker.
 210
 211     It takes the input file, the output directory, the source dirs and the
 212     top source dir as argument, and optionally the l10n dirs.
 213     '''
 214     if isinstance(infile, basestring):
 215       logging.info("processing " + infile)
 216     pp = self.pp.clone()
 217     pp.out = StringIO()
 218     pp.do_include(infile)
 219     lines = pushback_iter(pp.out.getvalue().splitlines())
 220     try:
 221       while True:
 222         l = lines.next()
 223         m = self.jarline.match(l)
 224         if not m:
 225           raise RuntimeError(l)
 226         if m.group('jarfile') is None:
 227           # comment
 228           continue
 229         self.processJarSection(m.group('jarfile'), lines,
 230                                jardir, sourcedirs, topsourcedir,
 231                                localedirs)
 232     except StopIteration:
 233       # we read the file
 234       pass
 235     return
 236
 237   def processJarSection(self, jarfile, lines,
 238                         jardir, sourcedirs, topsourcedir, localedirs):
 239     '''Internal method called by makeJar to actually process a section
 240     of a jar.mn file.
 241
 242     jarfile is the basename of the jarfile or the directory name for
 243     flat output, lines is a pushback_iterator of the lines of jar.mn,
 244     the remaining options are carried over from makeJar.
 245     '''
 246
 247     # chromebasepath is used for chrome registration manifests
 248     # %s is getting replaced with chrome/ for chrome.manifest, and with
 249     # an empty string for jarfile.manifest
 250     chromebasepath = '%s' + jarfile
 251     if self.outputFormat == 'jar':
 252       chromebasepath = 'jar:' + chromebasepath + '.jar!'
 253     chromebasepath += '/'
 254
 255     jarfile = os.path.join(jardir, jarfile)
 256     jf = None
 257     if self.outputFormat == 'jar':
 258       #jar
 259       jarfilepath = jarfile + '.jar'
 260       if os.path.isfile(jarfilepath) and \
 261             os.path.getsize(jarfilepath) > 0:
 262         jf = ZipFile(jarfilepath, 'a', lock = True)
 263       else:
 264         if not os.path.isdir(os.path.dirname(jarfilepath)):
 265           os.makedirs(os.path.dirname(jarfilepath))
 266         jf = ZipFile(jarfilepath, 'w', lock = True)
 267       outHelper = self.OutputHelper_jar(jf)
 268     else:
 269       outHelper = getattr(self, 'OutputHelper_' + self.outputFormat)(jarfile)
 270     register = {}
 271     # This loop exits on either
 272     # - the end of the jar.mn file
 273     # - an line in the jar.mn file that's not part of a jar section
 274     # - on an exception raised, close the jf in that case in a finally
 275     try:
 276       while True:
 277         try:
 278           l = lines.next()
 279         except StopIteration:
 280           # we're done with this jar.mn, and this jar section
 281           self.finalizeJar(jarfile, chromebasepath, register)
 282           if jf is not None:
 283             jf.close()
 284           # reraise the StopIteration for makeJar
 285           raise
 286         if self.ignore.match(l):
 287           continue
 288         m = self.regline.match(l)
 289         if  m:
 290           rline = m.group(1)
 291           register[rline] = 1
 292           continue
 293         m = self.entryline.match(l)
 294         if not m:
 295           # neither an entry line nor chrome reg, this jar section is done
 296           self.finalizeJar(jarfile, chromebasepath, register)
 297           if jf is not None:
 298             jf.close()
 299           lines.pushback(l)
 300           return
 301         self._processEntryLine(m, sourcedirs, topsourcedir, localedirs,
 302                               outHelper, jf)
 303     finally:
 304       if jf is not None:
 305         jf.close()
 306     return
 307
 308   def _processEntryLine(self, m,
 309                         sourcedirs, topsourcedir, localedirs,
 310                         outHelper, jf):
 311       out = m.group('output')
 312       src = m.group('source') or os.path.basename(out)
 313       # pick the right sourcedir -- l10n, topsrc or src
 314       if m.group('locale'):
 315         src_base = localedirs
 316       elif src.startswith('/'):
 317         # path/in/jar/file_name.xul     (/path/in/sourcetree/file_name.xul)
 318         # refers to a path relative to topsourcedir, use that as base
 319         # and strip the leading '/'
 320         src_base = [topsourcedir]
 321         src = src[1:]
 322       else:
 323         # use srcdirs and the objdir (current working dir) for relative paths
 324         src_base = sourcedirs + ['.']
 325       # check if the source file exists
 326       realsrc = None
 327       for _srcdir in src_base:
 328         if os.path.isfile(os.path.join(_srcdir, src)):
 329           realsrc = os.path.join(_srcdir, src)
 330           break
 331       if realsrc is None:
 332         if jf is not None:
 333           jf.close()
 334         raise RuntimeError("file not found: " + src)
 335       if m.group('optPreprocess'):
 336         outf = outHelper.getOutput(out)
 337         inf = open(realsrc)
 338         pp = self.pp.clone()
 339         if src[-4:] == '.css':
 340           pp.setMarker('%')
 341         pp.out = outf
 342         pp.do_include(inf)
 343         outf.close()
 344         inf.close()
 345         return
 346       # copy or symlink if newer or overwrite
 347       if (m.group('optOverwrite')
 348           or (getModTime(realsrc) >
 349               outHelper.getDestModTime(m.group('output')))):
 350         if self.outputFormat == 'symlink' and hasattr(os, 'symlink'):
 351           outHelper.symlink(realsrc, out)
 352           return
 353         outf = outHelper.getOutput(out)
 354         # open in binary mode, this can be images etc
 355         inf = open(realsrc, 'rb')
 356         outf.write(inf.read())
 357         outf.close()
 358         inf.close()
 359
 360
 361   class OutputHelper_jar(object):
 362     '''Provide getDestModTime and getOutput for a given jarfile.
 363     '''
 364     def __init__(self, jarfile):
 365       self.jarfile = jarfile
 366     def getDestModTime(self, aPath):
 367       try :
 368         info = self.jarfile.getinfo(aPath)
 369         return info.date_time
 370       except:
 371         return 0
 372     def getOutput(self, name):
 373       return ZipEntry(name, self.jarfile)
 374
 375   class OutputHelper_flat(object):
 376     '''Provide getDestModTime and getOutput for a given flat
 377     output directory. The helper method ensureDirFor is used by
 378     the symlink subclass.
 379     '''
 380     def __init__(self, basepath):
 381       self.basepath = basepath
 382     def getDestModTime(self, aPath):
 383       return getModTime(os.path.join(self.basepath, aPath))
 384     def getOutput(self, name):
 385       out = self.ensureDirFor(name)
 386       # remove previous link or file
 387       try:
 388         os.remove(out)
 389       except OSError, e:
 390         if e.errno != 2:
 391           raise
 392       return open(out, 'wb')
 393     def ensureDirFor(self, name):
 394       out = os.path.join(self.basepath, name)
 395       outdir = os.path.dirname(out)
 396       if not os.path.isdir(outdir):
 397         os.makedirs(outdir)
 398       return out
 399
 400   class OutputHelper_symlink(OutputHelper_flat):
 401     '''Subclass of OutputHelper_flat that provides a helper for
 402     creating a symlink including creating the parent directories.
 403     '''
 404     def symlink(self, src, dest):
 405       out = self.ensureDirFor(dest)
 406       # remove previous link or file
 407       try:
 408         os.remove(out)
 409       except OSError, e:
 410         if e.errno != 2:
 411           raise
 412       os.symlink(src, out)
 413
 414 def main():
 415   jm = JarMaker()
 416   p = jm.getCommandLineParser()
 417   (options, args) = p.parse_args()
 418   jm.processIncludes(options.I)
 419   jm.outputFormat = options.f
 420   if options.e:
 421     jm.useChromeManifest = True
 422     jm.useJarfileManifest = False
 423   if options.bothManifests:
 424     jm.useChromeManifest = True
 425     jm.useJarfileManifest = True
 426   noise = logging.INFO
 427   if options.verbose is not None:
 428     noise = (options.verbose and logging.DEBUG) or logging.WARN
 429   if sys.version_info[:2] > (2,3):
 430     logging.basicConfig(format = "%(message)s")
 431   else:
 432     logging.basicConfig()
 433   logging.getLogger().setLevel(noise)
 434   if not args:
 435     jm.makeJar(infile=sys.stdin,
 436                sourcedirs=options.s, topsourcedir=options.t,
 437                localedirs=options.l10n_src,
 438                jardir=options.j)
 439     return
 440   topsrc = options.t
 441   topsrc = os.path.normpath(os.path.abspath(topsrc))
 442   for infile in args:
 443     # guess srcdir and l10n dirs from jar.mn path and topsrcdir
 444     # srcdir is the dir of the jar.mn and
 445     # the l10n dirs are the relative path of topsrcdir to srcdir
 446     # resolved against all l10n base dirs.
 447     srcdir = os.path.normpath(os.path.abspath(os.path.dirname(infile)))
 448     l10ndir = srcdir
 449     if os.path.basename(srcdir) == 'locales':
 450       l10ndir = os.path.dirname(l10ndir)
 451     assert srcdir.startswith(topsrc), "src dir %s not in topsrcdir %s" % (srcdir, topsrc)
 452     rell10ndir = l10ndir[len(topsrc):].lstrip(os.sep)
 453     l10ndirs = map(lambda d: os.path.join(d, rell10ndir), options.l10n_base)
 454     if options.l10n_src is not None:
 455       l10ndirs += options.l10n_src
 456     srcdirs = options.s + [srcdir]
 457     jm.makeJar(infile=infile,
 458                sourcedirs=srcdirs, topsourcedir=options.t,
 459                localedirs=l10ndirs,
 460                jardir=options.j)
 461
 462 if __name__ == "__main__":
 463   main()