config/JarMaker.py

   1 # ***** BEGIN LICENSE BLOCK *****
   2 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   3 #
   4 # The contents of this file are subject to the Mozilla Public License Version
   5 # 1.1 (the "License"); you may not use this file except in compliance with
   6 # the License. You may obtain a copy of the License at
   7 # http://www.mozilla.org/MPL/
   8 #
   9 # Software distributed under the License is distributed on an "AS IS" basis,
  10 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  11 # for the specific language governing rights and limitations under the
  12 # License.
  13 #
  14 # The Original Code is Mozilla build system.
  15 #
  16 # The Initial Developer of the Original Code is
  17 # Mozilla Foundation.
  18 # Portions created by the Initial Developer are Copyright (C) 2008
  19 # the Initial Developer. All Rights Reserved.
  20 #
  21 # Contributor(s):
  22 #  Axel Hecht <l10n@mozilla.com>
  23 #
  24 # Alternatively, the contents of this file may be used under the terms of
  25 # either the GNU General Public License Version 2 or later (the "GPL"), or
  26 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  27 # in which case the provisions of the GPL or the LGPL are applicable instead
  28 # of those above. If you wish to allow use of your version of this file only
  29 # under the terms of either the GPL or the LGPL, and not to allow others to
  30 # use your version of this file under the terms of the MPL, indicate your
  31 # decision by deleting the provisions above and replace them with the notice
  32 # and other provisions required by the GPL or the LGPL. If you do not delete
  33 # the provisions above, a recipient may use your version of this file under
  34 # the terms of any one of the MPL, the GPL or the LGPL.
  35 #
  36 # ***** END LICENSE BLOCK *****
  37
  38 '''jarmaker.py provides a python class to package up chrome content by
  39 processing jar.mn files.
  40
  41 See the documentation for jar.mn on MDC for further details on the format.
  42 '''
  43
  44 import sys
  45 import os
  46 import os.path
  47 import re
  48 import logging
  49 from time import localtime
  50 from optparse import OptionParser
  51 from MozZipFile import ZipFile
  52 from cStringIO import StringIO
  53 from datetime import datetime
  54
  55 from utils import pushback_iter
  56 from Preprocessor import Preprocessor
  57
  58 __all__ = ['JarMaker']
  59
  60 class ZipEntry:
  61   '''Helper class for jar output.
  62
  63   This class defines a simple file-like object for a zipfile.ZipEntry
  64   so that we can consecutively write to it and then close it.
  65   This methods hooks into ZipFile.writestr on close().
  66   '''
  67   def __init__(self, name, zipfile):
  68     self._zipfile = zipfile
  69     self._name = name
  70     self._inner = StringIO()
  71
  72   def write(self, content):
  73     'Append the given content to this zip entry'
  74     self._inner.write(content)
  75     return
  76
  77   def close(self):
  78     'The close method writes the content back to the zip file.'
  79     self._zipfile.writestr(self._name, self._inner.getvalue())
  80
  81 def getModTime(aPath):
  82   if not os.path.isfile(aPath):
  83     return 0
  84   mtime = os.stat(aPath).st_mtime
  85   return localtime(mtime)
  86
  87
  88 class JarMaker(object):
  89   '''JarMaker reads jar.mn files and process those into jar files or
  90   flat directories, along with chrome.manifest files.
  91   '''
  92
  93   ignore = re.compile('\s*(\#.*)?$')
  94   jarline = re.compile('(?:(?P<jarfile>[\w\d.\-\_\\\/]+).jar\:)|(?:\s*(\#.*)?)\s*$')
  95   regline = re.compile('\%\s+(.*)$')
  96   entryre = '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+'
  97   entryline = re.compile(entryre + '(?P<output>[\w\d.\-\_\\\/\+]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/]+)\))?\s*$')
  98
  99   def __init__(self, outputFormat = 'flat', useJarfileManifest = True,
 100                useChromeManifest = False):
 101     self.outputFormat = outputFormat
 102     self.useJarfileManifest = useJarfileManifest
 103     self.useChromeManifest = useChromeManifest
 104     self.pp = Preprocessor()
 105
 106   def getCommandLineParser(self):
 107     '''Get a optparse.OptionParser for jarmaker.
 108
 109     This OptionParser has the options for jarmaker as well as
 110     the options for the inner PreProcessor.
 111     '''
 112     # HACK, we need to unescape the string variables we get,
 113     # the perl versions didn't grok strings right
 114     p = self.pp.getCommandLineParser(unescapeDefines = True)
 115     p.add_option('-f', type="choice", default="jar",
 116                  choices=('jar', 'flat', 'symlink'),
 117                  help="fileformat used for output", metavar="[jar, flat, symlink]")
 118     p.add_option('-v', action="store_true", dest="verbose",
 119                  help="verbose output")
 120     p.add_option('-q', action="store_false", dest="verbose",
 121                  help="verbose output")
 122     p.add_option('-e', action="store_true",
 123                  help="create chrome.manifest instead of jarfile.manifest")
 124     p.add_option('--both-manifests', action="store_true",
 125                  dest="bothManifests",
 126                  help="create chrome.manifest and jarfile.manifest")
 127     p.add_option('-s', type="string", action="append", default=[],
 128                  help="source directory")
 129     p.add_option('-t', type="string",
 130                  help="top source directory")
 131     p.add_option('-c', '--l10n-src', type="string", action="append",
 132                  help="localization directory")
 133     p.add_option('--l10n-base', type="string", action="append", default=[],
 134                  help="base directory to be used for localization (multiple)")
 135     p.add_option('-j', type="string",
 136                  help="jarfile directory")
 137     # backwards compat, not needed
 138     p.add_option('-a', action="store_false", default=True,
 139                  help="NOT SUPPORTED, turn auto-registration of chrome off (installed-chrome.txt)")
 140     p.add_option('-d', type="string",
 141                  help="UNUSED, chrome directory")
 142     p.add_option('-o', help="cross compile for auto-registration, ignored")
 143     p.add_option('-l', action="store_true",
 144                  help="ignored (used to switch off locks)")
 145     p.add_option('-x', action="store_true",
 146                  help="force Unix")
 147     p.add_option('-z', help="backwards compat, ignored")
 148     p.add_option('-p', help="backwards compat, ignored")
 149     return p
 150
 151   def processIncludes(self, includes):
 152     '''Process given includes with the inner PreProcessor.
 153
 154     Only use this for #defines, the includes shouldn't generate
 155     content.
 156     '''
 157     self.pp.out = StringIO()
 158     for inc in includes:
 159       self.pp.do_include(inc)
 160     includesvalue = self.pp.out.getvalue()
 161     if includesvalue:
 162       logging.info("WARNING: Includes produce non-empty output")
 163     self.pp.out = None
 164     pass
 165
 166   def finalizeJar(self, jarPath, chromebasepath, register,
 167                    doZip=True):
 168     '''Helper method to write out the chrome registration entries to
 169     jarfile.manifest or chrome.manifest, or both.
 170
 171     The actual file processing is done in updateManifest.
 172     '''
 173     # rewrite the manifest, if entries given
 174     if not register:
 175       return
 176     if self.useJarfileManifest:
 177       self.updateManifest(jarPath + '.manifest', chromebasepath % '',
 178                           register)
 179     if self.useChromeManifest:
 180       manifestPath = os.path.join(os.path.dirname(jarPath),
 181                                   '..', 'chrome.manifest')
 182       self.updateManifest(manifestPath, chromebasepath % 'chrome/',
 183                           register)
 184
 185   def updateManifest(self, manifestPath, chromebasepath, register):
 186     '''updateManifest replaces the % in the chrome registration entries
 187     with the given chrome base path, and updates the given manifest file.
 188     '''
 189     myregister = dict.fromkeys(map(lambda s: s.replace('%', chromebasepath),
 190                                    register.iterkeys()))
 191     manifestExists = os.path.isfile(manifestPath)
 192     mode = (manifestExists and 'r+b') or 'wb'
 193     mf = open(manifestPath, mode)
 194     if manifestExists:
 195       # import previous content into hash, ignoring empty ones and comments
 196       imf = re.compile('(#.*)?$')
 197       for l in re.split('[\r\n]+', mf.read()):
 198         if imf.match(l):
 199           continue
 200         myregister[l] = None
 201       mf.seek(0)
 202     for k in myregister.iterkeys():
 203       mf.write(k + os.linesep)
 204     mf.close()
 205
 206   def makeJar(self, infile=None,
 207                jardir='',
 208                sourcedirs=[], topsourcedir='', localedirs=None):
 209     '''makeJar is the main entry point to JarMaker.
 210
 211     It takes the input file, the output directory, the source dirs and the
 212     top source dir as argument, and optionally the l10n dirs.
 213     '''
 214     if isinstance(infile, basestring):
 215       logging.info("processing " + infile)
 216     pp = self.pp.clone()
 217     pp.out = StringIO()
 218     pp.do_include(infile)
 219     lines = pushback_iter(pp.out.getvalue().splitlines())
 220     try:
 221       while True:
 222         l = lines.next()
 223         m = self.jarline.match(l)
 224         if not m:
 225           raise RuntimeError(l)
 226         if m.group('jarfile') is None:
 227           # comment
 228           continue
 229         self.processJarSection(m.group('jarfile'), lines,
 230                                jardir, sourcedirs, topsourcedir,
 231                                localedirs)
 232     except StopIteration:
 233       # we read the file
 234       pass
 235     return
 236
 237   def makeJars(self, infiles, l10nbases,
 238                jardir='',
 239                sourcedirs=[], topsourcedir='', localedirs=None):
 240     '''makeJars is the second main entry point to JarMaker.
 241
 242     It takes an iterable sequence of input file names, the l10nbases,
 243     the output directory, the source dirs and the
 244     top source dir as argument, and optionally the l10n dirs.
 245
 246     It iterates over all inputs, guesses srcdir and l10ndir from the
 247     path and topsourcedir and calls into makeJar.
 248
 249     The l10ndirs are created by guessing the relativesrcdir, and resolving
 250     that against the l10nbases. l10nbases can either be path strings, or
 251     callables. In the latter case, that will be called with the
 252     relativesrcdir as argument, and is expected to return a path string.
 253     This logic is disabled if the jar.mn path is not inside the topsrcdir.
 254     '''
 255     topsourcedir = os.path.normpath(os.path.abspath(topsourcedir))
 256     def resolveL10nBase(relpath):
 257       def _resolve(base):
 258         if isinstance(base, basestring):
 259           return os.path.join(base, relpath)
 260         if callable(base):
 261           return base(relpath)
 262         return base
 263       return _resolve
 264     for infile in infiles:
 265       srcdir = os.path.normpath(os.path.abspath(os.path.dirname(infile)))
 266       l10ndir = srcdir
 267       if os.path.basename(srcdir) == 'locales':
 268         l10ndir = os.path.dirname(l10ndir)
 269
 270       l10ndirs = None
 271       # srcdir may not be a child of topsourcedir, in which case
 272       # we assume that the caller passed in suitable sourcedirs,
 273       # and just skip passing in localedirs
 274       if srcdir.startswith(topsourcedir):
 275         rell10ndir = l10ndir[len(topsourcedir):].lstrip(os.sep)
 276
 277         l10ndirs = map(resolveL10nBase(rell10ndir), l10nbases)
 278         if localedirs is not None:
 279           l10ndirs += [os.path.normpath(os.path.abspath(s))
 280                        for s in localedirs]
 281       srcdirs = [os.path.normpath(os.path.abspath(s))
 282                  for s in sourcedirs] + [srcdir]
 283       self.makeJar(infile=infile,
 284                    sourcedirs=srcdirs, topsourcedir=topsourcedir,
 285                    localedirs=l10ndirs,
 286                    jardir=jardir)
 287
 288
 289   def processJarSection(self, jarfile, lines,
 290                         jardir, sourcedirs, topsourcedir, localedirs):
 291     '''Internal method called by makeJar to actually process a section
 292     of a jar.mn file.
 293
 294     jarfile is the basename of the jarfile or the directory name for
 295     flat output, lines is a pushback_iterator of the lines of jar.mn,
 296     the remaining options are carried over from makeJar.
 297     '''
 298
 299     # chromebasepath is used for chrome registration manifests
 300     # %s is getting replaced with chrome/ for chrome.manifest, and with
 301     # an empty string for jarfile.manifest
 302     chromebasepath = '%s' + jarfile
 303     if self.outputFormat == 'jar':
 304       chromebasepath = 'jar:' + chromebasepath + '.jar!'
 305     chromebasepath += '/'
 306
 307     jarfile = os.path.join(jardir, jarfile)
 308     jf = None
 309     if self.outputFormat == 'jar':
 310       #jar
 311       jarfilepath = jarfile + '.jar'
 312       try:
 313         os.makedirs(os.path.dirname(jarfilepath))
 314       except OSError:
 315         pass
 316       jf = ZipFile(jarfilepath, 'a', lock = True)
 317       outHelper = self.OutputHelper_jar(jf)
 318     else:
 319       outHelper = getattr(self, 'OutputHelper_' + self.outputFormat)(jarfile)
 320     register = {}
 321     # This loop exits on either
 322     # - the end of the jar.mn file
 323     # - an line in the jar.mn file that's not part of a jar section
 324     # - on an exception raised, close the jf in that case in a finally
 325     try:
 326       while True:
 327         try:
 328           l = lines.next()
 329         except StopIteration:
 330           # we're done with this jar.mn, and this jar section
 331           self.finalizeJar(jarfile, chromebasepath, register)
 332           if jf is not None:
 333             jf.close()
 334           # reraise the StopIteration for makeJar
 335           raise
 336         if self.ignore.match(l):
 337           continue
 338         m = self.regline.match(l)
 339         if  m:
 340           rline = m.group(1)
 341           register[rline] = 1
 342           continue
 343         m = self.entryline.match(l)
 344         if not m:
 345           # neither an entry line nor chrome reg, this jar section is done
 346           self.finalizeJar(jarfile, chromebasepath, register)
 347           if jf is not None:
 348             jf.close()
 349           lines.pushback(l)
 350           return
 351         self._processEntryLine(m, sourcedirs, topsourcedir, localedirs,
 352                               outHelper, jf)
 353     finally:
 354       if jf is not None:
 355         jf.close()
 356     return
 357
 358   def _processEntryLine(self, m,
 359                         sourcedirs, topsourcedir, localedirs,
 360                         outHelper, jf):
 361       out = m.group('output')
 362       src = m.group('source') or os.path.basename(out)
 363       # pick the right sourcedir -- l10n, topsrc or src
 364       if m.group('locale'):
 365         src_base = localedirs
 366       elif src.startswith('/'):
 367         # path/in/jar/file_name.xul     (/path/in/sourcetree/file_name.xul)
 368         # refers to a path relative to topsourcedir, use that as base
 369         # and strip the leading '/'
 370         src_base = [topsourcedir]
 371         src = src[1:]
 372       else:
 373         # use srcdirs and the objdir (current working dir) for relative paths
 374         src_base = sourcedirs + ['.']
 375       # check if the source file exists
 376       realsrc = None
 377       for _srcdir in src_base:
 378         if os.path.isfile(os.path.join(_srcdir, src)):
 379           realsrc = os.path.join(_srcdir, src)
 380           break
 381       if realsrc is None:
 382         if jf is not None:
 383           jf.close()
 384         raise RuntimeError('File "%s" not found in %s' % (src, ', '.join(src_base)))
 385       if m.group('optPreprocess'):
 386         outf = outHelper.getOutput(out)
 387         inf = open(realsrc)
 388         pp = self.pp.clone()
 389         if src[-4:] == '.css':
 390           pp.setMarker('%')
 391         pp.out = outf
 392         pp.do_include(inf)
 393         outf.close()
 394         inf.close()
 395         return
 396       # copy or symlink if newer or overwrite
 397       if (m.group('optOverwrite')
 398           or (getModTime(realsrc) >
 399               outHelper.getDestModTime(m.group('output')))):
 400         if self.outputFormat == 'symlink' and hasattr(os, 'symlink'):
 401           outHelper.symlink(realsrc, out)
 402           return
 403         outf = outHelper.getOutput(out)
 404         # open in binary mode, this can be images etc
 405         inf = open(realsrc, 'rb')
 406         outf.write(inf.read())
 407         outf.close()
 408         inf.close()
 409
 410
 411   class OutputHelper_jar(object):
 412     '''Provide getDestModTime and getOutput for a given jarfile.
 413     '''
 414     def __init__(self, jarfile):
 415       self.jarfile = jarfile
 416     def getDestModTime(self, aPath):
 417       try :
 418         info = self.jarfile.getinfo(aPath)
 419         return info.date_time
 420       except:
 421         return 0
 422     def getOutput(self, name):
 423       return ZipEntry(name, self.jarfile)
 424
 425   class OutputHelper_flat(object):
 426     '''Provide getDestModTime and getOutput for a given flat
 427     output directory. The helper method ensureDirFor is used by
 428     the symlink subclass.
 429     '''
 430     def __init__(self, basepath):
 431       self.basepath = basepath
 432     def getDestModTime(self, aPath):
 433       return getModTime(os.path.join(self.basepath, aPath))
 434     def getOutput(self, name):
 435       out = self.ensureDirFor(name)
 436       # remove previous link or file
 437       try:
 438         os.remove(out)
 439       except OSError, e:
 440         if e.errno != 2:
 441           raise
 442       return open(out, 'wb')
 443     def ensureDirFor(self, name):
 444       out = os.path.join(self.basepath, name)
 445       outdir = os.path.dirname(out)
 446       if not os.path.isdir(outdir):
 447         os.makedirs(outdir)
 448       return out
 449
 450   class OutputHelper_symlink(OutputHelper_flat):
 451     '''Subclass of OutputHelper_flat that provides a helper for
 452     creating a symlink including creating the parent directories.
 453     '''
 454     def symlink(self, src, dest):
 455       out = self.ensureDirFor(dest)
 456       # remove previous link or file
 457       try:
 458         os.remove(out)
 459       except OSError, e:
 460         if e.errno != 2:
 461           raise
 462       os.symlink(src, out)
 463
 464 def main():
 465   jm = JarMaker()
 466   p = jm.getCommandLineParser()
 467   (options, args) = p.parse_args()
 468   jm.processIncludes(options.I)
 469   jm.outputFormat = options.f
 470   if options.e:
 471     jm.useChromeManifest = True
 472     jm.useJarfileManifest = False
 473   if options.bothManifests:
 474     jm.useChromeManifest = True
 475     jm.useJarfileManifest = True
 476   noise = logging.INFO
 477   if options.verbose is not None:
 478     noise = (options.verbose and logging.DEBUG) or logging.WARN
 479   if sys.version_info[:2] > (2,3):
 480     logging.basicConfig(format = "%(message)s")
 481   else:
 482     logging.basicConfig()
 483   logging.getLogger().setLevel(noise)
 484   topsrc = options.t
 485   topsrc = os.path.normpath(os.path.abspath(topsrc))
 486   if not args:
 487     jm.makeJar(infile=sys.stdin,
 488                sourcedirs=options.s, topsourcedir=topsrc,
 489                localedirs=options.l10n_src,
 490                jardir=options.j)
 491   else:
 492     jm.makeJars(args, options.l10n_base,
 493                 jardir=options.j,
 494                 sourcedirs=options.s, topsourcedir=topsrc,
 495                 localedirs=options.l10n_src)
 496
 497 if __name__ == "__main__":
 498   main()