config/JarMaker.py

   1 # ***** BEGIN LICENSE BLOCK *****
   2 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
   3 #
   4 # The contents of this file are subject to the Mozilla Public License Version
   5 # 1.1 (the "License"); you may not use this file except in compliance with
   6 # the License. You may obtain a copy of the License at
   7 # http://www.mozilla.org/MPL/
   8 #
   9 # Software distributed under the License is distributed on an "AS IS" basis,
  10 # WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
  11 # for the specific language governing rights and limitations under the
  12 # License.
  13 #
  14 # The Original Code is Mozilla build system.
  15 #
  16 # The Initial Developer of the Original Code is
  17 # Mozilla Foundation.
  18 # Portions created by the Initial Developer are Copyright (C) 2008
  19 # the Initial Developer. All Rights Reserved.
  20 #
  21 # Contributor(s):
  22 #  Axel Hecht <l10n@mozilla.com>
  23 #
  24 # Alternatively, the contents of this file may be used under the terms of
  25 # either the GNU General Public License Version 2 or later (the "GPL"), or
  26 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
  27 # in which case the provisions of the GPL or the LGPL are applicable instead
  28 # of those above. If you wish to allow use of your version of this file only
  29 # under the terms of either the GPL or the LGPL, and not to allow others to
  30 # use your version of this file under the terms of the MPL, indicate your
  31 # decision by deleting the provisions above and replace them with the notice
  32 # and other provisions required by the GPL or the LGPL. If you do not delete
  33 # the provisions above, a recipient may use your version of this file under
  34 # the terms of any one of the MPL, the GPL or the LGPL.
  35 #
  36 # ***** END LICENSE BLOCK *****
  37
  38 '''jarmaker.py provides a python class to package up chrome content by
  39 processing jar.mn files.
  40
  41 See the documentation for jar.mn on MDC for further details on the format.
  42 '''
  43
  44 import sys
  45 import os
  46 import os.path
  47 import re
  48 import logging
  49 from time import localtime
  50 from optparse import OptionParser
  51 from MozZipFile import ZipFile
  52 from cStringIO import StringIO
  53 from datetime import datetime
  54
  55 from utils import pushback_iter
  56 from Preprocessor import Preprocessor
  57
  58 __all__ = ['JarMaker']
  59
  60 class ZipEntry:
  61   '''Helper class for jar output.
  62
  63   This class defines a simple file-like object for a zipfile.ZipEntry
  64   so that we can consecutively write to it and then close it.
  65   This methods hooks into ZipFile.writestr on close().
  66   '''
  67   def __init__(self, name, zipfile):
  68     self._zipfile = zipfile
  69     self._name = name
  70     self._inner = StringIO()
  71
  72   def write(self, content):
  73     'Append the given content to this zip entry'
  74     self._inner.write(content)
  75     return
  76
  77   def close(self):
  78     'The close method writes the content back to the zip file.'
  79     self._zipfile.writestr(self._name, self._inner.getvalue())
  80
  81 def getModTime(aPath):
  82   if not os.path.isfile(aPath):
  83     return 0
  84   mtime = os.stat(aPath).st_mtime
  85   return localtime(mtime)
  86
  87
  88 class JarMaker(object):
  89   '''JarMaker reads jar.mn files and process those into jar files or
  90   flat directories, along with chrome.manifest files.
  91   '''
  92
  93   ignore = re.compile('\s*(\#.*)?$')
  94   jarline = re.compile('(?:(?P<jarfile>[\w\d.\-\_\\\/]+).jar\:)|(?:\s*(\#.*)?)\s*$')
  95   regline = re.compile('\%\s+(.*)$')
  96   entryre = '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+'
  97   entryline = re.compile(entryre + '(?P<output>[\w\d.\-\_\\\/\+]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/]+)\))?\s*$')
  98
  99   def __init__(self, outputFormat = 'flat', useJarfileManifest = True,
 100                useChromeManifest = False):
 101     self.outputFormat = outputFormat
 102     self.useJarfileManifest = useJarfileManifest
 103     self.useChromeManifest = useChromeManifest
 104     self.pp = Preprocessor()
 105
 106   def getCommandLineParser(self):
 107     '''Get a optparse.OptionParser for jarmaker.
 108
 109     This OptionParser has the options for jarmaker as well as
 110     the options for the inner PreProcessor.
 111     '''
 112     # HACK, we need to unescape the string variables we get,
 113     # the perl versions didn't grok strings right
 114     p = self.pp.getCommandLineParser(unescapeDefines = True)
 115     p.add_option('-f', type="choice", default="jar",
 116                  choices=('jar', 'flat', 'symlink'),
 117                  help="fileformat used for output", metavar="[jar, flat, symlink]")
 118     p.add_option('-v', action="store_true", dest="verbose",
 119                  help="verbose output")
 120     p.add_option('-q', action="store_false", dest="verbose",
 121                  help="verbose output")
 122     p.add_option('-e', action="store_true",
 123                  help="create chrome.manifest instead of jarfile.manifest")
 124     p.add_option('--both-manifests', action="store_true",
 125                  dest="bothManifests",
 126                  help="create chrome.manifest and jarfile.manifest")
 127     p.add_option('-s', type="string", action="append", default=[],
 128                  help="source directory")
 129     p.add_option('-t', type="string",
 130                  help="top source directory")
 131     p.add_option('-c', '--l10n-src', type="string", action="append",
 132                  help="localization directory")
 133     p.add_option('--l10n-base', type="string", action="append", default=[],
 134                  help="base directory to be used for localization (multiple)")
 135     p.add_option('-j', type="string",
 136                  help="jarfile directory")
 137     # backwards compat, not needed
 138     p.add_option('-a', action="store_false", default=True,
 139                  help="NOT SUPPORTED, turn auto-registration of chrome off (installed-chrome.txt)")
 140     p.add_option('-d', type="string",
 141                  help="UNUSED, chrome directory")
 142     p.add_option('-o', help="cross compile for auto-registration, ignored")
 143     p.add_option('-l', action="store_true",
 144                  help="ignored (used to switch off locks)")
 145     p.add_option('-x', action="store_true",
 146                  help="force Unix")
 147     p.add_option('-z', help="backwards compat, ignored")
 148     p.add_option('-p', help="backwards compat, ignored")
 149     return p
 150
 151   def processIncludes(self, includes):
 152     '''Process given includes with the inner PreProcessor.
 153
 154     Only use this for #defines, the includes shouldn't generate
 155     content.
 156     '''
 157     self.pp.out = StringIO()
 158     for inc in includes:
 159       self.pp.do_include(inc)
 160     includesvalue = self.pp.out.getvalue()
 161     if includesvalue:
 162       logging.info("WARNING: Includes produce non-empty output")
 163     self.pp.out = None
 164     pass
 165
 166   def finalizeJar(self, jarPath, chromebasepath, register,
 167                    doZip=True):
 168     '''Helper method to write out the chrome registration entries to
 169     jarfile.manifest or chrome.manifest, or both.
 170
 171     The actual file processing is done in updateManifest.
 172     '''
 173     # rewrite the manifest, if entries given
 174     if not register:
 175       return
 176     if self.useJarfileManifest:
 177       self.updateManifest(jarPath + '.manifest', chromebasepath % '',
 178                           register)
 179     if self.useChromeManifest:
 180       manifestPath = os.path.join(os.path.dirname(jarPath),
 181                                   '..', 'chrome.manifest')
 182       self.updateManifest(manifestPath, chromebasepath % 'chrome/',
 183                           register)
 184
 185   def updateManifest(self, manifestPath, chromebasepath, register):
 186     '''updateManifest replaces the % in the chrome registration entries
 187     with the given chrome base path, and updates the given manifest file.
 188     '''
 189     myregister = dict.fromkeys(map(lambda s: s.replace('%', chromebasepath),
 190                                    register.iterkeys()))
 191     manifestExists = os.path.isfile(manifestPath)
 192     mode = (manifestExists and 'r+b') or 'wb'
 193     mf = open(manifestPath, mode)
 194     if manifestExists:
 195       # import previous content into hash, ignoring empty ones and comments
 196       imf = re.compile('(#.*)?$')
 197       for l in re.split('[\r\n]+', mf.read()):
 198         if imf.match(l):
 199           continue
 200         myregister[l] = None
 201       mf.seek(0)
 202     for k in myregister.iterkeys():
 203       mf.write(k + os.linesep)
 204     mf.close()
 205
 206   def makeJar(self, infile=None,
 207                jardir='',
 208                sourcedirs=[], topsourcedir='', localedirs=None):
 209     '''makeJar is the main entry point to JarMaker.
 210
 211     It takes the input file, the output directory, the source dirs and the
 212     top source dir as argument, and optionally the l10n dirs.
 213     '''
 214     if isinstance(infile, basestring):
 215       logging.info("processing " + infile)
 216     pp = self.pp.clone()
 217     pp.out = StringIO()
 218     pp.do_include(infile)
 219     lines = pushback_iter(pp.out.getvalue().splitlines())
 220     try:
 221       while True:
 222         l = lines.next()
 223         m = self.jarline.match(l)
 224         if not m:
 225           raise RuntimeError(l)
 226         if m.group('jarfile') is None:
 227           # comment
 228           continue
 229         self.processJarSection(m.group('jarfile'), lines,
 230                                jardir, sourcedirs, topsourcedir,
 231                                localedirs)
 232     except StopIteration:
 233       # we read the file
 234       pass
 235     return
 236
 237   def makeJars(self, infiles, l10nbases,
 238                jardir='',
 239                sourcedirs=[], topsourcedir='', localedirs=None):
 240     '''makeJars is the second main entry point to JarMaker.
 241
 242     It takes an iterable sequence of input file names, the l10nbases,
 243     the output directory, the source dirs and the
 244     top source dir as argument, and optionally the l10n dirs.
 245
 246     It iterates over all inputs, guesses srcdir and l10ndir from the
 247     path and topsourcedir and calls into makeJar.
 248
 249     The l10ndirs are created by guessing the relativesrcdir, and resolving
 250     that against the l10nbases. l10nbases can either be path strings, or
 251     callables. In the latter case, that will be called with the
 252     relativesrcdir as argument, and is expected to return a path string.
 253     '''
 254     topsourcedir = os.path.normpath(os.path.abspath(topsourcedir))
 255     def resolveL10nBase(relpath):
 256       def _resolve(base):
 257         if isinstance(base, basestring):
 258           return os.path.join(base, relpath)
 259         if callable(base):
 260           return base(relpath)
 261         return base
 262       return _resolve
 263     for infile in infiles:
 264       srcdir = os.path.normpath(os.path.abspath(os.path.dirname(infile)))
 265       l10ndir = srcdir
 266       if os.path.basename(srcdir) == 'locales':
 267         l10ndir = os.path.dirname(l10ndir)
 268       assert srcdir.startswith(topsourcedir), "src dir %s not in topsourcedir %s" % (srcdir, topsourcedir)
 269       rell10ndir = l10ndir[len(topsourcedir):].lstrip(os.sep)
 270
 271       l10ndirs = map(resolveL10nBase(rell10ndir), l10nbases)
 272       if localedirs is not None:
 273         l10ndirs += [os.path.normpath(os.path.abspath(s))
 274                      for s in localedirs]
 275       srcdirs = [os.path.normpath(os.path.abspath(s))
 276                  for s in sourcedirs] + [srcdir]
 277       self.makeJar(infile=infile,
 278                    sourcedirs=srcdirs, topsourcedir=topsourcedir,
 279                    localedirs=l10ndirs,
 280                    jardir=jardir)
 281
 282
 283   def processJarSection(self, jarfile, lines,
 284                         jardir, sourcedirs, topsourcedir, localedirs):
 285     '''Internal method called by makeJar to actually process a section
 286     of a jar.mn file.
 287
 288     jarfile is the basename of the jarfile or the directory name for
 289     flat output, lines is a pushback_iterator of the lines of jar.mn,
 290     the remaining options are carried over from makeJar.
 291     '''
 292
 293     # chromebasepath is used for chrome registration manifests
 294     # %s is getting replaced with chrome/ for chrome.manifest, and with
 295     # an empty string for jarfile.manifest
 296     chromebasepath = '%s' + jarfile
 297     if self.outputFormat == 'jar':
 298       chromebasepath = 'jar:' + chromebasepath + '.jar!'
 299     chromebasepath += '/'
 300
 301     jarfile = os.path.join(jardir, jarfile)
 302     jf = None
 303     if self.outputFormat == 'jar':
 304       #jar
 305       jarfilepath = jarfile + '.jar'
 306       try:
 307         os.makedirs(os.path.dirname(jarfilepath))
 308       except OSError:
 309         pass
 310       jf = ZipFile(jarfilepath, 'a', lock = True)
 311       outHelper = self.OutputHelper_jar(jf)
 312     else:
 313       outHelper = getattr(self, 'OutputHelper_' + self.outputFormat)(jarfile)
 314     register = {}
 315     # This loop exits on either
 316     # - the end of the jar.mn file
 317     # - an line in the jar.mn file that's not part of a jar section
 318     # - on an exception raised, close the jf in that case in a finally
 319     try:
 320       while True:
 321         try:
 322           l = lines.next()
 323         except StopIteration:
 324           # we're done with this jar.mn, and this jar section
 325           self.finalizeJar(jarfile, chromebasepath, register)
 326           if jf is not None:
 327             jf.close()
 328           # reraise the StopIteration for makeJar
 329           raise
 330         if self.ignore.match(l):
 331           continue
 332         m = self.regline.match(l)
 333         if  m:
 334           rline = m.group(1)
 335           register[rline] = 1
 336           continue
 337         m = self.entryline.match(l)
 338         if not m:
 339           # neither an entry line nor chrome reg, this jar section is done
 340           self.finalizeJar(jarfile, chromebasepath, register)
 341           if jf is not None:
 342             jf.close()
 343           lines.pushback(l)
 344           return
 345         self._processEntryLine(m, sourcedirs, topsourcedir, localedirs,
 346                               outHelper, jf)
 347     finally:
 348       if jf is not None:
 349         jf.close()
 350     return
 351
 352   def _processEntryLine(self, m,
 353                         sourcedirs, topsourcedir, localedirs,
 354                         outHelper, jf):
 355       out = m.group('output')
 356       src = m.group('source') or os.path.basename(out)
 357       # pick the right sourcedir -- l10n, topsrc or src
 358       if m.group('locale'):
 359         src_base = localedirs
 360       elif src.startswith('/'):
 361         # path/in/jar/file_name.xul     (/path/in/sourcetree/file_name.xul)
 362         # refers to a path relative to topsourcedir, use that as base
 363         # and strip the leading '/'
 364         src_base = [topsourcedir]
 365         src = src[1:]
 366       else:
 367         # use srcdirs and the objdir (current working dir) for relative paths
 368         src_base = sourcedirs + ['.']
 369       # check if the source file exists
 370       realsrc = None
 371       for _srcdir in src_base:
 372         if os.path.isfile(os.path.join(_srcdir, src)):
 373           realsrc = os.path.join(_srcdir, src)
 374           break
 375       if realsrc is None:
 376         if jf is not None:
 377           jf.close()
 378         raise RuntimeError("file not found: " + src)
 379       if m.group('optPreprocess'):
 380         outf = outHelper.getOutput(out)
 381         inf = open(realsrc)
 382         pp = self.pp.clone()
 383         if src[-4:] == '.css':
 384           pp.setMarker('%')
 385         pp.out = outf
 386         pp.do_include(inf)
 387         outf.close()
 388         inf.close()
 389         return
 390       # copy or symlink if newer or overwrite
 391       if (m.group('optOverwrite')
 392           or (getModTime(realsrc) >
 393               outHelper.getDestModTime(m.group('output')))):
 394         if self.outputFormat == 'symlink' and hasattr(os, 'symlink'):
 395           outHelper.symlink(realsrc, out)
 396           return
 397         outf = outHelper.getOutput(out)
 398         # open in binary mode, this can be images etc
 399         inf = open(realsrc, 'rb')
 400         outf.write(inf.read())
 401         outf.close()
 402         inf.close()
 403
 404
 405   class OutputHelper_jar(object):
 406     '''Provide getDestModTime and getOutput for a given jarfile.
 407     '''
 408     def __init__(self, jarfile):
 409       self.jarfile = jarfile
 410     def getDestModTime(self, aPath):
 411       try :
 412         info = self.jarfile.getinfo(aPath)
 413         return info.date_time
 414       except:
 415         return 0
 416     def getOutput(self, name):
 417       return ZipEntry(name, self.jarfile)
 418
 419   class OutputHelper_flat(object):
 420     '''Provide getDestModTime and getOutput for a given flat
 421     output directory. The helper method ensureDirFor is used by
 422     the symlink subclass.
 423     '''
 424     def __init__(self, basepath):
 425       self.basepath = basepath
 426     def getDestModTime(self, aPath):
 427       return getModTime(os.path.join(self.basepath, aPath))
 428     def getOutput(self, name):
 429       out = self.ensureDirFor(name)
 430       # remove previous link or file
 431       try:
 432         os.remove(out)
 433       except OSError, e:
 434         if e.errno != 2:
 435           raise
 436       return open(out, 'wb')
 437     def ensureDirFor(self, name):
 438       out = os.path.join(self.basepath, name)
 439       outdir = os.path.dirname(out)
 440       if not os.path.isdir(outdir):
 441         os.makedirs(outdir)
 442       return out
 443
 444   class OutputHelper_symlink(OutputHelper_flat):
 445     '''Subclass of OutputHelper_flat that provides a helper for
 446     creating a symlink including creating the parent directories.
 447     '''
 448     def symlink(self, src, dest):
 449       out = self.ensureDirFor(dest)
 450       # remove previous link or file
 451       try:
 452         os.remove(out)
 453       except OSError, e:
 454         if e.errno != 2:
 455           raise
 456       os.symlink(src, out)
 457
 458 def main():
 459   jm = JarMaker()
 460   p = jm.getCommandLineParser()
 461   (options, args) = p.parse_args()
 462   jm.processIncludes(options.I)
 463   jm.outputFormat = options.f
 464   if options.e:
 465     jm.useChromeManifest = True
 466     jm.useJarfileManifest = False
 467   if options.bothManifests:
 468     jm.useChromeManifest = True
 469     jm.useJarfileManifest = True
 470   noise = logging.INFO
 471   if options.verbose is not None:
 472     noise = (options.verbose and logging.DEBUG) or logging.WARN
 473   if sys.version_info[:2] > (2,3):
 474     logging.basicConfig(format = "%(message)s")
 475   else:
 476     logging.basicConfig()
 477   logging.getLogger().setLevel(noise)
 478   topsrc = options.t
 479   topsrc = os.path.normpath(os.path.abspath(topsrc))
 480   if not args:
 481     jm.makeJar(infile=sys.stdin,
 482                sourcedirs=options.s, topsourcedir=topsrc,
 483                localedirs=options.l10n_src,
 484                jardir=options.j)
 485   else:
 486     jm.makeJars(args, options.l10n_base,
 487                 jardir=options.j,
 488                 sourcedirs=options.s, topsourcedir=topsrc,
 489                 localedirs=options.l10n_src)
 490
 491 if __name__ == "__main__":
 492   main()