config/JarMaker.py

   1 # This Source Code Form is subject to the terms of the Mozilla Public
   2 # License, v. 2.0. If a copy of the MPL was not distributed with this
   3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   4
   5 '''jarmaker.py provides a python class to package up chrome content by
   6 processing jar.mn files.
   7
   8 See the documentation for jar.mn on MDC for further details on the format.
   9 '''
  10 import sys
  11 import os
  12 import os.path
  13 import errno
  14 import re
  15 import logging
  16 from time import localtime
  17 from optparse import OptionParser
  18 from MozZipFile import ZipFile
  19 from cStringIO import StringIO
  20 from datetime import datetime
  21
  22 from utils import pushback_iter, lockFile
  23 from Preprocessor import Preprocessor
  24 from buildlist import addEntriesToListFile
  25 if sys.platform == "win32":
  26   from ctypes import windll, WinError
  27   CreateHardLink = windll.kernel32.CreateHardLinkA
  28
  29 __all__ = ['JarMaker']
  30
  31 class ZipEntry:
  32   '''Helper class for jar output.
  33
  34   This class defines a simple file-like object for a zipfile.ZipEntry
  35   so that we can consecutively write to it and then close it.
  36   This methods hooks into ZipFile.writestr on close().
  37   '''
  38   def __init__(self, name, zipfile):
  39     self._zipfile = zipfile
  40     self._name = name
  41     self._inner = StringIO()
  42
  43   def write(self, content):
  44     'Append the given content to this zip entry'
  45     self._inner.write(content)
  46     return
  47
  48   def close(self):
  49     'The close method writes the content back to the zip file.'
  50     self._zipfile.writestr(self._name, self._inner.getvalue())
  51
  52 def getModTime(aPath):
  53   if not os.path.isfile(aPath):
  54     return 0
  55   mtime = os.stat(aPath).st_mtime
  56   return localtime(mtime)
  57
  58
  59 class JarMaker(object):
  60   '''JarMaker reads jar.mn files and process those into jar files or
  61   flat directories, along with chrome.manifest files.
  62   '''
  63
  64   ignore = re.compile('\s*(\#.*)?$')
  65   jarline = re.compile('(?:(?P<jarfile>[\w\d.\-\_\\\/]+).jar\:)|(?:\s*(\#.*)?)\s*$')
  66   relsrcline = re.compile('relativesrcdir\s+(?P<relativesrcdir>.+?):')
  67   regline = re.compile('\%\s+(.*)$')
  68   entryre = '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+'
  69   entryline = re.compile(entryre + '(?P<output>[\w\d.\-\_\\\/\+\@]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/\@]+)\))?\s*$')
  70
  71   def __init__(self, outputFormat = 'flat', useJarfileManifest = True,
  72                useChromeManifest = False):
  73     self.outputFormat = outputFormat
  74     self.useJarfileManifest = useJarfileManifest
  75     self.useChromeManifest = useChromeManifest
  76     self.pp = Preprocessor()
  77     self.topsourcedir = None
  78     self.sourcedirs = []
  79     self.localedirs = None
  80     self.l10nbase = None
  81     self.l10nmerge = None
  82     self.relativesrcdir = None
  83     self.rootManifestAppId = None
  84
  85   def getCommandLineParser(self):
  86     '''Get a optparse.OptionParser for jarmaker.
  87
  88     This OptionParser has the options for jarmaker as well as
  89     the options for the inner PreProcessor.
  90     '''
  91     # HACK, we need to unescape the string variables we get,
  92     # the perl versions didn't grok strings right
  93     p = self.pp.getCommandLineParser(unescapeDefines = True)
  94     p.add_option('-f', type="choice", default="jar",
  95                  choices=('jar', 'flat', 'symlink'),
  96                  help="fileformat used for output", metavar="[jar, flat, symlink]")
  97     p.add_option('-v', action="store_true", dest="verbose",
  98                  help="verbose output")
  99     p.add_option('-q', action="store_false", dest="verbose",
 100                  help="verbose output")
 101     p.add_option('-e', action="store_true",
 102                  help="create chrome.manifest instead of jarfile.manifest")
 103     p.add_option('--both-manifests', action="store_true",
 104                  dest="bothManifests",
 105                  help="create chrome.manifest and jarfile.manifest")
 106     p.add_option('-s', type="string", action="append", default=[],
 107                  help="source directory")
 108     p.add_option('-t', type="string",
 109                  help="top source directory")
 110     p.add_option('-c', '--l10n-src', type="string", action="append",
 111                  help="localization directory")
 112     p.add_option('--l10n-base', type="string", action="store",
 113                  help="base directory to be used for localization (requires relativesrcdir)")
 114     p.add_option('--locale-mergedir', type="string", action="store",
 115                  help="base directory to be used for l10n-merge (requires l10n-base and relativesrcdir)")
 116     p.add_option('--relativesrcdir', type="string",
 117                  help="relativesrcdir to be used for localization")
 118     p.add_option('-j', type="string",
 119                  help="jarfile directory")
 120     p.add_option('--root-manifest-entry-appid', type="string",
 121                  help="add an app id specific root chrome manifest entry.")
 122     return p
 123
 124   def processIncludes(self, includes):
 125     '''Process given includes with the inner PreProcessor.
 126
 127     Only use this for #defines, the includes shouldn't generate
 128     content.
 129     '''
 130     self.pp.out = StringIO()
 131     for inc in includes:
 132       self.pp.do_include(inc)
 133     includesvalue = self.pp.out.getvalue()
 134     if includesvalue:
 135       logging.info("WARNING: Includes produce non-empty output")
 136     self.pp.out = None
 137     pass
 138
 139   def finalizeJar(self, jarPath, chromebasepath, register,
 140                   doZip=True):
 141     '''Helper method to write out the chrome registration entries to
 142     jarfile.manifest or chrome.manifest, or both.
 143
 144     The actual file processing is done in updateManifest.
 145     '''
 146     # rewrite the manifest, if entries given
 147     if not register:
 148       return
 149
 150     chromeManifest = os.path.join(os.path.dirname(jarPath),
 151                                   '..', 'chrome.manifest')
 152
 153     if self.useJarfileManifest:
 154       self.updateManifest(jarPath + '.manifest', chromebasepath.format(''),
 155                           register)
 156       addEntriesToListFile(chromeManifest, ['manifest chrome/{0}.manifest'
 157                                             .format(os.path.basename(jarPath))])
 158     if self.useChromeManifest:
 159       self.updateManifest(chromeManifest, chromebasepath.format('chrome/'),
 160                           register)
 161
 162     # If requested, add a root chrome manifest entry (assumed to be in the parent directory
 163     # of chromeManifest) with the application specific id. In cases where we're building
 164     # lang packs, the root manifest must know about application sub directories.
 165     if self.rootManifestAppId:
 166       rootChromeManifest = os.path.join(os.path.normpath(os.path.dirname(chromeManifest)),
 167                                         '..', 'chrome.manifest')
 168       rootChromeManifest = os.path.normpath(rootChromeManifest)
 169       chromeDir = os.path.basename(os.path.dirname(os.path.normpath(chromeManifest)))
 170       logging.info("adding '%s' entry to root chrome manifest appid=%s" % (chromeDir, self.rootManifestAppId))
 171       addEntriesToListFile(rootChromeManifest, ['manifest %s/chrome.manifest application=%s' % (chromeDir, self.rootManifestAppId)])
 172
 173   def updateManifest(self, manifestPath, chromebasepath, register):
 174     '''updateManifest replaces the % in the chrome registration entries
 175     with the given chrome base path, and updates the given manifest file.
 176     '''
 177     lock = lockFile(manifestPath + '.lck')
 178     try:
 179       myregister = dict.fromkeys(map(lambda s: s.replace('%', chromebasepath),
 180                                      register.iterkeys()))
 181       manifestExists = os.path.isfile(manifestPath)
 182       mode = (manifestExists and 'r+b') or 'wb'
 183       mf = open(manifestPath, mode)
 184       if manifestExists:
 185         # import previous content into hash, ignoring empty ones and comments
 186         imf = re.compile('(#.*)?$')
 187         for l in re.split('[\r\n]+', mf.read()):
 188           if imf.match(l):
 189             continue
 190           myregister[l] = None
 191         mf.seek(0)
 192       for k in myregister.iterkeys():
 193         mf.write(k + os.linesep)
 194       mf.close()
 195     finally:
 196       lock = None
 197
 198   def makeJar(self, infile, jardir):
 199     '''makeJar is the main entry point to JarMaker.
 200
 201     It takes the input file, the output directory, the source dirs and the
 202     top source dir as argument, and optionally the l10n dirs.
 203     '''
 204     # making paths absolute, guess srcdir if file and add to sourcedirs
 205     _normpath = lambda p: os.path.normpath(os.path.abspath(p))
 206     self.topsourcedir = _normpath(self.topsourcedir)
 207     self.sourcedirs = [_normpath(p) for p in self.sourcedirs]
 208     if self.localedirs:
 209       self.localedirs = [_normpath(p) for p in self.localedirs]
 210     elif self.relativesrcdir:
 211       self.localedirs = self.generateLocaleDirs(self.relativesrcdir)
 212     if isinstance(infile, basestring):
 213       logging.info("processing " + infile)
 214       self.sourcedirs.append(_normpath(os.path.dirname(infile)))
 215     pp = self.pp.clone()
 216     pp.out = StringIO()
 217     pp.do_include(infile)
 218     lines = pushback_iter(pp.out.getvalue().splitlines())
 219     try:
 220       while True:
 221         l = lines.next()
 222         m = self.jarline.match(l)
 223         if not m:
 224           raise RuntimeError(l)
 225         if m.group('jarfile') is None:
 226           # comment
 227           continue
 228         self.processJarSection(m.group('jarfile'), lines, jardir)
 229     except StopIteration:
 230       # we read the file
 231       pass
 232     return
 233
 234   def generateLocaleDirs(self, relativesrcdir):
 235     if os.path.basename(relativesrcdir) == 'locales':
 236       # strip locales
 237       l10nrelsrcdir = os.path.dirname(relativesrcdir)
 238     else:
 239       l10nrelsrcdir = relativesrcdir
 240     locdirs = []
 241     # generate locales dirs, merge, l10nbase, en-US
 242     if self.l10nmerge:
 243       locdirs.append(os.path.join(self.l10nmerge, l10nrelsrcdir))
 244     if self.l10nbase:
 245       locdirs.append(os.path.join(self.l10nbase, l10nrelsrcdir))
 246     if self.l10nmerge or not self.l10nbase:
 247       # add en-US if we merge, or if it's not l10n
 248       locdirs.append(os.path.join(self.topsourcedir, relativesrcdir, 'en-US'))
 249     return locdirs
 250
 251   def processJarSection(self, jarfile, lines, jardir):
 252     '''Internal method called by makeJar to actually process a section
 253     of a jar.mn file.
 254
 255     jarfile is the basename of the jarfile or the directory name for
 256     flat output, lines is a pushback_iterator of the lines of jar.mn,
 257     the remaining options are carried over from makeJar.
 258     '''
 259
 260     # chromebasepath is used for chrome registration manifests
 261     # {0} is getting replaced with chrome/ for chrome.manifest, and with
 262     # an empty string for jarfile.manifest
 263     chromebasepath = '{0}' + os.path.basename(jarfile)
 264     if self.outputFormat == 'jar':
 265       chromebasepath = 'jar:' + chromebasepath + '.jar!'
 266     chromebasepath += '/'
 267
 268     jarfile = os.path.join(jardir, jarfile)
 269     jf = None
 270     if self.outputFormat == 'jar':
 271       #jar
 272       jarfilepath = jarfile + '.jar'
 273       try:
 274         os.makedirs(os.path.dirname(jarfilepath))
 275       except OSError as error:
 276         if error.errno != errno.EEXIST:
 277           raise
 278       jf = ZipFile(jarfilepath, 'a', lock = True)
 279       outHelper = self.OutputHelper_jar(jf)
 280     else:
 281       outHelper = getattr(self, 'OutputHelper_' + self.outputFormat)(jarfile)
 282     register = {}
 283     # This loop exits on either
 284     # - the end of the jar.mn file
 285     # - an line in the jar.mn file that's not part of a jar section
 286     # - on an exception raised, close the jf in that case in a finally
 287     try:
 288       while True:
 289         try:
 290           l = lines.next()
 291         except StopIteration:
 292           # we're done with this jar.mn, and this jar section
 293           self.finalizeJar(jarfile, chromebasepath, register)
 294           if jf is not None:
 295             jf.close()
 296           # reraise the StopIteration for makeJar
 297           raise
 298         if self.ignore.match(l):
 299           continue
 300         m = self.relsrcline.match(l)
 301         if m:
 302           relativesrcdir = m.group('relativesrcdir')
 303           self.localedirs = self.generateLocaleDirs(relativesrcdir)
 304           continue
 305         m = self.regline.match(l)
 306         if  m:
 307           rline = m.group(1)
 308           register[rline] = 1
 309           continue
 310         m = self.entryline.match(l)
 311         if not m:
 312           # neither an entry line nor chrome reg, this jar section is done
 313           self.finalizeJar(jarfile, chromebasepath, register)
 314           if jf is not None:
 315             jf.close()
 316           lines.pushback(l)
 317           return
 318         self._processEntryLine(m, outHelper, jf)
 319     finally:
 320       if jf is not None:
 321         jf.close()
 322     return
 323
 324   def _processEntryLine(self, m, outHelper, jf):
 325       out = m.group('output')
 326       src = m.group('source') or os.path.basename(out)
 327       # pick the right sourcedir -- l10n, topsrc or src
 328       if m.group('locale'):
 329         src_base = self.localedirs
 330       elif src.startswith('/'):
 331         # path/in/jar/file_name.xul     (/path/in/sourcetree/file_name.xul)
 332         # refers to a path relative to topsourcedir, use that as base
 333         # and strip the leading '/'
 334         src_base = [self.topsourcedir]
 335         src = src[1:]
 336       else:
 337         # use srcdirs and the objdir (current working dir) for relative paths
 338         src_base = self.sourcedirs + [os.getcwd()]
 339       # check if the source file exists
 340       realsrc = None
 341       for _srcdir in src_base:
 342         if os.path.isfile(os.path.join(_srcdir, src)):
 343           realsrc = os.path.join(_srcdir, src)
 344           break
 345       if realsrc is None:
 346         if jf is not None:
 347           jf.close()
 348         raise RuntimeError('File "{0}" not found in {1}'
 349                            .format(src, ', '.join(src_base)))
 350       if m.group('optPreprocess'):
 351         outf = outHelper.getOutput(out)
 352         inf = open(realsrc)
 353         pp = self.pp.clone()
 354         if src[-4:] == '.css':
 355           pp.setMarker('%')
 356         pp.out = outf
 357         pp.do_include(inf)
 358         pp.warnUnused(realsrc)
 359         outf.close()
 360         inf.close()
 361         return
 362       # copy or symlink if newer or overwrite
 363       if (m.group('optOverwrite')
 364           or (getModTime(realsrc) >
 365               outHelper.getDestModTime(m.group('output')))):
 366         if self.outputFormat == 'symlink':
 367           outHelper.symlink(realsrc, out)
 368           return
 369         outf = outHelper.getOutput(out)
 370         # open in binary mode, this can be images etc
 371         inf = open(realsrc, 'rb')
 372         outf.write(inf.read())
 373         outf.close()
 374         inf.close()
 375
 376
 377   class OutputHelper_jar(object):
 378     '''Provide getDestModTime and getOutput for a given jarfile.
 379     '''
 380     def __init__(self, jarfile):
 381       self.jarfile = jarfile
 382     def getDestModTime(self, aPath):
 383       try :
 384         info = self.jarfile.getinfo(aPath)
 385         return info.date_time
 386       except:
 387         return 0
 388     def getOutput(self, name):
 389       return ZipEntry(name, self.jarfile)
 390
 391   class OutputHelper_flat(object):
 392     '''Provide getDestModTime and getOutput for a given flat
 393     output directory. The helper method ensureDirFor is used by
 394     the symlink subclass.
 395     '''
 396     def __init__(self, basepath):
 397       self.basepath = basepath
 398     def getDestModTime(self, aPath):
 399       return getModTime(os.path.join(self.basepath, aPath))
 400     def getOutput(self, name):
 401       out = self.ensureDirFor(name)
 402       # remove previous link or file
 403       try:
 404         os.remove(out)
 405       except OSError as e:
 406         if e.errno != errno.ENOENT:
 407           raise
 408       return open(out, 'wb')
 409     def ensureDirFor(self, name):
 410       out = os.path.join(self.basepath, name)
 411       outdir = os.path.dirname(out)
 412       if not os.path.isdir(outdir):
 413         try:
 414           os.makedirs(outdir)
 415         except OSError as error:
 416           if error.errno != errno.EEXIST:
 417             raise
 418       return out
 419
 420   class OutputHelper_symlink(OutputHelper_flat):
 421     '''Subclass of OutputHelper_flat that provides a helper for
 422     creating a symlink including creating the parent directories.
 423     '''
 424     def symlink(self, src, dest):
 425       out = self.ensureDirFor(dest)
 426       # remove previous link or file
 427       try:
 428         os.remove(out)
 429       except OSError as e:
 430         if e.errno != errno.ENOENT:
 431           raise
 432       if sys.platform != "win32":
 433         os.symlink(src, out)
 434       else:
 435         # On Win32, use ctypes to create a hardlink
 436         rv = CreateHardLink(out, src, None)
 437         if rv == 0:
 438           raise WinError()
 439
 440 def main():
 441   jm = JarMaker()
 442   p = jm.getCommandLineParser()
 443   (options, args) = p.parse_args()
 444   jm.processIncludes(options.I)
 445   jm.outputFormat = options.f
 446   jm.sourcedirs = options.s
 447   jm.topsourcedir = options.t
 448   if options.e:
 449     jm.useChromeManifest = True
 450     jm.useJarfileManifest = False
 451   if options.bothManifests:
 452     jm.useChromeManifest = True
 453     jm.useJarfileManifest = True
 454   if options.l10n_base:
 455     if not options.relativesrcdir:
 456       p.error('relativesrcdir required when using l10n-base')
 457     if options.l10n_src:
 458       p.error('both l10n-src and l10n-base are not supported')
 459     jm.l10nbase = options.l10n_base
 460     jm.relativesrcdir = options.relativesrcdir
 461     jm.l10nmerge = options.locale_mergedir
 462   elif options.locale_mergedir:
 463     p.error('l10n-base required when using locale-mergedir')
 464   jm.localedirs = options.l10n_src
 465   if options.root_manifest_entry_appid:
 466     jm.rootManifestAppId = options.root_manifest_entry_appid
 467   noise = logging.INFO
 468   if options.verbose is not None:
 469     noise = (options.verbose and logging.DEBUG) or logging.WARN
 470   if sys.version_info[:2] > (2,3):
 471     logging.basicConfig(format = "%(message)s")
 472   else:
 473     logging.basicConfig()
 474   logging.getLogger().setLevel(noise)
 475   topsrc = options.t
 476   topsrc = os.path.normpath(os.path.abspath(topsrc))
 477   if not args:
 478     infile = sys.stdin
 479   else:
 480     infile,  = args
 481   jm.makeJar(infile, options.j)
 482
 483 if __name__ == "__main__":
 484   main()