1 # This Source Code Form is subject to the terms of the Mozilla Public
2 # License, v. 2.0. If a copy of the MPL was not distributed with this
3 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
5 '''jarmaker.py provides a python class to package up chrome content by
6 processing jar.mn files.
8 See the documentation for jar.mn on MDC for further details on the format.
16 from time
import localtime
17 from optparse
import OptionParser
18 from MozZipFile
import ZipFile
19 from cStringIO
import StringIO
20 from datetime
import datetime
22 from utils
import pushback_iter
, lockFile
23 from Preprocessor
import Preprocessor
24 from buildlist
import addEntriesToListFile
25 if sys
.platform
== "win32":
26 from ctypes
import windll
, WinError
27 CreateHardLink
= windll
.kernel32
.CreateHardLinkA
29 __all__
= ['JarMaker']
32 '''Helper class for jar output.
34 This class defines a simple file-like object for a zipfile.ZipEntry
35 so that we can consecutively write to it and then close it.
36 This methods hooks into ZipFile.writestr on close().
38 def __init__(self
, name
, zipfile
):
39 self
._zipfile
= zipfile
41 self
._inner
= StringIO()
43 def write(self
, content
):
44 'Append the given content to this zip entry'
45 self
._inner
.write(content
)
49 'The close method writes the content back to the zip file.'
50 self
._zipfile
.writestr(self
._name
, self
._inner
.getvalue())
52 def getModTime(aPath
):
53 if not os
.path
.isfile(aPath
):
55 mtime
= os
.stat(aPath
).st_mtime
56 return localtime(mtime
)
59 class JarMaker(object):
60 '''JarMaker reads jar.mn files and process those into jar files or
61 flat directories, along with chrome.manifest files.
64 ignore
= re
.compile('\s*(\#.*)?$')
65 jarline
= re
.compile('(?:(?P<jarfile>[\w\d.\-\_\\\/]+).jar\:)|(?:\s*(\#.*)?)\s*$')
66 relsrcline
= re
.compile('relativesrcdir\s+(?P<relativesrcdir>.+?):')
67 regline
= re
.compile('\%\s+(.*)$')
68 entryre
= '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+'
69 entryline
= re
.compile(entryre
+ '(?P<output>[\w\d.\-\_\\\/\+\@]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/\@]+)\))?\s*$')
71 def __init__(self
, outputFormat
= 'flat', useJarfileManifest
= True,
72 useChromeManifest
= False):
73 self
.outputFormat
= outputFormat
74 self
.useJarfileManifest
= useJarfileManifest
75 self
.useChromeManifest
= useChromeManifest
76 self
.pp
= Preprocessor()
77 self
.topsourcedir
= None
79 self
.localedirs
= None
82 self
.relativesrcdir
= None
83 self
.rootManifestAppId
= None
85 def getCommandLineParser(self
):
86 '''Get a optparse.OptionParser for jarmaker.
88 This OptionParser has the options for jarmaker as well as
89 the options for the inner PreProcessor.
91 # HACK, we need to unescape the string variables we get,
92 # the perl versions didn't grok strings right
93 p
= self
.pp
.getCommandLineParser(unescapeDefines
= True)
94 p
.add_option('-f', type="choice", default
="jar",
95 choices
=('jar', 'flat', 'symlink'),
96 help="fileformat used for output", metavar
="[jar, flat, symlink]")
97 p
.add_option('-v', action
="store_true", dest
="verbose",
98 help="verbose output")
99 p
.add_option('-q', action
="store_false", dest
="verbose",
100 help="verbose output")
101 p
.add_option('-e', action
="store_true",
102 help="create chrome.manifest instead of jarfile.manifest")
103 p
.add_option('--both-manifests', action
="store_true",
104 dest
="bothManifests",
105 help="create chrome.manifest and jarfile.manifest")
106 p
.add_option('-s', type="string", action
="append", default
=[],
107 help="source directory")
108 p
.add_option('-t', type="string",
109 help="top source directory")
110 p
.add_option('-c', '--l10n-src', type="string", action
="append",
111 help="localization directory")
112 p
.add_option('--l10n-base', type="string", action
="store",
113 help="base directory to be used for localization (requires relativesrcdir)")
114 p
.add_option('--locale-mergedir', type="string", action
="store",
115 help="base directory to be used for l10n-merge (requires l10n-base and relativesrcdir)")
116 p
.add_option('--relativesrcdir', type="string",
117 help="relativesrcdir to be used for localization")
118 p
.add_option('-j', type="string",
119 help="jarfile directory")
120 p
.add_option('--root-manifest-entry-appid', type="string",
121 help="add an app id specific root chrome manifest entry.")
124 def processIncludes(self
, includes
):
125 '''Process given includes with the inner PreProcessor.
127 Only use this for #defines, the includes shouldn't generate
130 self
.pp
.out
= StringIO()
132 self
.pp
.do_include(inc
)
133 includesvalue
= self
.pp
.out
.getvalue()
135 logging
.info("WARNING: Includes produce non-empty output")
139 def finalizeJar(self
, jarPath
, chromebasepath
, register
,
141 '''Helper method to write out the chrome registration entries to
142 jarfile.manifest or chrome.manifest, or both.
144 The actual file processing is done in updateManifest.
146 # rewrite the manifest, if entries given
150 chromeManifest
= os
.path
.join(os
.path
.dirname(jarPath
),
151 '..', 'chrome.manifest')
153 if self
.useJarfileManifest
:
154 self
.updateManifest(jarPath
+ '.manifest', chromebasepath
.format(''),
156 addEntriesToListFile(chromeManifest
, ['manifest chrome/{0}.manifest'
157 .format(os
.path
.basename(jarPath
))])
158 if self
.useChromeManifest
:
159 self
.updateManifest(chromeManifest
, chromebasepath
.format('chrome/'),
162 # If requested, add a root chrome manifest entry (assumed to be in the parent directory
163 # of chromeManifest) with the application specific id. In cases where we're building
164 # lang packs, the root manifest must know about application sub directories.
165 if self
.rootManifestAppId
:
166 rootChromeManifest
= os
.path
.join(os
.path
.normpath(os
.path
.dirname(chromeManifest
)),
167 '..', 'chrome.manifest')
168 rootChromeManifest
= os
.path
.normpath(rootChromeManifest
)
169 chromeDir
= os
.path
.basename(os
.path
.dirname(os
.path
.normpath(chromeManifest
)))
170 logging
.info("adding '%s' entry to root chrome manifest appid=%s" % (chromeDir
, self
.rootManifestAppId
))
171 addEntriesToListFile(rootChromeManifest
, ['manifest %s/chrome.manifest application=%s' % (chromeDir
, self
.rootManifestAppId
)])
173 def updateManifest(self
, manifestPath
, chromebasepath
, register
):
174 '''updateManifest replaces the % in the chrome registration entries
175 with the given chrome base path, and updates the given manifest file.
177 lock
= lockFile(manifestPath
+ '.lck')
179 myregister
= dict.fromkeys(map(lambda s
: s
.replace('%', chromebasepath
),
180 register
.iterkeys()))
181 manifestExists
= os
.path
.isfile(manifestPath
)
182 mode
= (manifestExists
and 'r+b') or 'wb'
183 mf
= open(manifestPath
, mode
)
185 # import previous content into hash, ignoring empty ones and comments
186 imf
= re
.compile('(#.*)?$')
187 for l
in re
.split('[\r\n]+', mf
.read()):
192 for k
in myregister
.iterkeys():
193 mf
.write(k
+ os
.linesep
)
198 def makeJar(self
, infile
, jardir
):
199 '''makeJar is the main entry point to JarMaker.
201 It takes the input file, the output directory, the source dirs and the
202 top source dir as argument, and optionally the l10n dirs.
204 # making paths absolute, guess srcdir if file and add to sourcedirs
205 _normpath
= lambda p
: os
.path
.normpath(os
.path
.abspath(p
))
206 self
.topsourcedir
= _normpath(self
.topsourcedir
)
207 self
.sourcedirs
= [_normpath(p
) for p
in self
.sourcedirs
]
209 self
.localedirs
= [_normpath(p
) for p
in self
.localedirs
]
210 elif self
.relativesrcdir
:
211 self
.localedirs
= self
.generateLocaleDirs(self
.relativesrcdir
)
212 if isinstance(infile
, basestring
):
213 logging
.info("processing " + infile
)
214 self
.sourcedirs
.append(_normpath(os
.path
.dirname(infile
)))
217 pp
.do_include(infile
)
218 lines
= pushback_iter(pp
.out
.getvalue().splitlines())
222 m
= self
.jarline
.match(l
)
224 raise RuntimeError(l
)
225 if m
.group('jarfile') is None:
228 self
.processJarSection(m
.group('jarfile'), lines
, jardir
)
229 except StopIteration:
234 def generateLocaleDirs(self
, relativesrcdir
):
235 if os
.path
.basename(relativesrcdir
) == 'locales':
237 l10nrelsrcdir
= os
.path
.dirname(relativesrcdir
)
239 l10nrelsrcdir
= relativesrcdir
241 # generate locales dirs, merge, l10nbase, en-US
243 locdirs
.append(os
.path
.join(self
.l10nmerge
, l10nrelsrcdir
))
245 locdirs
.append(os
.path
.join(self
.l10nbase
, l10nrelsrcdir
))
246 if self
.l10nmerge
or not self
.l10nbase
:
247 # add en-US if we merge, or if it's not l10n
248 locdirs
.append(os
.path
.join(self
.topsourcedir
, relativesrcdir
, 'en-US'))
251 def processJarSection(self
, jarfile
, lines
, jardir
):
252 '''Internal method called by makeJar to actually process a section
255 jarfile is the basename of the jarfile or the directory name for
256 flat output, lines is a pushback_iterator of the lines of jar.mn,
257 the remaining options are carried over from makeJar.
260 # chromebasepath is used for chrome registration manifests
261 # {0} is getting replaced with chrome/ for chrome.manifest, and with
262 # an empty string for jarfile.manifest
263 chromebasepath
= '{0}' + os
.path
.basename(jarfile
)
264 if self
.outputFormat
== 'jar':
265 chromebasepath
= 'jar:' + chromebasepath
+ '.jar!'
266 chromebasepath
+= '/'
268 jarfile
= os
.path
.join(jardir
, jarfile
)
270 if self
.outputFormat
== 'jar':
272 jarfilepath
= jarfile
+ '.jar'
274 os
.makedirs(os
.path
.dirname(jarfilepath
))
275 except OSError as error
:
276 if error
.errno
!= errno
.EEXIST
:
278 jf
= ZipFile(jarfilepath
, 'a', lock
= True)
279 outHelper
= self
.OutputHelper_jar(jf
)
281 outHelper
= getattr(self
, 'OutputHelper_' + self
.outputFormat
)(jarfile
)
283 # This loop exits on either
284 # - the end of the jar.mn file
285 # - an line in the jar.mn file that's not part of a jar section
286 # - on an exception raised, close the jf in that case in a finally
291 except StopIteration:
292 # we're done with this jar.mn, and this jar section
293 self
.finalizeJar(jarfile
, chromebasepath
, register
)
296 # reraise the StopIteration for makeJar
298 if self
.ignore
.match(l
):
300 m
= self
.relsrcline
.match(l
)
302 relativesrcdir
= m
.group('relativesrcdir')
303 self
.localedirs
= self
.generateLocaleDirs(relativesrcdir
)
305 m
= self
.regline
.match(l
)
310 m
= self
.entryline
.match(l
)
312 # neither an entry line nor chrome reg, this jar section is done
313 self
.finalizeJar(jarfile
, chromebasepath
, register
)
318 self
._processEntryLine
(m
, outHelper
, jf
)
324 def _processEntryLine(self
, m
, outHelper
, jf
):
325 out
= m
.group('output')
326 src
= m
.group('source') or os
.path
.basename(out
)
327 # pick the right sourcedir -- l10n, topsrc or src
328 if m
.group('locale'):
329 src_base
= self
.localedirs
330 elif src
.startswith('/'):
331 # path/in/jar/file_name.xul (/path/in/sourcetree/file_name.xul)
332 # refers to a path relative to topsourcedir, use that as base
333 # and strip the leading '/'
334 src_base
= [self
.topsourcedir
]
337 # use srcdirs and the objdir (current working dir) for relative paths
338 src_base
= self
.sourcedirs
+ [os
.getcwd()]
339 # check if the source file exists
341 for _srcdir
in src_base
:
342 if os
.path
.isfile(os
.path
.join(_srcdir
, src
)):
343 realsrc
= os
.path
.join(_srcdir
, src
)
348 raise RuntimeError('File "{0}" not found in {1}'
349 .format(src
, ', '.join(src_base
)))
350 if m
.group('optPreprocess'):
351 outf
= outHelper
.getOutput(out
)
354 if src
[-4:] == '.css':
358 pp
.warnUnused(realsrc
)
362 # copy or symlink if newer or overwrite
363 if (m
.group('optOverwrite')
364 or (getModTime(realsrc
) >
365 outHelper
.getDestModTime(m
.group('output')))):
366 if self
.outputFormat
== 'symlink':
367 outHelper
.symlink(realsrc
, out
)
369 outf
= outHelper
.getOutput(out
)
370 # open in binary mode, this can be images etc
371 inf
= open(realsrc
, 'rb')
372 outf
.write(inf
.read())
377 class OutputHelper_jar(object):
378 '''Provide getDestModTime and getOutput for a given jarfile.
380 def __init__(self
, jarfile
):
381 self
.jarfile
= jarfile
382 def getDestModTime(self
, aPath
):
384 info
= self
.jarfile
.getinfo(aPath
)
385 return info
.date_time
388 def getOutput(self
, name
):
389 return ZipEntry(name
, self
.jarfile
)
391 class OutputHelper_flat(object):
392 '''Provide getDestModTime and getOutput for a given flat
393 output directory. The helper method ensureDirFor is used by
394 the symlink subclass.
396 def __init__(self
, basepath
):
397 self
.basepath
= basepath
398 def getDestModTime(self
, aPath
):
399 return getModTime(os
.path
.join(self
.basepath
, aPath
))
400 def getOutput(self
, name
):
401 out
= self
.ensureDirFor(name
)
402 # remove previous link or file
406 if e
.errno
!= errno
.ENOENT
:
408 return open(out
, 'wb')
409 def ensureDirFor(self
, name
):
410 out
= os
.path
.join(self
.basepath
, name
)
411 outdir
= os
.path
.dirname(out
)
412 if not os
.path
.isdir(outdir
):
415 except OSError as error
:
416 if error
.errno
!= errno
.EEXIST
:
420 class OutputHelper_symlink(OutputHelper_flat
):
421 '''Subclass of OutputHelper_flat that provides a helper for
422 creating a symlink including creating the parent directories.
424 def symlink(self
, src
, dest
):
425 out
= self
.ensureDirFor(dest
)
426 # remove previous link or file
430 if e
.errno
!= errno
.ENOENT
:
432 if sys
.platform
!= "win32":
435 # On Win32, use ctypes to create a hardlink
436 rv
= CreateHardLink(out
, src
, None)
442 p
= jm
.getCommandLineParser()
443 (options
, args
) = p
.parse_args()
444 jm
.processIncludes(options
.I
)
445 jm
.outputFormat
= options
.f
446 jm
.sourcedirs
= options
.s
447 jm
.topsourcedir
= options
.t
449 jm
.useChromeManifest
= True
450 jm
.useJarfileManifest
= False
451 if options
.bothManifests
:
452 jm
.useChromeManifest
= True
453 jm
.useJarfileManifest
= True
454 if options
.l10n_base
:
455 if not options
.relativesrcdir
:
456 p
.error('relativesrcdir required when using l10n-base')
458 p
.error('both l10n-src and l10n-base are not supported')
459 jm
.l10nbase
= options
.l10n_base
460 jm
.relativesrcdir
= options
.relativesrcdir
461 jm
.l10nmerge
= options
.locale_mergedir
462 elif options
.locale_mergedir
:
463 p
.error('l10n-base required when using locale-mergedir')
464 jm
.localedirs
= options
.l10n_src
465 if options
.root_manifest_entry_appid
:
466 jm
.rootManifestAppId
= options
.root_manifest_entry_appid
468 if options
.verbose
is not None:
469 noise
= (options
.verbose
and logging
.DEBUG
) or logging
.WARN
470 if sys
.version_info
[:2] > (2,3):
471 logging
.basicConfig(format
= "%(message)s")
473 logging
.basicConfig()
474 logging
.getLogger().setLevel(noise
)
476 topsrc
= os
.path
.normpath(os
.path
.abspath(topsrc
))
481 jm
.makeJar(infile
, options
.j
)
483 if __name__
== "__main__":