build/manifestparser.py

   1 #!/usr/bin/env python
   2
   3 # This Source Code Form is subject to the terms of the Mozilla Public
   4 # License, v. 2.0. If a copy of the MPL was not distributed with this
   5 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
   6
   7 """
   8 Mozilla universal manifest parser
   9 """
  10
  11 # this file lives at
  12 # http://hg.mozilla.org/automation/ManifestDestiny/raw-file/tip/manifestparser.py
  13
  14 __all__ = ['read_ini', # .ini reader
  15            'ManifestParser', 'TestManifest', 'convert', # manifest handling
  16            'parse', 'ParseError', 'ExpressionParser'] # conditional expression parser
  17
  18 import os
  19 import re
  20 import shutil
  21 import sys
  22 from fnmatch import fnmatch
  23 from optparse import OptionParser
  24
  25 version = '0.5.3' # package version
  26 try:
  27     from setuptools import setup
  28 except:
  29     setup = None
  30
  31 # we need relpath, but it is introduced in python 2.6
  32 # http://docs.python.org/library/os.path.html
  33 try:
  34     relpath = os.path.relpath
  35 except AttributeError:
  36     def relpath(path, start):
  37         """
  38         Return a relative version of a path
  39         from /usr/lib/python2.6/posixpath.py
  40         """
  41
  42         if not path:
  43             raise ValueError("no path specified")
  44
  45         start_list = os.path.abspath(start).split(os.path.sep)
  46         path_list = os.path.abspath(path).split(os.path.sep)
  47
  48         # Work out how much of the filepath is shared by start and path.
  49         i = len(os.path.commonprefix([start_list, path_list]))
  50
  51         rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
  52         if not rel_list:
  53             return os.curdir
  54         return os.path.join(*rel_list)
  55
  56 # expr.py
  57 # from:
  58 # http://k0s.org/mozilla/hg/expressionparser
  59 # http://hg.mozilla.org/users/tmielczarek_mozilla.com/expressionparser
  60
  61 # Implements a top-down parser/evaluator for simple boolean expressions.
  62 # ideas taken from http://effbot.org/zone/simple-top-down-parsing.htm
  63 #
  64 # Rough grammar:
  65 # expr := literal
  66 #       | '(' expr ')'
  67 #       | expr '&&' expr
  68 #       | expr '||' expr
  69 #       | expr '==' expr
  70 #       | expr '!=' expr
  71 # literal := BOOL
  72 #          | INT
  73 #          | STRING
  74 #          | IDENT
  75 # BOOL   := true|false
  76 # INT    := [0-9]+
  77 # STRING := "[^"]*"
  78 # IDENT  := [A-Za-z_]\w*
  79
  80 # Identifiers take their values from a mapping dictionary passed as the second
  81 # argument.
  82
  83 # Glossary (see above URL for details):
  84 # - nud: null denotation
  85 # - led: left detonation
  86 # - lbp: left binding power
  87 # - rbp: right binding power
  88
  89 class ident_token(object):
  90     def __init__(self, value):
  91         self.value = value
  92     def nud(self, parser):
  93         # identifiers take their value from the value mappings passed
  94         # to the parser
  95         return parser.value(self.value)
  96
  97 class literal_token(object):
  98     def __init__(self, value):
  99         self.value = value
 100     def nud(self, parser):
 101         return self.value
 102
 103 class eq_op_token(object):
 104     "=="
 105     def led(self, parser, left):
 106         return left == parser.expression(self.lbp)
 107
 108 class neq_op_token(object):
 109     "!="
 110     def led(self, parser, left):
 111         return left != parser.expression(self.lbp)
 112
 113 class not_op_token(object):
 114     "!"
 115     def nud(self, parser):
 116         return not parser.expression()
 117
 118 class and_op_token(object):
 119     "&&"
 120     def led(self, parser, left):
 121         right = parser.expression(self.lbp)
 122         return left and right
 123
 124 class or_op_token(object):
 125     "||"
 126     def led(self, parser, left):
 127         right = parser.expression(self.lbp)
 128         return left or right
 129
 130 class lparen_token(object):
 131     "("
 132     def nud(self, parser):
 133         expr = parser.expression()
 134         parser.advance(rparen_token)
 135         return expr
 136
 137 class rparen_token(object):
 138     ")"
 139
 140 class end_token(object):
 141     """always ends parsing"""
 142
 143 ### derived literal tokens
 144
 145 class bool_token(literal_token):
 146     def __init__(self, value):
 147         value = {'true':True, 'false':False}[value]
 148         literal_token.__init__(self, value)
 149
 150 class int_token(literal_token):
 151     def __init__(self, value):
 152         literal_token.__init__(self, int(value))
 153
 154 class string_token(literal_token):
 155     def __init__(self, value):
 156         literal_token.__init__(self, value[1:-1])
 157
 158 precedence = [(end_token, rparen_token),
 159               (or_op_token,),
 160               (and_op_token,),
 161               (eq_op_token, neq_op_token),
 162               (lparen_token,),
 163               ]
 164 for index, rank in enumerate(precedence):
 165     for token in rank:
 166         token.lbp = index # lbp = lowest left binding power
 167
 168 class ParseError(Exception):
 169     """errror parsing conditional expression"""
 170
 171 class ExpressionParser(object):
 172     def __init__(self, text, valuemapping, strict=False):
 173         """
 174         Initialize the parser with input |text|, and |valuemapping| as
 175         a dict mapping identifier names to values.
 176         """
 177         self.text = text
 178         self.valuemapping = valuemapping
 179         self.strict = strict
 180
 181     def _tokenize(self):
 182         """
 183         Lex the input text into tokens and yield them in sequence.
 184         """
 185         # scanner callbacks
 186         def bool_(scanner, t): return bool_token(t)
 187         def identifier(scanner, t): return ident_token(t)
 188         def integer(scanner, t): return int_token(t)
 189         def eq(scanner, t): return eq_op_token()
 190         def neq(scanner, t): return neq_op_token()
 191         def or_(scanner, t): return or_op_token()
 192         def and_(scanner, t): return and_op_token()
 193         def lparen(scanner, t): return lparen_token()
 194         def rparen(scanner, t): return rparen_token()
 195         def string_(scanner, t): return string_token(t)
 196         def not_(scanner, t): return not_op_token()
 197
 198         scanner = re.Scanner([
 199             (r"true|false", bool_),
 200             (r"[a-zA-Z_]\w*", identifier),
 201             (r"[0-9]+", integer),
 202             (r'("[^"]*")|(\'[^\']*\')', string_),
 203             (r"==", eq),
 204             (r"!=", neq),
 205             (r"\|\|", or_),
 206             (r"!", not_),
 207             (r"&&", and_),
 208             (r"\(", lparen),
 209             (r"\)", rparen),
 210             (r"\s+", None), # skip whitespace
 211             ])
 212         tokens, remainder = scanner.scan(self.text)
 213         for t in tokens:
 214             yield t
 215         yield end_token()
 216
 217     def value(self, ident):
 218         """
 219         Look up the value of |ident| in the value mapping passed in the
 220         constructor.
 221         """
 222         if self.strict:
 223             return self.valuemapping[ident]
 224         else:
 225             return self.valuemapping.get(ident, None)
 226
 227     def advance(self, expected):
 228         """
 229         Assert that the next token is an instance of |expected|, and advance
 230         to the next token.
 231         """
 232         if not isinstance(self.token, expected):
 233             raise Exception, "Unexpected token!"
 234         self.token = self.iter.next()
 235
 236     def expression(self, rbp=0):
 237         """
 238         Parse and return the value of an expression until a token with
 239         right binding power greater than rbp is encountered.
 240         """
 241         t = self.token
 242         self.token = self.iter.next()
 243         left = t.nud(self)
 244         while rbp < self.token.lbp:
 245             t = self.token
 246             self.token = self.iter.next()
 247             left = t.led(self, left)
 248         return left
 249
 250     def parse(self):
 251         """
 252         Parse and return the value of the expression in the text
 253         passed to the constructor. Raises a ParseError if the expression
 254         could not be parsed.
 255         """
 256         try:
 257             self.iter = self._tokenize()
 258             self.token = self.iter.next()
 259             return self.expression()
 260         except:
 261             raise ParseError("could not parse: %s; variables: %s" % (self.text, self.valuemapping))
 262
 263     __call__ = parse
 264
 265 def parse(text, **values):
 266     """
 267     Parse and evaluate a boolean expression in |text|. Use |values| to look
 268     up the value of identifiers referenced in the expression. Returns the final
 269     value of the expression. A ParseError will be raised if parsing fails.
 270     """
 271     return ExpressionParser(text, values).parse()
 272
 273 def normalize_path(path):
 274     """normalize a relative path"""
 275     if sys.platform.startswith('win'):
 276         return path.replace('/', os.path.sep)
 277     return path
 278
 279 def denormalize_path(path):
 280     """denormalize a relative path"""
 281     if sys.platform.startswith('win'):
 282         return path.replace(os.path.sep, '/')
 283     return path
 284
 285
 286 def read_ini(fp, variables=None, default='DEFAULT',
 287              comments=';#', separators=('=', ':'),
 288              strict=True):
 289     """
 290     read an .ini file and return a list of [(section, values)]
 291     - fp : file pointer or path to read
 292     - variables : default set of variables
 293     - default : name of the section for the default section
 294     - comments : characters that if they start a line denote a comment
 295     - separators : strings that denote key, value separation in order
 296     - strict : whether to be strict about parsing
 297     """
 298
 299     if variables is None:
 300         variables = {}
 301
 302     if isinstance(fp, basestring):
 303         fp = file(fp)
 304
 305     sections = []
 306     key = value = None
 307     section_names = set([])
 308
 309     # read the lines
 310     for line in fp.readlines():
 311
 312         stripped = line.strip()
 313
 314         # ignore blank lines
 315         if not stripped:
 316             # reset key and value to avoid continuation lines
 317             key = value = None
 318             continue
 319
 320         # ignore comment lines
 321         if stripped[0] in comments:
 322             continue
 323
 324         # check for a new section
 325         if len(stripped) > 2 and stripped[0] == '[' and stripped[-1] == ']':
 326             section = stripped[1:-1].strip()
 327             key = value = None
 328
 329             # deal with DEFAULT section
 330             if section.lower() == default.lower():
 331                 if strict:
 332                     assert default not in section_names
 333                 section_names.add(default)
 334                 current_section = variables
 335                 continue
 336
 337             if strict:
 338                 # make sure this section doesn't already exist
 339                 assert section not in section_names
 340
 341             section_names.add(section)
 342             current_section = {}
 343             sections.append((section, current_section))
 344             continue
 345
 346         # if there aren't any sections yet, something bad happen
 347         if not section_names:
 348             raise Exception('No sections found')
 349
 350         # (key, value) pair
 351         for separator in separators:
 352             if separator in stripped:
 353                 key, value = stripped.split(separator, 1)
 354                 key = key.strip()
 355                 value = value.strip()
 356
 357                 if strict:
 358                     # make sure this key isn't already in the section or empty
 359                     assert key
 360                     if current_section is not variables:
 361                         assert key not in current_section
 362
 363                 current_section[key] = value
 364                 break
 365         else:
 366             # continuation line ?
 367             if line[0].isspace() and key:
 368                 value = '%s%s%s' % (value, os.linesep, stripped)
 369                 current_section[key] = value
 370             else:
 371                 # something bad happen!
 372                 raise Exception("Not sure what you're trying to do")
 373
 374     # interpret the variables
 375     def interpret_variables(global_dict, local_dict):
 376         variables = global_dict.copy()
 377         variables.update(local_dict)
 378         return variables
 379
 380     sections = [(i, interpret_variables(variables, j)) for i, j in sections]
 381     return sections
 382
 383
 384 ### objects for parsing manifests
 385
 386 class ManifestParser(object):
 387     """read .ini manifests"""
 388
 389     ### methods for reading manifests
 390
 391     def __init__(self, manifests=(), defaults=None, strict=True):
 392         self._defaults = defaults or {}
 393         self.tests = []
 394         self.strict = strict
 395         self.rootdir = None
 396         self.relativeRoot = None
 397         if manifests:
 398             self.read(*manifests)
 399
 400     def getRelativeRoot(self, root):
 401         return root
 402
 403     def read(self, *filenames, **defaults):
 404
 405         # ensure all files exist
 406         missing = [ filename for filename in filenames
 407                     if not os.path.exists(filename) ]
 408         if missing:
 409             raise IOError('Missing files: %s' % ', '.join(missing))
 410
 411         # process each file
 412         for filename in filenames:
 413
 414             # set the per file defaults
 415             defaults = defaults.copy() or self._defaults.copy()
 416             here = os.path.dirname(os.path.abspath(filename))
 417             defaults['here'] = here
 418
 419             if self.rootdir is None:
 420                 # set the root directory
 421                 # == the directory of the first manifest given
 422                 self.rootdir = here
 423
 424             # read the configuration
 425             sections = read_ini(fp=filename, variables=defaults, strict=self.strict)
 426
 427             # get the tests
 428             for section, data in sections:
 429
 430                 # a file to include
 431                 # TODO: keep track of included file structure:
 432                 # self.manifests = {'manifest.ini': 'relative/path.ini'}
 433                 if section.startswith('include:'):
 434                     include_file = section.split('include:', 1)[-1]
 435                     include_file = normalize_path(include_file)
 436                     if not os.path.isabs(include_file):
 437                         include_file = os.path.join(self.getRelativeRoot(here), include_file)
 438                     if not os.path.exists(include_file):
 439                         if self.strict:
 440                             raise IOError("File '%s' does not exist" % include_file)
 441                         else:
 442                             continue
 443                     include_defaults = data.copy()
 444                     self.read(include_file, **include_defaults)
 445                     continue
 446
 447                 # otherwise an item
 448                 test = data
 449                 test['name'] = section
 450                 test['manifest'] = os.path.abspath(filename)
 451
 452                 # determine the path
 453                 path = test.get('path', section)
 454                 if '://' not in path: # don't futz with URLs
 455                     path = normalize_path(path)
 456                     if not os.path.isabs(path):
 457                         path = os.path.join(here, path)
 458                 test['path'] = path
 459
 460                 # append the item
 461                 self.tests.append(test)
 462
 463     ### methods for querying manifests
 464
 465     def query(self, *checks, **kw):
 466         """
 467         general query function for tests
 468         - checks : callable conditions to test if the test fulfills the query
 469         """
 470         tests = kw.get('tests', None)
 471         if tests is None:
 472             tests = self.tests
 473         retval = []
 474         for test in tests:
 475             for check in checks:
 476                 if not check(test):
 477                     break
 478             else:
 479                 retval.append(test)
 480         return retval
 481
 482     def get(self, _key=None, inverse=False, tags=None, tests=None, **kwargs):
 483         # TODO: pass a dict instead of kwargs since you might hav
 484         # e.g. 'inverse' as a key in the dict
 485
 486         # TODO: tags should just be part of kwargs with None values
 487         # (None == any is kinda weird, but probably still better)
 488
 489         # fix up tags
 490         if tags:
 491             tags = set(tags)
 492         else:
 493             tags = set()
 494
 495         # make some check functions
 496         if inverse:
 497             has_tags = lambda test: not tags.intersection(test.keys())
 498             def dict_query(test):
 499                 for key, value in kwargs.items():
 500                     if test.get(key) == value:
 501                         return False
 502                 return True
 503         else:
 504             has_tags = lambda test: tags.issubset(test.keys())
 505             def dict_query(test):
 506                 for key, value in kwargs.items():
 507                     if test.get(key) != value:
 508                         return False
 509                 return True
 510
 511         # query the tests
 512         tests = self.query(has_tags, dict_query, tests=tests)
 513
 514         # if a key is given, return only a list of that key
 515         # useful for keys like 'name' or 'path'
 516         if _key:
 517             return [test[_key] for test in tests]
 518
 519         # return the tests
 520         return tests
 521
 522     def missing(self, tests=None):
 523         """return list of tests that do not exist on the filesystem"""
 524         if tests is None:
 525             tests = self.tests
 526         return [test for test in tests
 527                 if not os.path.exists(test['path'])]
 528
 529     def manifests(self, tests=None):
 530         """
 531         return manifests in order in which they appear in the tests
 532         """
 533         if tests is None:
 534             tests = self.tests
 535         manifests = []
 536         for test in tests:
 537             manifest = test.get('manifest')
 538             if not manifest:
 539                 continue
 540             if manifest not in manifests:
 541                 manifests.append(manifest)
 542         return manifests
 543
 544     ### methods for outputting from manifests
 545
 546     def write(self, fp=sys.stdout, rootdir=None,
 547               global_tags=None, global_kwargs=None,
 548               local_tags=None, local_kwargs=None):
 549         """
 550         write a manifest given a query
 551         global and local options will be munged to do the query
 552         globals will be written to the top of the file
 553         locals (if given) will be written per test
 554         """
 555
 556         # root directory
 557         if rootdir is None:
 558             rootdir = self.rootdir
 559
 560         # sanitize input
 561         global_tags = global_tags or set()
 562         local_tags = local_tags or set()
 563         global_kwargs = global_kwargs or {}
 564         local_kwargs = local_kwargs or {}
 565
 566         # create the query
 567         tags = set([])
 568         tags.update(global_tags)
 569         tags.update(local_tags)
 570         kwargs = {}
 571         kwargs.update(global_kwargs)
 572         kwargs.update(local_kwargs)
 573
 574         # get matching tests
 575         tests = self.get(tags=tags, **kwargs)
 576
 577         # print the .ini manifest
 578         if global_tags or global_kwargs:
 579             print >> fp, '[DEFAULT]'
 580             for tag in global_tags:
 581                 print >> fp, '%s =' % tag
 582             for key, value in global_kwargs.items():
 583                 print >> fp, '%s = %s' % (key, value)
 584             print >> fp
 585
 586         for test in tests:
 587             test = test.copy() # don't overwrite
 588
 589             path = test['name']
 590             if not os.path.isabs(path):
 591                 path = denormalize_path(relpath(test['path'], self.rootdir))
 592             print >> fp, '[%s]' % path
 593
 594             # reserved keywords:
 595             reserved = ['path', 'name', 'here', 'manifest']
 596             for key in sorted(test.keys()):
 597                 if key in reserved:
 598                     continue
 599                 if key in global_kwargs:
 600                     continue
 601                 if key in global_tags and not test[key]:
 602                     continue
 603                 print >> fp, '%s = %s' % (key, test[key])
 604             print >> fp
 605
 606     def copy(self, directory, rootdir=None, *tags, **kwargs):
 607         """
 608         copy the manifests and associated tests
 609         - directory : directory to copy to
 610         - rootdir : root directory to copy to (if not given from manifests)
 611         - tags : keywords the tests must have
 612         - kwargs : key, values the tests must match
 613         """
 614         # XXX note that copy does *not* filter the tests out of the
 615         # resulting manifest; it just stupidly copies them over.
 616         # ideally, it would reread the manifests and filter out the
 617         # tests that don't match *tags and **kwargs
 618
 619         # destination
 620         if not os.path.exists(directory):
 621             os.path.makedirs(directory)
 622         else:
 623             # sanity check
 624             assert os.path.isdir(directory)
 625
 626         # tests to copy
 627         tests = self.get(tags=tags, **kwargs)
 628         if not tests:
 629             return # nothing to do!
 630
 631         # root directory
 632         if rootdir is None:
 633             rootdir = self.rootdir
 634
 635         # copy the manifests + tests
 636         manifests = [relpath(manifest, rootdir) for manifest in self.manifests()]
 637         for manifest in manifests:
 638             destination = os.path.join(directory, manifest)
 639             dirname = os.path.dirname(destination)
 640             if not os.path.exists(dirname):
 641                 os.makedirs(dirname)
 642             else:
 643                 # sanity check
 644                 assert os.path.isdir(dirname)
 645             shutil.copy(os.path.join(rootdir, manifest), destination)
 646         for test in tests:
 647             if os.path.isabs(test['name']):
 648                 continue
 649             source = test['path']
 650             if not os.path.exists(source):
 651                 print >> sys.stderr, "Missing test: '%s' does not exist!" % source
 652                 continue
 653                 # TODO: should err on strict
 654             destination = os.path.join(directory, relpath(test['path'], rootdir))
 655             shutil.copy(source, destination)
 656             # TODO: ensure that all of the tests are below the from_dir
 657
 658     def update(self, from_dir, rootdir=None, *tags, **kwargs):
 659         """
 660         update the tests as listed in a manifest from a directory
 661         - from_dir : directory where the tests live
 662         - rootdir : root directory to copy to (if not given from manifests)
 663         - tags : keys the tests must have
 664         - kwargs : key, values the tests must match
 665         """
 666
 667         # get the tests
 668         tests = self.get(tags=tags, **kwargs)
 669
 670         # get the root directory
 671         if not rootdir:
 672             rootdir = self.rootdir
 673
 674         # copy them!
 675         for test in tests:
 676             if not os.path.isabs(test['name']):
 677                 _relpath = relpath(test['path'], rootdir)
 678                 source = os.path.join(from_dir, _relpath)
 679                 if not os.path.exists(source):
 680                     # TODO err on strict
 681                     print >> sys.stderr, "Missing test: '%s'; skipping" % test['name']
 682                     continue
 683                 destination = os.path.join(rootdir, _relpath)
 684                 shutil.copy(source, destination)
 685
 686
 687 class TestManifest(ManifestParser):
 688     """
 689     apply logic to manifests;  this is your integration layer :)
 690     specific harnesses may subclass from this if they need more logic
 691     """
 692
 693     def filter(self, values, tests):
 694         """
 695         filter on a specific list tag, e.g.:
 696         run-if.os = win linux
 697         skip-if.os = mac
 698         """
 699
 700         # tags:
 701         run_tag = 'run-if'
 702         skip_tag = 'skip-if'
 703         fail_tag = 'fail-if'
 704
 705         # loop over test
 706         for test in tests:
 707             reason = None # reason to disable
 708
 709             # tagged-values to run
 710             if run_tag in test:
 711                 condition = test[run_tag]
 712                 if not parse(condition, **values):
 713                     reason = '%s: %s' % (run_tag, condition)
 714
 715             # tagged-values to skip
 716             if skip_tag in test:
 717                 condition = test[skip_tag]
 718                 if parse(condition, **values):
 719                     reason = '%s: %s' % (skip_tag, condition)
 720
 721             # mark test as disabled if there's a reason
 722             if reason:
 723                 test.setdefault('disabled', reason)
 724
 725             # mark test as a fail if so indicated
 726             if fail_tag in test:
 727                 condition = test[fail_tag]
 728                 if parse(condition, **values):
 729                     test['expected'] = 'fail'
 730
 731     def active_tests(self, exists=True, disabled=True, **values):
 732         """
 733         - exists : return only existing tests
 734         - disabled : whether to return disabled tests
 735         - tags : keys and values to filter on (e.g. `os = linux mac`)
 736         """
 737
 738         tests = [i.copy() for i in self.tests] # shallow copy
 739
 740         # mark all tests as passing unless indicated otherwise
 741         for test in tests:
 742             test['expected'] = test.get('expected', 'pass')
 743
 744         # ignore tests that do not exist
 745         if exists:
 746             tests = [test for test in tests if os.path.exists(test['path'])]
 747
 748         # filter by tags
 749         self.filter(values, tests)
 750
 751         # ignore disabled tests if specified
 752         if not disabled:
 753             tests = [test for test in tests
 754                      if not 'disabled' in test]
 755
 756         # return active tests
 757         return tests
 758
 759     def test_paths(self):
 760         return [test['path'] for test in self.active_tests()]
 761
 762
 763 ### utility function(s); probably belongs elsewhere
 764
 765 def convert(directories, pattern=None, ignore=(), write=None):
 766     """
 767     convert directories to a simple manifest
 768     """
 769
 770     retval = []
 771     include = []
 772     for directory in directories:
 773         for dirpath, dirnames, filenames in os.walk(directory):
 774
 775             # filter out directory names
 776             dirnames = [ i for i in dirnames if i not in ignore ]
 777             dirnames.sort()
 778
 779             # reference only the subdirectory
 780             _dirpath = dirpath
 781             dirpath = dirpath.split(directory, 1)[-1].strip(os.path.sep)
 782
 783             if dirpath.split(os.path.sep)[0] in ignore:
 784                 continue
 785
 786             # filter by glob
 787             if pattern:
 788                 filenames = [filename for filename in filenames
 789                              if fnmatch(filename, pattern)]
 790
 791             filenames.sort()
 792
 793             # write a manifest for each directory
 794             if write and (dirnames or filenames):
 795                 manifest = file(os.path.join(_dirpath, write), 'w')
 796                 for dirname in dirnames:
 797                     print >> manifest, '[include:%s]' % os.path.join(dirname, write)
 798                 for filename in filenames:
 799                     print >> manifest, '[%s]' % filename
 800                 manifest.close()
 801
 802             # add to the list
 803             retval.extend([denormalize_path(os.path.join(dirpath, filename))
 804                            for filename in filenames])
 805
 806     if write:
 807         return # the manifests have already been written!
 808
 809     retval.sort()
 810     retval = ['[%s]' % filename for filename in retval]
 811     return '\n'.join(retval)
 812
 813 ### command line attributes
 814
 815 class ParserError(Exception):
 816   """error for exceptions while parsing the command line"""
 817
 818 def parse_args(_args):
 819     """
 820     parse and return:
 821     --keys=value (or --key value)
 822     -tags
 823     args
 824     """
 825
 826     # return values
 827     _dict = {}
 828     tags = []
 829     args = []
 830
 831     # parse the arguments
 832     key = None
 833     for arg in _args:
 834         if arg.startswith('---'):
 835             raise ParserError("arguments should start with '-' or '--' only")
 836         elif arg.startswith('--'):
 837             if key:
 838                 raise ParserError("Key %s still open" % key)
 839             key = arg[2:]
 840             if '=' in key:
 841                 key, value = key.split('=', 1)
 842                 _dict[key] = value
 843                 key = None
 844                 continue
 845         elif arg.startswith('-'):
 846             if key:
 847                 raise ParserError("Key %s still open" % key)
 848             tags.append(arg[1:])
 849             continue
 850         else:
 851             if key:
 852                 _dict[key] = arg
 853                 continue
 854             args.append(arg)
 855
 856     # return values
 857     return (_dict, tags, args)
 858
 859
 860 ### classes for subcommands
 861
 862 class CLICommand(object):
 863     usage = '%prog [options] command'
 864     def __init__(self, parser):
 865       self._parser = parser # master parser
 866     def parser(self):
 867       return OptionParser(usage=self.usage, description=self.__doc__,
 868                           add_help_option=False)
 869
 870 class Copy(CLICommand):
 871     usage = '%prog [options] copy manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
 872     def __call__(self, options, args):
 873       # parse the arguments
 874       try:
 875         kwargs, tags, args = parse_args(args)
 876       except ParserError, e:
 877         self._parser.error(e.message)
 878
 879       # make sure we have some manifests, otherwise it will
 880       # be quite boring
 881       if not len(args) == 2:
 882         HelpCLI(self._parser)(options, ['copy'])
 883         return
 884
 885       # read the manifests
 886       # TODO: should probably ensure these exist here
 887       manifests = ManifestParser()
 888       manifests.read(args[0])
 889
 890       # print the resultant query
 891       manifests.copy(args[1], None, *tags, **kwargs)
 892
 893
 894 class CreateCLI(CLICommand):
 895     """
 896     create a manifest from a list of directories
 897     """
 898     usage = '%prog [options] create directory <directory> <...>'
 899
 900     def parser(self):
 901         parser = CLICommand.parser(self)
 902         parser.add_option('-p', '--pattern', dest='pattern',
 903                           help="glob pattern for files")
 904         parser.add_option('-i', '--ignore', dest='ignore',
 905                           default=[], action='append',
 906                           help='directories to ignore')
 907         parser.add_option('-w', '--in-place', dest='in_place',
 908                           help='Write .ini files in place; filename to write to')
 909         return parser
 910
 911     def __call__(self, _options, args):
 912         parser = self.parser()
 913         options, args = parser.parse_args(args)
 914
 915         # need some directories
 916         if not len(args):
 917             parser.print_usage()
 918             return
 919
 920         # add the directories to the manifest
 921         for arg in args:
 922             assert os.path.exists(arg)
 923             assert os.path.isdir(arg)
 924             manifest = convert(args, pattern=options.pattern, ignore=options.ignore,
 925                                write=options.in_place)
 926         if manifest:
 927             print manifest
 928
 929
 930 class WriteCLI(CLICommand):
 931     """
 932     write a manifest based on a query
 933     """
 934     usage = '%prog [options] write manifest <manifest> -tag1 -tag2 --key1=value1 --key2=value2 ...'
 935     def __call__(self, options, args):
 936
 937         # parse the arguments
 938         try:
 939             kwargs, tags, args = parse_args(args)
 940         except ParserError, e:
 941             self._parser.error(e.message)
 942
 943         # make sure we have some manifests, otherwise it will
 944         # be quite boring
 945         if not args:
 946             HelpCLI(self._parser)(options, ['write'])
 947             return
 948
 949         # read the manifests
 950         # TODO: should probably ensure these exist here
 951         manifests = ManifestParser()
 952         manifests.read(*args)
 953
 954         # print the resultant query
 955         manifests.write(global_tags=tags, global_kwargs=kwargs)
 956
 957
 958 class HelpCLI(CLICommand):
 959     """
 960     get help on a command
 961     """
 962     usage = '%prog [options] help [command]'
 963
 964     def __call__(self, options, args):
 965         if len(args) == 1 and args[0] in commands:
 966             commands[args[0]](self._parser).parser().print_help()
 967         else:
 968             self._parser.print_help()
 969             print '\nCommands:'
 970             for command in sorted(commands):
 971                 print '  %s : %s' % (command, commands[command].__doc__.strip())
 972
 973 class SetupCLI(CLICommand):
 974     """
 975     setup using setuptools
 976     """
 977     # use setup.py from the repo when you want to distribute to python!
 978     # otherwise setuptools will complain that it can't find setup.py
 979     # and result in a useless package
 980
 981     usage = '%prog [options] setup [setuptools options]'
 982
 983     def __call__(self, options, args):
 984         sys.argv = [sys.argv[0]] + args
 985         assert setup is not None, "You must have setuptools installed to use SetupCLI"
 986         here = os.path.dirname(os.path.abspath(__file__))
 987         try:
 988             filename = os.path.join(here, 'README.txt')
 989             description = file(filename).read()
 990         except:
 991             description = ''
 992         os.chdir(here)
 993
 994         setup(name='ManifestDestiny',
 995               version=version,
 996               description="Universal manifests for Mozilla test harnesses",
 997               long_description=description,
 998               classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
 999               keywords='mozilla manifests',
1000               author='Jeff Hammel',
1001               author_email='jhammel@mozilla.com',
1002               url='https://wiki.mozilla.org/Auto-tools/Projects/ManifestDestiny',
1003               license='MPL',
1004               zip_safe=False,
1005               py_modules=['manifestparser'],
1006               install_requires=[
1007                   # -*- Extra requirements: -*-
1008                   ],
1009               entry_points="""
1010               [console_scripts]
1011               manifestparser = manifestparser:main
1012               """,
1013               )
1014
1015
1016 class UpdateCLI(CLICommand):
1017     """
1018     update the tests as listed in a manifest from a directory
1019     """
1020     usage = '%prog [options] update manifest directory -tag1 -tag2 --key1=value1 --key2=value2 ...'
1021
1022     def __call__(self, options, args):
1023         # parse the arguments
1024         try:
1025             kwargs, tags, args = parse_args(args)
1026         except ParserError, e:
1027             self._parser.error(e.message)
1028
1029         # make sure we have some manifests, otherwise it will
1030         # be quite boring
1031         if not len(args) == 2:
1032             HelpCLI(self._parser)(options, ['update'])
1033             return
1034
1035         # read the manifests
1036         # TODO: should probably ensure these exist here
1037         manifests = ManifestParser()
1038         manifests.read(args[0])
1039
1040         # print the resultant query
1041         manifests.update(args[1], None, *tags, **kwargs)
1042
1043
1044 # command -> class mapping
1045 commands = { 'create': CreateCLI,
1046              'help': HelpCLI,
1047              'update': UpdateCLI,
1048              'write': WriteCLI }
1049 if setup is not None:
1050     commands['setup'] = SetupCLI
1051
1052 def main(args=sys.argv[1:]):
1053     """console_script entry point"""
1054
1055     # set up an option parser
1056     usage = '%prog [options] [command] ...'
1057     description = __doc__
1058     parser = OptionParser(usage=usage, description=description)
1059     parser.add_option('-s', '--strict', dest='strict',
1060                       action='store_true', default=False,
1061                       help='adhere strictly to errors')
1062     parser.disable_interspersed_args()
1063
1064     options, args = parser.parse_args(args)
1065
1066     if not args:
1067         HelpCLI(parser)(options, args)
1068         parser.exit()
1069
1070     # get the command
1071     command = args[0]
1072     if command not in commands:
1073         parser.error("Command must be one of %s (you gave '%s')" % (', '.join(sorted(commands.keys())), command))
1074
1075     handler = commands[command](parser)
1076     handler(options, args[1:])
1077
1078 if __name__ == '__main__':
1079     main()