scripts/ninjatool.py

   1 #! /bin/sh
   2
   3 # Python module for parsing and processing .ninja files.
   4 #
   5 # Author: Paolo Bonzini
   6 #
   7 # Copyright (C) 2019 Red Hat, Inc.
   8
   9
  10 # We don't want to put "#! @PYTHON@" as the shebang and
  11 # make the file executable, so instead we make this a
  12 # Python/shell polyglot.  The first line below starts a
  13 # multiline string literal for Python, while it is just
  14 # ":" for bash.  The closing of the multiline string literal
  15 # is never parsed by bash since it exits before.
  16
  17 '''':
  18 case "$0" in
  19   /*) me=$0 ;;
  20   *) me=$(command -v "$0") ;;
  21 esac
  22 python="@PYTHON@"
  23 case $python in
  24   @*) python=python3 ;;
  25 esac
  26 exec $python "$me" "$@"
  27 exit 1
  28 '''
  29
  30
  31 from collections import namedtuple, defaultdict
  32 import sys
  33 import os
  34 import re
  35 import json
  36 import argparse
  37 import shutil
  38
  39
  40 class InvalidArgumentError(Exception):
  41     pass
  42
  43 # faster version of os.path.normpath: do nothing unless there is a double
  44 # slash or a "." or ".." component.  The filter does not have to be super
  45 # precise, but it has to be fast.  os.path.normpath is the hottest function
  46 # for ninja2make without this optimization!
  47 if os.path.sep == '/':
  48     def normpath(path, _slow_re=re.compile('/[./]')):
  49         return os.path.normpath(path) if _slow_re.search(path) or path[0] == '.' else path
  50 else:
  51     normpath = os.path.normpath
  52
  53
  54 # ---- lexer and parser ----
  55
  56 PATH_RE = r"[^$\s:|]+|\$[$ :]|\$[a-zA-Z0-9_-]+|\$\{[a-zA-Z0-9_.-]+\}"
  57
  58 SIMPLE_PATH_RE = re.compile(r"[^$\s:|]+")
  59 IDENT_RE = re.compile(r"[a-zA-Z0-9_.-]+$")
  60 STRING_RE = re.compile(r"(" + PATH_RE + r"|[\s:|])(?:\r?\n)?|.")
  61 TOPLEVEL_RE = re.compile(r"([=:#]|\|\|?|^ +|(?:" + PATH_RE + r")+)\s*|.")
  62 VAR_RE=re.compile(r'\$\$|\$\{([^}]*)\}')
  63
  64 BUILD = 1
  65 POOL = 2
  66 RULE = 3
  67 DEFAULT = 4
  68 EQUALS = 5
  69 COLON = 6
  70 PIPE = 7
  71 PIPE2 = 8
  72 IDENT = 9
  73 INCLUDE = 10
  74 INDENT = 11
  75 EOL = 12
  76
  77
  78 class LexerError(Exception):
  79     pass
  80
  81
  82 class ParseError(Exception):
  83     pass
  84
  85
  86 class NinjaParserEvents(object):
  87     def __init__(self, parser):
  88         self.parser = parser
  89
  90     def dollar_token(self, word, in_path=False):
  91         return '$$' if word == '$' else word
  92
  93     def variable_expansion_token(self, varname):
  94         return '${%s}' % varname
  95
  96     def variable(self, name, arg):
  97         pass
  98
  99     def begin_file(self):
 100         pass
 101
 102     def end_file(self):
 103         pass
 104
 105     def end_scope(self):
 106         pass
 107
 108     def begin_pool(self, name):
 109         pass
 110
 111     def begin_rule(self, name):
 112         pass
 113
 114     def begin_build(self, out, iout, rule, in_, iin, orderdep):
 115         pass
 116
 117     def default(self, targets):
 118         pass
 119
 120
 121 class NinjaParser(object):
 122
 123     InputFile = namedtuple('InputFile', 'filename iter lineno')
 124
 125     def __init__(self, filename, input):
 126         self.stack = []
 127         self.top = None
 128         self.iter = None
 129         self.lineno = None
 130         self.match_keyword = False
 131         self.push(filename, input)
 132
 133     def file_changed(self):
 134         self.iter = self.top.iter
 135         self.lineno = self.top.lineno
 136         if self.top.filename is not None:
 137             os.chdir(os.path.dirname(self.top.filename) or '.')
 138
 139     def push(self, filename, input):
 140         if self.top:
 141             self.top.lineno = self.lineno
 142             self.top.iter = self.iter
 143             self.stack.append(self.top)
 144         self.top = self.InputFile(filename=filename or 'stdin',
 145                                   iter=self._tokens(input), lineno=0)
 146         self.file_changed()
 147
 148     def pop(self):
 149         if len(self.stack):
 150             self.top = self.stack[-1]
 151             self.stack.pop()
 152             self.file_changed()
 153         else:
 154             self.top = self.iter = None
 155
 156     def next_line(self, input):
 157         line = next(input).rstrip()
 158         self.lineno += 1
 159         while len(line) and line[-1] == '$':
 160             line = line[0:-1] + next(input).strip()
 161             self.lineno += 1
 162         return line
 163
 164     def print_token(self, tok):
 165         if tok == EOL:
 166             return "end of line"
 167         if tok == BUILD:
 168             return '"build"'
 169         if tok == POOL:
 170             return '"pool"'
 171         if tok == RULE:
 172             return '"rule"'
 173         if tok == DEFAULT:
 174             return '"default"'
 175         if tok == EQUALS:
 176             return '"="'
 177         if tok == COLON:
 178             return '":"'
 179         if tok == PIPE:
 180             return '"|"'
 181         if tok == PIPE2:
 182             return '"||"'
 183         if tok == INCLUDE:
 184             return '"include"'
 185         if tok == IDENT:
 186             return 'identifier'
 187         return '"%s"' % tok
 188
 189     def error(self, msg):
 190         raise LexerError("%s:%d: %s" % (self.stack[-1].filename, self.lineno, msg))
 191
 192     def parse_error(self, msg):
 193         raise ParseError("%s:%d: %s" % (self.stack[-1].filename, self.lineno, msg))
 194
 195     def expected(self, expected, tok):
 196         msg = "found %s, expected " % (self.print_token(tok), )
 197         for i, exp_tok in enumerate(expected):
 198             if i > 0:
 199                 msg = msg + (' or ' if i == len(expected) - 1 else ', ')
 200             msg = msg + self.print_token(exp_tok)
 201         self.parse_error(msg)
 202
 203     def _variable_tokens(self, value):
 204         for m in STRING_RE.finditer(value):
 205             match = m.group(1)
 206             if not match:
 207                 self.error("unexpected '%s'" % (m.group(0), ))
 208             yield match
 209
 210     def _tokens(self, input):
 211         while True:
 212             try:
 213                 line = self.next_line(input)
 214             except StopIteration:
 215                 return
 216             for m in TOPLEVEL_RE.finditer(line):
 217                 match = m.group(1)
 218                 if not match:
 219                     self.error("unexpected '%s'" % (m.group(0), ))
 220                 if match == ':':
 221                     yield COLON
 222                     continue
 223                 if match == '|':
 224                     yield PIPE
 225                     continue
 226                 if match == '||':
 227                     yield PIPE2
 228                     continue
 229                 if match[0] == ' ':
 230                     yield INDENT
 231                     continue
 232                 if match[0] == '=':
 233                     yield EQUALS
 234                     value = line[m.start() + 1:].lstrip()
 235                     yield from self._variable_tokens(value)
 236                     break
 237                 if match[0] == '#':
 238                     break
 239
 240                 # identifier
 241                 if self.match_keyword:
 242                     if match == 'build':
 243                         yield BUILD
 244                         continue
 245                     if match == 'pool':
 246                         yield POOL
 247                         continue
 248                     if match == 'rule':
 249                         yield RULE
 250                         continue
 251                     if match == 'default':
 252                         yield DEFAULT
 253                         continue
 254                     if match == 'include':
 255                         filename = line[m.start() + 8:].strip()
 256                         self.push(filename, open(filename, 'r'))
 257                         break
 258                     if match == 'subninja':
 259                         self.error('subninja is not supported')
 260                 yield match
 261             yield EOL
 262
 263     def parse(self, events):
 264         global_var = True
 265
 266         def look_for(*expected):
 267             # The last token in the token stream is always EOL.  This
 268             # is exploited to avoid catching StopIteration everywhere.
 269             tok = next(self.iter)
 270             if tok not in expected:
 271                 self.expected(expected, tok)
 272             return tok
 273
 274         def look_for_ident(*expected):
 275             tok = next(self.iter)
 276             if isinstance(tok, str):
 277                 if not IDENT_RE.match(tok):
 278                     self.parse_error('variable expansion not allowed')
 279             elif tok not in expected:
 280                 self.expected(expected + (IDENT,), tok)
 281             return tok
 282
 283         def parse_assignment_rhs(gen, expected, in_path):
 284             tokens = []
 285             for tok in gen:
 286                 if not isinstance(tok, str):
 287                     if tok in expected:
 288                         break
 289                     self.expected(expected + (IDENT,), tok)
 290                 if tok[0] != '$':
 291                     tokens.append(tok)
 292                 elif tok == '$ ' or tok == '$$' or tok == '$:':
 293                     tokens.append(events.dollar_token(tok[1], in_path))
 294                 else:
 295                     var = tok[2:-1] if tok[1] == '{' else tok[1:]
 296                     tokens.append(events.variable_expansion_token(var))
 297             else:
 298                 # gen must have raised StopIteration
 299                 tok = None
 300
 301             if tokens:
 302                 # Fast path avoiding str.join()
 303                 value = tokens[0] if len(tokens) == 1 else ''.join(tokens)
 304             else:
 305                 value = None
 306             return value, tok
 307
 308         def look_for_path(*expected):
 309             # paths in build rules are parsed one space-separated token
 310             # at a time and expanded
 311             token = next(self.iter)
 312             if not isinstance(token, str):
 313                 return None, token
 314             # Fast path if there are no dollar and variable expansion
 315             if SIMPLE_PATH_RE.match(token):
 316                 return token, None
 317             gen = self._variable_tokens(token)
 318             return parse_assignment_rhs(gen, expected, True)
 319
 320         def parse_assignment(tok):
 321             name = tok
 322             assert isinstance(name, str)
 323             look_for(EQUALS)
 324             value, tok = parse_assignment_rhs(self.iter, (EOL,), False)
 325             assert tok == EOL
 326             events.variable(name, value)
 327
 328         def parse_build():
 329             # parse outputs
 330             out = []
 331             iout = []
 332             while True:
 333                 value, tok = look_for_path(COLON, PIPE)
 334                 if value is None:
 335                     break
 336                 out.append(value)
 337             if tok == PIPE:
 338                 while True:
 339                     value, tok = look_for_path(COLON)
 340                     if value is None:
 341                         break
 342                     iout.append(value)
 343
 344             # parse rule
 345             assert tok == COLON
 346             rule = look_for_ident()
 347
 348             # parse inputs and dependencies
 349             in_ = []
 350             iin = []
 351             orderdep = []
 352             while True:
 353                 value, tok = look_for_path(PIPE, PIPE2, EOL)
 354                 if value is None:
 355                     break
 356                 in_.append(value)
 357             if tok == PIPE:
 358                 while True:
 359                     value, tok = look_for_path(PIPE2, EOL)
 360                     if value is None:
 361                         break
 362                     iin.append(value)
 363             if tok == PIPE2:
 364                 while True:
 365                     value, tok = look_for_path(EOL)
 366                     if value is None:
 367                         break
 368                     orderdep.append(value)
 369             assert tok == EOL
 370             events.begin_build(out, iout, rule, in_, iin, orderdep)
 371             nonlocal global_var
 372             global_var = False
 373
 374         def parse_pool():
 375             # pool declarations are ignored.  Just gobble all the variables
 376             ident = look_for_ident()
 377             look_for(EOL)
 378             events.begin_pool(ident)
 379             nonlocal global_var
 380             global_var = False
 381
 382         def parse_rule():
 383             ident = look_for_ident()
 384             look_for(EOL)
 385             events.begin_rule(ident)
 386             nonlocal global_var
 387             global_var = False
 388
 389         def parse_default():
 390             idents = []
 391             while True:
 392                 ident = look_for_ident(EOL)
 393                 if ident == EOL:
 394                     break
 395                 idents.append(ident)
 396             events.default(idents)
 397
 398         def parse_declaration(tok):
 399             if tok == EOL:
 400                 return
 401
 402             nonlocal global_var
 403             if tok == INDENT:
 404                 if global_var:
 405                     self.parse_error('indented line outside rule or edge')
 406                 tok = look_for_ident(EOL)
 407                 if tok == EOL:
 408                     return
 409                 parse_assignment(tok)
 410                 return
 411
 412             if not global_var:
 413                 events.end_scope()
 414                 global_var = True
 415             if tok == POOL:
 416                 parse_pool()
 417             elif tok == BUILD:
 418                 parse_build()
 419             elif tok == RULE:
 420                 parse_rule()
 421             elif tok == DEFAULT:
 422                 parse_default()
 423             elif isinstance(tok, str):
 424                 parse_assignment(tok)
 425             else:
 426                 self.expected((POOL, BUILD, RULE, INCLUDE, DEFAULT, IDENT), tok)
 427
 428         events.begin_file()
 429         while self.iter:
 430             try:
 431                 self.match_keyword = True
 432                 token = next(self.iter)
 433                 self.match_keyword = False
 434                 parse_declaration(token)
 435             except StopIteration:
 436                 self.pop()
 437         events.end_file()
 438
 439
 440 # ---- variable handling ----
 441
 442 def expand(x, rule_vars=None, build_vars=None, global_vars=None):
 443     if x is None:
 444         return None
 445     changed = True
 446     have_dollar_replacement = False
 447     while changed:
 448         changed = False
 449         matches = list(VAR_RE.finditer(x))
 450         if not matches:
 451             break
 452
 453         # Reverse the match so that expanding later matches does not
 454         # invalidate m.start()/m.end() for earlier ones.  Do not reduce $$ to $
 455         # until all variables are dealt with.
 456         for m in reversed(matches):
 457             name = m.group(1)
 458             if not name:
 459                 have_dollar_replacement = True
 460                 continue
 461             changed = True
 462             if build_vars and name in build_vars:
 463                 value = build_vars[name]
 464             elif rule_vars and name in rule_vars:
 465                 value = rule_vars[name]
 466             elif name in global_vars:
 467                 value = global_vars[name]
 468             else:
 469                 value = ''
 470             x = x[:m.start()] + value + x[m.end():]
 471     return x.replace('$$', '$') if have_dollar_replacement else x
 472
 473
 474 class Scope(object):
 475     def __init__(self, events):
 476         self.events = events
 477
 478     def on_left_scope(self):
 479         pass
 480
 481     def on_variable(self, key, value):
 482         pass
 483
 484
 485 class BuildScope(Scope):
 486     def __init__(self, events, out, iout, rule, in_, iin, orderdep, rule_vars):
 487         super().__init__(events)
 488         self.rule = rule
 489         self.out = [events.expand_and_normalize(x) for x in out]
 490         self.in_ = [events.expand_and_normalize(x) for x in in_]
 491         self.iin = [events.expand_and_normalize(x) for x in iin]
 492         self.orderdep = [events.expand_and_normalize(x) for x in orderdep]
 493         self.iout = [events.expand_and_normalize(x) for x in iout]
 494         self.rule_vars = rule_vars
 495         self.build_vars = dict()
 496         self._define_variable('out', ' '.join(self.out))
 497         self._define_variable('in', ' '.join(self.in_))
 498
 499     def expand(self, x):
 500         return self.events.expand(x, self.rule_vars, self.build_vars)
 501
 502     def on_left_scope(self):
 503         self.events.variable('out', self.build_vars['out'])
 504         self.events.variable('in', self.build_vars['in'])
 505         self.events.end_build(self, self.out, self.iout, self.rule, self.in_,
 506                               self.iin, self.orderdep)
 507
 508     def _define_variable(self, key, value):
 509         # The value has been expanded already, quote it for further
 510         # expansion from rule variables
 511         value = value.replace('$', '$$')
 512         self.build_vars[key] = value
 513
 514     def on_variable(self, key, value):
 515         # in and out are at the top of the lookup order and cannot
 516         # be overridden.  Also, unlike what the manual says, build
 517         # variables only lookup global variables.  They never lookup
 518         # rule variables, earlier build variables, or in/out.
 519         if key not in ('in', 'in_newline', 'out'):
 520             self._define_variable(key, self.events.expand(value))
 521
 522
 523 class RuleScope(Scope):
 524     def __init__(self, events, name, vars_dict):
 525         super().__init__(events)
 526         self.name = name
 527         self.vars_dict = vars_dict
 528         self.generator = False
 529
 530     def on_left_scope(self):
 531         self.events.end_rule(self, self.name)
 532
 533     def on_variable(self, key, value):
 534         self.vars_dict[key] = value
 535         if key == 'generator':
 536             self.generator = True
 537
 538
 539 class NinjaParserEventsWithVars(NinjaParserEvents):
 540     def __init__(self, parser):
 541         super().__init__(parser)
 542         self.rule_vars = defaultdict(lambda: dict())
 543         self.global_vars = dict()
 544         self.scope = None
 545
 546     def variable(self, name, value):
 547         if self.scope:
 548             self.scope.on_variable(name, value)
 549         else:
 550             self.global_vars[name] = self.expand(value)
 551
 552     def begin_build(self, out, iout, rule, in_, iin, orderdep):
 553         if rule != 'phony' and rule not in self.rule_vars:
 554             self.parser.parse_error("undefined rule '%s'" % rule)
 555
 556         self.scope = BuildScope(self, out, iout, rule, in_, iin, orderdep, self.rule_vars[rule])
 557
 558     def begin_pool(self, name):
 559         # pool declarations are ignored.  Just gobble all the variables
 560         self.scope = Scope(self)
 561
 562     def begin_rule(self, name):
 563         if name in self.rule_vars:
 564             self.parser.parse_error("duplicate rule '%s'" % name)
 565         self.scope = RuleScope(self, name, self.rule_vars[name])
 566
 567     def end_scope(self):
 568         self.scope.on_left_scope()
 569         self.scope = None
 570
 571     # utility functions:
 572
 573     def expand(self, x, rule_vars=None, build_vars=None):
 574         return expand(x, rule_vars, build_vars, self.global_vars)
 575
 576     def expand_and_normalize(self, x):
 577         return normpath(self.expand(x))
 578
 579     # extra events not present in the superclass:
 580
 581     def end_build(self, scope, out, iout, rule, in_, iin, orderdep):
 582         pass
 583
 584     def end_rule(self, scope, name):
 585         pass
 586
 587
 588 # ---- test client that just prints back whatever it parsed  ----
 589
 590 class Writer(NinjaParserEvents):
 591     ARGS = argparse.ArgumentParser(description='Rewrite input build.ninja to stdout.')
 592
 593     def __init__(self, output, parser, args):
 594         super().__init__(parser)
 595         self.output = output
 596         self.indent = ''
 597         self.had_vars = False
 598
 599     def dollar_token(self, word, in_path=False):
 600         return '$' + word
 601
 602     def print(self, *args, **kwargs):
 603         if len(args):
 604             self.output.write(self.indent)
 605         print(*args, **kwargs, file=self.output)
 606
 607     def variable(self, name, value):
 608         self.print('%s = %s' % (name, value))
 609         self.had_vars = True
 610
 611     def begin_scope(self):
 612         self.indent = '  '
 613         self.had_vars = False
 614
 615     def end_scope(self):
 616         if self.had_vars:
 617             self.print()
 618         self.indent = ''
 619         self.had_vars = False
 620
 621     def begin_pool(self, name):
 622         self.print('pool %s' % name)
 623         self.begin_scope()
 624
 625     def begin_rule(self, name):
 626         self.print('rule %s' % name)
 627         self.begin_scope()
 628
 629     def begin_build(self, outputs, implicit_outputs, rule, inputs, implicit, order_only):
 630         all_outputs = list(outputs)
 631         all_inputs = list(inputs)
 632
 633         if implicit:
 634             all_inputs.append('|')
 635             all_inputs.extend(implicit)
 636         if order_only:
 637             all_inputs.append('||')
 638             all_inputs.extend(order_only)
 639         if implicit_outputs:
 640             all_outputs.append('|')
 641             all_outputs.extend(implicit_outputs)
 642
 643         self.print('build %s: %s' % (' '.join(all_outputs),
 644                                      ' '.join([rule] + all_inputs)))
 645         self.begin_scope()
 646
 647     def default(self, targets):
 648         self.print('default %s' % ' '.join(targets))
 649
 650
 651 # ---- emit compile_commands.json ----
 652
 653 class Compdb(NinjaParserEventsWithVars):
 654     ARGS = argparse.ArgumentParser(description='Emit compile_commands.json.')
 655     ARGS.add_argument('rules', nargs='*',
 656                       help='The ninja rules to emit compilation commands for.')
 657
 658     def __init__(self, output, parser, args):
 659         super().__init__(parser)
 660         self.output = output
 661         self.rules = args.rules
 662         self.sep = ''
 663
 664     def begin_file(self):
 665         self.output.write('[')
 666         self.directory = os.getcwd()
 667
 668     def print_entry(self, **entry):
 669         entry['directory'] = self.directory
 670         self.output.write(self.sep + json.dumps(entry))
 671         self.sep = ',\n'
 672
 673     def begin_build(self, out, iout, rule, in_, iin, orderdep):
 674         if in_ and rule in self.rules:
 675             super().begin_build(out, iout, rule, in_, iin, orderdep)
 676         else:
 677             self.scope = Scope(self)
 678
 679     def end_build(self, scope, out, iout, rule, in_, iin, orderdep):
 680         self.print_entry(command=scope.expand('${command}'), file=in_[0])
 681
 682     def end_file(self):
 683         self.output.write(']\n')
 684
 685
 686 # ---- clean output files ----
 687
 688 class Clean(NinjaParserEventsWithVars):
 689     ARGS = argparse.ArgumentParser(description='Remove output build files.')
 690     ARGS.add_argument('-g', dest='generator', action='store_true',
 691                       help='clean generated files too')
 692
 693     def __init__(self, output, parser, args):
 694         super().__init__(parser)
 695         self.dry_run = args.dry_run
 696         self.verbose = args.verbose or args.dry_run
 697         self.generator = args.generator
 698
 699     def begin_file(self):
 700         print('Cleaning... ', end=(None if self.verbose else ''), flush=True)
 701         self.cnt = 0
 702
 703     def end_file(self):
 704         print('%d files' % self.cnt)
 705
 706     def do_clean(self, *files):
 707         for f in files:
 708             if self.dry_run:
 709                 if os.path.exists(f):
 710                     self.cnt += 1
 711                     print('Would remove ' + f)
 712                     continue
 713             else:
 714                 try:
 715                     if os.path.isdir(f):
 716                         shutil.rmtree(f)
 717                     else:
 718                         os.unlink(f)
 719                     self.cnt += 1
 720                     if self.verbose:
 721                         print('Removed ' + f)
 722                 except FileNotFoundError:
 723                     pass
 724
 725     def end_build(self, scope, out, iout, rule, in_, iin, orderdep):
 726         if rule == 'phony':
 727             return
 728         if self.generator:
 729             rspfile = scope.expand('${rspfile}')
 730             if rspfile:
 731                 self.do_clean(rspfile)
 732         if self.generator or not scope.expand('${generator}'):
 733             self.do_clean(*out, *iout)
 734             depfile = scope.expand('${depfile}')
 735             if depfile:
 736                 self.do_clean(depfile)
 737
 738
 739 # ---- convert build.ninja to makefile ----
 740
 741 class Ninja2Make(NinjaParserEventsWithVars):
 742     ARGS = argparse.ArgumentParser(description='Convert build.ninja to a Makefile.')
 743     ARGS.add_argument('--clean', dest='emit_clean', action='store_true',
 744                       help='Emit clean/distclean rules.')
 745     ARGS.add_argument('--doublecolon', action='store_true',
 746                       help='Emit double-colon rules for phony targets.')
 747     ARGS.add_argument('--omit', metavar='TARGET', nargs='+',
 748                       help='Targets to omit.')
 749
 750     def __init__(self, output, parser, args):
 751         super().__init__(parser)
 752         self.output = output
 753
 754         self.emit_clean = args.emit_clean
 755         self.doublecolon = args.doublecolon
 756         self.omit = set(args.omit)
 757
 758         if self.emit_clean:
 759             self.omit.update(['clean', 'distclean'])
 760
 761         # Lists of targets are kept in memory and emitted only at the
 762         # end because appending is really inefficient in GNU make.
 763         # We only do it when it's O(#rules) or O(#variables), but
 764         # never when it could be O(#targets).
 765         self.depfiles = list()
 766         self.rspfiles = list()
 767         self.build_vars = defaultdict(lambda: dict())
 768         self.rule_targets = defaultdict(lambda: list())
 769         self.stamp_targets = defaultdict(lambda: list())
 770         self.num_stamp = defaultdict(lambda: 0)
 771         self.all_outs = set()
 772         self.all_ins = set()
 773         self.all_phony = set()
 774         self.seen_default = False
 775
 776     def print(self, *args, **kwargs):
 777         print(*args, **kwargs, file=self.output)
 778
 779     def dollar_token(self, word, in_path=False):
 780         if in_path and word == ' ':
 781             self.parser.parse_error('Make does not support spaces in filenames')
 782         return '$$' if word == '$' else word
 783
 784     def print_phony(self, outs, ins):
 785         targets = ' '.join(outs).replace('$', '$$')
 786         deps = ' '.join(ins).replace('$', '$$')
 787         deps = deps.strip()
 788         if self.doublecolon:
 789             self.print(targets + '::' + (' ' if deps else '') + deps + ';@:')
 790         else:
 791             self.print(targets + ':' + (' ' if deps else '') + deps)
 792         self.all_phony.update(outs)
 793
 794     def begin_file(self):
 795         self.print(r'# This is an automatically generated file, and it shows.')
 796         self.print(r'ninja-default:')
 797         self.print(r'.PHONY: ninja-default ninja-clean ninja-distclean')
 798         if self.emit_clean:
 799             self.print(r'ninja-clean:: ninja-clean-start; $(if $V,,@)rm -f ${ninja-depfiles}')
 800             self.print(r'ninja-clean-start:; $(if $V,,@echo Cleaning...)')
 801             self.print(r'ninja-distclean:: clean; $(if $V,,@)rm -f ${ninja-rspfiles}')
 802             self.print(r'.PHONY: ninja-clean-start')
 803             self.print_phony(['clean'], ['ninja-clean'])
 804             self.print_phony(['distclean'], ['ninja-distclean'])
 805         self.print(r'vpath')
 806         self.print(r'NULL :=')
 807         self.print(r'SPACE := ${NULL} #')
 808         self.print(r'MAKEFLAGS += -rR')
 809         self.print(r'define NEWLINE')
 810         self.print(r'')
 811         self.print(r'endef')
 812         self.print(r'.var.in_newline = $(subst $(SPACE),$(NEWLINE),${.var.in})')
 813         self.print(r"ninja-command = $(if $V,,$(if ${.var.description},@printf '%s\n' '$(subst ','\'',${.var.description})' && ))${.var.command}")
 814         self.print(r"ninja-command-restat = $(if $V,,$(if ${.var.description},@printf '%s\n' '$(subst ','\'',${.var.description})' && ))${.var.command} && if test -e $(firstword ${.var.out}); then printf '%s\n' ${.var.out} > $@; fi")
 815
 816     def end_file(self):
 817         def natural_sort_key(s, _nsre=re.compile('([0-9]+)')):
 818             return [int(text) if text.isdigit() else text.lower()
 819                     for text in _nsre.split(s)]
 820
 821         self.print()
 822         self.print('ninja-outputdirs :=')
 823         for rule in self.rule_vars:
 824             if rule == 'phony':
 825                 continue
 826             self.print('ninja-targets-%s := %s' % (rule, ' '.join(self.rule_targets[rule])))
 827             self.print('ninja-stamp-%s := %s' % (rule, ' '.join(self.stamp_targets[rule])))
 828             self.print('ninja-outputdirs += $(sort $(dir ${ninja-targets-%s}))' % rule)
 829             self.print()
 830         self.print('dummy := $(shell mkdir -p . $(sort $(ninja-outputdirs)))')
 831         self.print('ninja-depfiles :=' + ' '.join(self.depfiles))
 832         self.print('ninja-rspfiles :=' + ' '.join(self.rspfiles))
 833         self.print('-include ${ninja-depfiles}')
 834         self.print()
 835         for targets in self.build_vars:
 836             for name, value in self.build_vars[targets].items():
 837                 self.print('%s: private .var.%s := %s' % (targets, name, value))
 838             self.print()
 839         if not self.seen_default:
 840             default_targets = sorted(self.all_outs - self.all_ins, key=natural_sort_key)
 841             self.print('ninja-default: ' + ' '.join(default_targets))
 842
 843         # This is a hack...  Meson declares input meson.build files as
 844         # phony, because Ninja does not have an equivalent of Make's
 845         # "path/to/file:" declaration that ignores "path/to/file" even
 846         # if it is absent.  However, Makefile.ninja wants to depend on
 847         # build.ninja, which in turn depends on these phony targets which
 848         # would cause Makefile.ninja to be rebuilt in a loop.
 849         phony_targets = sorted(self.all_phony - self.all_ins, key=natural_sort_key)
 850         self.print('.PHONY: ' + ' '.join(phony_targets))
 851
 852     def variable(self, name, value):
 853         super().variable(name, value)
 854         if self.scope is None:
 855             self.global_vars[name] = self.expand(value)
 856             self.print('.var.%s := %s' % (name, self.global_vars[name]))
 857
 858     def begin_build(self, out, iout, rule, in_, iin, orderdep):
 859         if any(x in self.omit for x in out):
 860             self.scope = Scope(self)
 861             return
 862
 863         super().begin_build(out, iout, rule, in_, iin, orderdep)
 864         self.current_targets = ' '.join(self.scope.out + self.scope.iout).replace('$', '$$')
 865
 866     def end_build(self, scope, out, iout, rule, in_, iin, orderdep):
 867         self.rule_targets[rule] += self.scope.out
 868         self.rule_targets[rule] += self.scope.iout
 869
 870         self.all_outs.update(self.scope.iout)
 871         self.all_outs.update(self.scope.out)
 872         self.all_ins.update(self.scope.in_)
 873         self.all_ins.update(self.scope.iin)
 874
 875         targets = self.current_targets
 876         self.current_targets = None
 877         if rule == 'phony':
 878             # Phony rules treat order-only dependencies as normal deps
 879             self.print_phony(out + iout, in_ + iin + orderdep)
 880             return
 881
 882         inputs = ' '.join(in_ + iin).replace('$', '$$')
 883         orderonly = ' '.join(orderdep).replace('$', '$$')
 884
 885         rspfile = scope.expand('${rspfile}')
 886         if rspfile:
 887             rspfile_content = scope.expand('${rspfile_content}')
 888             with open(rspfile, 'w') as f:
 889                 f.write(rspfile_content)
 890             inputs += ' ' + rspfile
 891             self.rspfiles.append(rspfile)
 892
 893         restat = 'restat' in self.scope.build_vars or 'restat' in self.rule_vars[rule]
 894         depfile = scope.expand('${depfile}')
 895         build_vars = {
 896             'command': scope.expand('${command}'),
 897             'description': scope.expand('${description}'),
 898             'out': scope.expand('${out}')
 899         }
 900
 901         if restat and not depfile:
 902             if len(out) == 1:
 903                 stamp = out[0] + '.stamp'
 904             else:
 905                 stamp = '%s%d.stamp' %(rule, self.num_stamp[rule])
 906                 self.num_stamp[rule] += 1
 907             self.print('%s: %s; @:' % (targets, stamp))
 908             self.print('%s: %s | %s; ${ninja-command-restat}' % (stamp, inputs, orderonly))
 909             self.rule_targets[rule].append(stamp)
 910             self.stamp_targets[rule].append(stamp)
 911             self.build_vars[stamp] = build_vars
 912         else:
 913             self.print('%s: %s | %s; ${ninja-command}' % (targets, inputs, orderonly))
 914             self.build_vars[targets] = build_vars
 915             if depfile:
 916                 self.depfiles.append(depfile)
 917
 918     def end_rule(self, scope, name):
 919         # Note that the generator pseudo-variable could also be attached
 920         # to a build block rather than a rule.  This is not handled here
 921         # in order to reduce the number of "rm" invocations.  However,
 922         # "ninjatool.py -t clean" does that correctly.
 923         target = 'distclean' if scope.generator else 'clean'
 924         self.print('ninja-%s:: ; $(if $V,,@)rm -f ${ninja-stamp-%s}' % (target, name))
 925         if self.emit_clean:
 926             self.print('ninja-%s:: ; $(if $V,,@)rm -rf ${ninja-targets-%s}' % (target, name))
 927
 928     def default(self, targets):
 929         self.print("ninja-default: " + ' '.join(targets))
 930         self.seen_default = True
 931
 932
 933 # ---- command line parsing ----
 934
 935 # we cannot use subparsers because tools are chosen through the "-t"
 936 # option.
 937
 938 class ToolAction(argparse.Action):
 939     def __init__(self, option_strings, dest, choices, metavar='TOOL', nargs=None, **kwargs):
 940         if nargs is not None:
 941             raise ValueError("nargs not allowed")
 942         super().__init__(option_strings, dest, required=True, choices=choices,
 943                          metavar=metavar, **kwargs)
 944
 945     def __call__(self, parser, namespace, value, option_string):
 946         tool = self.choices[value]
 947         setattr(namespace, self.dest, tool)
 948         tool.ARGS.prog = '%s %s %s' % (parser.prog, option_string, value)
 949
 950
 951 class ToolHelpAction(argparse.Action):
 952     def __init__(self, option_strings, dest, nargs=None, **kwargs):
 953         if nargs is not None:
 954             raise ValueError("nargs not allowed")
 955         super().__init__(option_strings, dest, nargs=0, **kwargs)
 956
 957     def __call__(self, parser, namespace, values, option_string=None):
 958         if namespace.tool:
 959             namespace.tool.ARGS.print_help()
 960         else:
 961             parser.print_help()
 962         parser.exit()
 963
 964
 965 tools = {
 966     'test': Writer,
 967     'ninja2make': Ninja2Make,
 968     'compdb': Compdb,
 969     'clean': Clean,
 970 }
 971
 972 parser = argparse.ArgumentParser(description='Process and transform build.ninja files.',
 973                                  add_help=False)
 974 parser.add_argument('-C', metavar='DIR', dest='dir', default='.',
 975                     help='change to DIR before doing anything else')
 976 parser.add_argument('-f', metavar='FILE', dest='file', default='build.ninja',
 977                     help='specify input build file [default=build.ninja]')
 978 parser.add_argument('-n', dest='dry_run', action='store_true',
 979                     help='do not actually do anything')
 980 parser.add_argument('-v', dest='verbose', action='store_true',
 981                     help='be more verbose')
 982
 983 parser.add_argument('-t', dest='tool', choices=tools, action=ToolAction,
 984                     help='choose the tool to run')
 985 parser.add_argument('-h', '--help', action=ToolHelpAction,
 986                     help='show this help message and exit')
 987
 988 if len(sys.argv) >= 2 and sys.argv[1] == '--version':
 989     print('1.8')
 990     sys.exit(0)
 991
 992 args, tool_args = parser.parse_known_args()
 993 args.tool.ARGS.parse_args(tool_args, args)
 994
 995 os.chdir(args.dir)
 996 with open(args.file, 'r') as f:
 997     parser = NinjaParser(args.file, f)
 998     try:
 999         events = args.tool(sys.stdout, parser, args)
1000     except InvalidArgumentError as e:
1001         parser.error(str(e))
1002     parser.parse(events)