tools/idl_parser/idl_parser.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """ Parser for PPAPI IDL """
   7
   8 #
   9 # IDL Parser
  10 #
  11 # The parser is uses the PLY yacc library to build a set of parsing rules based
  12 # on WebIDL.
  13 #
  14 # WebIDL, and WebIDL grammar can be found at:
  15 #   http://heycam.github.io/webidl/
  16 # PLY can be found at:
  17 #   http://www.dabeaz.com/ply/
  18 #
  19 # The parser generates a tree by recursively matching sets of items against
  20 # defined patterns.  When a match is made, that set of items is reduced
  21 # to a new item.   The new item can provide a match for parent patterns.
  22 # In this way an AST is built (reduced) depth first.
  23 #
  24
  25 #
  26 # Disable check for line length and Member as Function due to how grammar rules
  27 # are defined with PLY
  28 #
  29 # pylint: disable=R0201
  30 # pylint: disable=C0301
  31
  32 import os.path
  33 import sys
  34 import time
  35
  36 from idl_lexer import IDLLexer
  37 from idl_node import IDLAttribute, IDLNode
  38
  39 #
  40 # Try to load the ply module, if not, then assume it is in the third_party
  41 # directory.
  42 #
  43 try:
  44   # Disable lint check which fails to find the ply module.
  45   # pylint: disable=F0401
  46   from ply import lex
  47   from ply import yacc
  48 except ImportError:
  49   module_path, module_name = os.path.split(__file__)
  50   third_party = os.path.join(module_path, os.par, os.par, 'third_party')
  51   sys.path.append(third_party)
  52   # pylint: disable=F0401
  53   from ply import lex
  54   from ply import yacc
  55
  56 #
  57 # ERROR_REMAP
  58 #
  59 # Maps the standard error formula into a more friendly error message.
  60 #
  61 ERROR_REMAP = {
  62   'Unexpected ")" after "(".' : 'Empty argument list.',
  63   'Unexpected ")" after ",".' : 'Missing argument.',
  64   'Unexpected "}" after ",".' : 'Trailing comma in block.',
  65   'Unexpected "}" after "{".' : 'Unexpected empty block.',
  66   'Unexpected comment after "}".' : 'Unexpected trailing comment.',
  67   'Unexpected "{" after keyword "enum".' : 'Enum missing name.',
  68   'Unexpected "{" after keyword "struct".' : 'Struct missing name.',
  69   'Unexpected "{" after keyword "interface".' : 'Interface missing name.',
  70 }
  71
  72
  73 def Boolean(val):
  74   """Convert to strict boolean type."""
  75   if val:
  76     return True
  77   return False
  78
  79
  80 def ListFromConcat(*items):
  81   """Generate list by concatenating inputs"""
  82   itemsout = []
  83   for item in items:
  84     if item is None:
  85       continue
  86     if type(item) is not type([]):
  87       itemsout.append(item)
  88     else:
  89       itemsout.extend(item)
  90
  91   return itemsout
  92
  93 def ExpandProduction(p):
  94   if type(p) == list:
  95     return '[' + ', '.join([ExpandProduction(x) for x in p]) + ']'
  96   if type(p) == IDLNode:
  97     return 'Node:' + str(p)
  98   if type(p) == IDLAttribute:
  99     return 'Attr:' + str(p)
 100   if type(p) == str:
 101     return 'str:' + p
 102   return '%s:%s' % (p.__class__.__name__, str(p))
 103
 104 # TokenTypeName
 105 #
 106 # Generate a string which has the type and value of the token.
 107 #
 108 def TokenTypeName(t):
 109   if t.type == 'SYMBOL':
 110     return 'symbol %s' % t.value
 111   if t.type in ['HEX', 'INT', 'OCT', 'FLOAT']:
 112     return 'value %s' % t.value
 113   if t.type == 'string' :
 114     return 'string "%s"' % t.value
 115   if t.type == 'COMMENT' :
 116     return 'comment'
 117   if t.type == t.value:
 118     return '"%s"' % t.value
 119   if t.type == ',':
 120     return 'Comma'
 121   if t.type == 'identifier':
 122     return 'identifier "%s"' % t.value
 123   return 'keyword "%s"' % t.value
 124
 125
 126 #
 127 # IDL Parser
 128 #
 129 # The Parser inherits the from the Lexer to provide PLY with the tokenizing
 130 # definitions.  Parsing patterns are encoded as functions where p_<name> is
 131 # is called any time a patern matching the function documentation is found.
 132 # Paterns are expressed in the form of:
 133 # """ <new item> : <item> ....
 134 #                | <item> ...."""
 135 #
 136 # Where new item is the result of a match against one or more sets of items
 137 # separated by the "|".
 138 #
 139 # The function is called with an object 'p' where p[0] is the output object
 140 # and p[n] is the set of inputs for positive values of 'n'.  Len(p) can be
 141 # used to distinguish between multiple item sets in the pattern.
 142 #
 143 # For more details on parsing refer to the PLY documentation at
 144 #    http://www.dabeaz.com/ply/
 145 #
 146 # The parser is based on the WebIDL standard.  See:
 147 #    http://heycam.github.io/webidl/#idl-grammar
 148 #
 149 # The various productions are annotated so that the WHOLE number greater than
 150 # zero in the comment denotes the matching WebIDL grammar definition.
 151 #
 152 # Productions with a fractional component in the comment denote additions to
 153 # the WebIDL spec, such as comments.
 154 #
 155
 156
 157 class IDLParser(object):
 158 #
 159 # We force all input files to start with two comments.  The first comment is a
 160 # Copyright notice followed by a file comment and finally by file level
 161 # productions.
 162 #
 163   # [0] Insert a TOP definition for Copyright and Comments
 164   def p_Top(self, p):
 165     """Top : COMMENT COMMENT Definitions"""
 166     Copyright = self.BuildComment('Copyright', p, 1)
 167     Filedoc = self.BuildComment('Comment', p, 2)
 168     p[0] = ListFromConcat(Copyright, Filedoc, p[3])
 169
 170   # [0.1] Add support for Multiple COMMENTS
 171   def p_Comments(self, p):
 172     """Comments : CommentsRest"""
 173     if len(p) > 1:
 174       p[0] = p[1]
 175
 176   # [0.2] Produce a COMMENT and aggregate sibling comments
 177   def p_CommentsRest(self, p):
 178     """CommentsRest : COMMENT CommentsRest
 179                     | """
 180     if len(p) > 1:
 181       p[0] = ListFromConcat(self.BuildComment('Comment', p, 1), p[2])
 182
 183
 184 #
 185 #The parser is based on the WebIDL standard.  See:
 186 # http://heycam.github.io/webidl/#idl-grammar
 187 #
 188   # [1]
 189   def p_Definitions(self, p):
 190     """Definitions : ExtendedAttributeList Definition Definitions
 191            | """
 192     if len(p) > 1:
 193       p[2].AddChildren(p[1])
 194       p[0] = ListFromConcat(p[2], p[3])
 195
 196   # [2]
 197   def p_Definition(self, p):
 198     """Definition : CallbackOrInterface
 199                   | Partial
 200                   | Dictionary
 201                   | Exception
 202                   | Enum
 203                   | Typedef
 204                   | ImplementsStatement"""
 205     p[0] = p[1]
 206
 207   # [2.1] Error recovery for definition
 208   def p_DefinitionError(self, p):
 209     """Definition : error ';'"""
 210     p[0] = self.BuildError(p, 'Definition')
 211
 212   # [3]
 213   def p_CallbackOrInterface(self, p):
 214     """CallbackOrInterface : CALLBACK CallbackRestOrInterface
 215                            | Interface"""
 216     if len(p) > 2:
 217       p[0] = p[2]
 218     else:
 219       p[0] = p[1]
 220
 221   # [4]
 222   def p_CallbackRestOrInterface(self, p):
 223     """CallbackRestOrInterface : CallbackRest
 224                                | Interface"""
 225     p[0] = p[1]
 226
 227   # [5]
 228   def p_Interface(self, p):
 229     """Interface : INTERFACE identifier Inheritance '{' InterfaceMembers '}' ';'"""
 230     p[0] = self.BuildNamed('Interface', p, 2, ListFromConcat(p[3], p[5]))
 231
 232   # [5.1] Error recovery for interface.
 233   def p_InterfaceError(self, p):
 234     """Interface : INTERFACE identifier Inheritance '{' error"""
 235     p[0] = self.BuildError(p, 'Interface')
 236
 237   # [6]
 238   def p_Partial(self, p):
 239     """Partial : PARTIAL PartialDefinition"""
 240     p[2].AddChildren(self.BuildTrue('Partial'))
 241     p[0] = p[2]
 242
 243   # [6.1] Error recovery for Partial
 244   def p_PartialError(self, p):
 245     """Partial : PARTIAL error"""
 246     p[0] = self.BuildError(p, 'Partial')
 247
 248   # [7]
 249   def p_PartialDefinition(self, p):
 250     """PartialDefinition : PartialDictionary
 251                          | PartialInterface"""
 252     p[0] = p[1]
 253
 254   # [8]
 255   def p_PartialInterface(self, p):
 256     """PartialInterface : INTERFACE identifier '{' InterfaceMembers '}' ';'"""
 257     p[0] = self.BuildNamed('Interface', p, 2, p[4])
 258
 259   # [9]
 260   def p_InterfaceMembers(self, p):
 261     """InterfaceMembers : ExtendedAttributeList InterfaceMember InterfaceMembers
 262                         |"""
 263     if len(p) > 1:
 264       p[2].AddChildren(p[1])
 265       p[0] = ListFromConcat(p[2], p[3])
 266
 267   # [9.1] Error recovery for InterfaceMembers
 268   def p_InterfaceMembersError(self, p):
 269     """InterfaceMembers : error"""
 270     p[0] = self.BuildError(p, 'InterfaceMembers')
 271
 272   # [10] Removed unsupported: Serializer
 273   def p_InterfaceMember(self, p):
 274     """InterfaceMember : Const
 275                        | Operation
 276                        | Stringifier
 277                        | StaticMember
 278                        | Iterable
 279                        | ReadonlyMember
 280                        | ReadWriteAttribute
 281                        | ReadWriteMaplike
 282                        | ReadWriteSetlike"""
 283     p[0] = p[1]
 284
 285   # [11]
 286   def p_Dictionary(self, p):
 287     """Dictionary : DICTIONARY identifier Inheritance '{' DictionaryMembers '}' ';'"""
 288     p[0] = self.BuildNamed('Dictionary', p, 2, ListFromConcat(p[3], p[5]))
 289
 290   # [11.1] Error recovery for regular Dictionary
 291   def p_DictionaryError(self, p):
 292     """Dictionary : DICTIONARY error ';'"""
 293     p[0] = self.BuildError(p, 'Dictionary')
 294
 295   # [11.2] Error recovery for regular Dictionary
 296   # (for errors inside dictionary definition)
 297   def p_DictionaryError2(self, p):
 298     """Dictionary : DICTIONARY identifier Inheritance '{' error"""
 299     p[0] = self.BuildError(p, 'Dictionary')
 300
 301   # [12]
 302   def p_DictionaryMembers(self, p):
 303     """DictionaryMembers : ExtendedAttributeList DictionaryMember DictionaryMembers
 304                          |"""
 305     if len(p) > 1:
 306       p[2].AddChildren(p[1])
 307       p[0] = ListFromConcat(p[2], p[3])
 308
 309   # [13]
 310   def p_DictionaryMember(self, p):
 311     """DictionaryMember : Type identifier Default ';'"""
 312     p[0] = self.BuildNamed('Key', p, 2, ListFromConcat(p[1], p[3]))
 313
 314   # [14] NOT IMPLEMENTED (Required)
 315
 316   # [15]
 317   def p_PartialDictionary(self, p):
 318     """PartialDictionary : DICTIONARY identifier '{' DictionaryMembers '}' ';'"""
 319     partial = self.BuildTrue('Partial')
 320     p[0] = self.BuildNamed('Dictionary', p, 2, ListFromConcat(p[4], partial))
 321
 322   # [15.1] Error recovery for Partial Dictionary
 323   def p_PartialDictionaryError(self, p):
 324     """PartialDictionary : DICTIONARY error ';'"""
 325     p[0] = self.BuildError(p, 'PartialDictionary')
 326
 327   # [16]
 328   def p_Default(self, p):
 329     """Default : '=' DefaultValue
 330                |"""
 331     if len(p) > 1:
 332       p[0] = self.BuildProduction('Default', p, 2, p[2])
 333
 334   # [17]
 335   def p_DefaultValue(self, p):
 336     """DefaultValue : ConstValue
 337                     | string
 338                     | '[' ']'"""
 339     if len(p) == 3:
 340       p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'sequence'),
 341                             self.BuildAttribute('VALUE', '[]'))
 342     elif type(p[1]) == str:
 343       p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'DOMString'),
 344                             self.BuildAttribute('NAME', p[1]))
 345     else:
 346       p[0] = p[1]
 347
 348   # [] - Not specified
 349   def p_Exception(self, p):
 350     """Exception : EXCEPTION identifier Inheritance '{' ExceptionMembers '}' ';'"""
 351     p[0] = self.BuildNamed('Exception', p, 2, ListFromConcat(p[3], p[5]))
 352
 353   # [] - Not specified
 354   def p_ExceptionMembers(self, p):
 355     """ExceptionMembers : ExtendedAttributeList ExceptionMember ExceptionMembers
 356                         |"""
 357     if len(p) > 1:
 358       p[2].AddChildren(p[1])
 359       p[0] = ListFromConcat(p[2], p[3])
 360
 361   # [.1] Error recovery for ExceptionMembers - Not specified
 362   def p_ExceptionMembersError(self, p):
 363     """ExceptionMembers : error"""
 364     p[0] = self.BuildError(p, 'ExceptionMembers')
 365
 366   # [18]
 367   def p_Inheritance(self, p):
 368     """Inheritance : ':' identifier
 369                    |"""
 370     if len(p) > 1:
 371       p[0] = self.BuildNamed('Inherit', p, 2)
 372
 373   # [19]
 374   def p_Enum(self, p):
 375     """Enum : ENUM identifier '{' EnumValueList '}' ';'"""
 376     p[0] = self.BuildNamed('Enum', p, 2, p[4])
 377
 378   # [19.1] Error recovery for Enums
 379   def p_EnumError(self, p):
 380     """Enum : ENUM error ';'"""
 381     p[0] = self.BuildError(p, 'Enum')
 382
 383   # [20]
 384   def p_EnumValueList(self, p):
 385     """EnumValueList : ExtendedAttributeList string EnumValueListComma"""
 386     enum = self.BuildNamed('EnumItem', p, 2, p[1])
 387     p[0] = ListFromConcat(enum, p[3])
 388
 389   # [21]
 390   def p_EnumValueListComma(self, p):
 391     """EnumValueListComma : ',' EnumValueListString
 392                           |"""
 393     if len(p) > 1:
 394       p[0] = p[2]
 395
 396   # [22]
 397   def p_EnumValueListString(self, p):
 398     """EnumValueListString : ExtendedAttributeList string EnumValueListComma
 399                            |"""
 400     if len(p) > 1:
 401       enum = self.BuildNamed('EnumItem', p, 2, p[1])
 402       p[0] = ListFromConcat(enum, p[3])
 403
 404   # [23]
 405   def p_CallbackRest(self, p):
 406     """CallbackRest : identifier '=' ReturnType '(' ArgumentList ')' ';'"""
 407     arguments = self.BuildProduction('Arguments', p, 4, p[5])
 408     p[0] = self.BuildNamed('Callback', p, 1, ListFromConcat(p[3], arguments))
 409
 410   # [24]
 411   def p_Typedef(self, p):
 412     """Typedef : TYPEDEF ExtendedAttributeListNoComments Type identifier ';'"""
 413     p[0] = self.BuildNamed('Typedef', p, 4, ListFromConcat(p[2], p[3]))
 414
 415   # [24.1] Error recovery for Typedefs
 416   def p_TypedefError(self, p):
 417     """Typedef : TYPEDEF error ';'"""
 418     p[0] = self.BuildError(p, 'Typedef')
 419
 420   # [25]
 421   def p_ImplementsStatement(self, p):
 422     """ImplementsStatement : identifier IMPLEMENTS identifier ';'"""
 423     name = self.BuildAttribute('REFERENCE', p[3])
 424     p[0] = self.BuildNamed('Implements', p, 1, name)
 425
 426   # [26]
 427   def p_Const(self,  p):
 428     """Const : CONST ConstType identifier '=' ConstValue ';'"""
 429     value = self.BuildProduction('Value', p, 5, p[5])
 430     p[0] = self.BuildNamed('Const', p, 3, ListFromConcat(p[2], value))
 431
 432   # [27]
 433   def p_ConstValue(self, p):
 434     """ConstValue : BooleanLiteral
 435                   | FloatLiteral
 436                   | integer
 437                   | null"""
 438     if type(p[1]) == str:
 439       p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'integer'),
 440                             self.BuildAttribute('NAME', p[1]))
 441     else:
 442       p[0] = p[1]
 443
 444   # [27.1] Add definition for NULL
 445   def p_null(self, p):
 446     """null : NULL"""
 447     p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'NULL'),
 448                           self.BuildAttribute('NAME', 'NULL'))
 449
 450   # [28]
 451   def p_BooleanLiteral(self, p):
 452     """BooleanLiteral : TRUE
 453                       | FALSE"""
 454     value = self.BuildAttribute('VALUE', Boolean(p[1] == 'true'))
 455     p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'boolean'), value)
 456
 457   # [29]
 458   def p_FloatLiteral(self, p):
 459     """FloatLiteral : float
 460                     | '-' INFINITY
 461                     | INFINITY
 462                     | NAN """
 463     if len(p) > 2:
 464       val = '-Infinity'
 465     else:
 466       val = p[1]
 467     p[0] = ListFromConcat(self.BuildAttribute('TYPE', 'float'),
 468                           self.BuildAttribute('VALUE', val))
 469
 470   # [30-34] NOT IMPLEMENTED (Serializer)
 471
 472   # [35]
 473   def p_Stringifier(self, p):
 474     """Stringifier : STRINGIFIER StringifierRest"""
 475     p[0] = self.BuildProduction('Stringifier', p, 1, p[2])
 476
 477   # [36]
 478   def p_StringifierRest(self, p):
 479     """StringifierRest : AttributeRest
 480                        | ReturnType OperationRest
 481                        | ';'"""
 482     if len(p) == 3:
 483       p[2].AddChildren(p[1])
 484       p[0] = p[2]
 485     elif p[1] != ';':
 486       p[0] = p[1]
 487
 488   # [37]
 489   def p_StaticMember(self, p):
 490     """StaticMember : STATIC StaticMemberRest"""
 491     p[2].AddChildren(self.BuildTrue('STATIC'))
 492     p[0] = p[2]
 493
 494   # [38]
 495   def p_StaticMemberRest(self, p):
 496     """StaticMemberRest : ReadOnly AttributeRest
 497                         | ReturnType OperationRest"""
 498     if len(p) == 2:
 499       p[0] = p[1]
 500     else:
 501       p[2].AddChildren(p[1])
 502       p[0] = p[2]
 503
 504   # [39]
 505   def p_ReadonlyMember(self, p):
 506     """ReadonlyMember : READONLY ReadonlyMemberRest"""
 507     p[2].AddChildren(self.BuildTrue('READONLY'))
 508     p[0] = p[2]
 509
 510   # [40]
 511   def p_ReadonlyMemberRest(self, p):
 512     """ReadonlyMemberRest : AttributeRest
 513                           | MaplikeRest
 514                           | SetlikeRest"""
 515     p[0] = p[1]
 516
 517   # [41]
 518   def p_ReadWriteAttribute(self, p):
 519     """ReadWriteAttribute : INHERIT ReadOnly AttributeRest
 520                           | AttributeRest"""
 521     if len(p) > 2:
 522       inherit = self.BuildTrue('INHERIT')
 523       p[3].AddChildren(ListFromConcat(inherit, p[2]))
 524       p[0] = p[3]
 525     else:
 526       p[0] = p[1]
 527
 528   # [42]
 529   def p_AttributeRest(self, p):
 530     """AttributeRest : ATTRIBUTE Type AttributeName ';'"""
 531     p[0] = self.BuildNamed('Attribute', p, 3, p[2])
 532
 533   # [43]
 534   def p_AttributeName(self, p):
 535     """AttributeName : AttributeNameKeyword
 536                      | identifier"""
 537     p[0] = p[1]
 538
 539   # [44]
 540   def p_AttributeNameKeyword(self, p):
 541     """AttributeNameKeyword : REQUIRED"""
 542     p[0] = p[1]
 543
 544   # [45] Unreferenced in the specification
 545
 546   # [46]
 547   def p_ReadOnly(self, p):
 548     """ReadOnly : READONLY
 549                 |"""
 550     if len(p) > 1:
 551       p[0] = self.BuildTrue('READONLY')
 552
 553   # [47]
 554   def p_Operation(self, p):
 555     """Operation : ReturnType OperationRest
 556                  | SpecialOperation"""
 557     if len(p) == 3:
 558       p[2].AddChildren(p[1])
 559       p[0] = p[2]
 560     else:
 561       p[0] = p[1]
 562
 563   # [48]
 564   def p_SpecialOperation(self, p):
 565     """SpecialOperation : Special Specials ReturnType OperationRest"""
 566     p[4].AddChildren(ListFromConcat(p[1], p[2], p[3]))
 567     p[0] = p[4]
 568
 569   # [49]
 570   def p_Specials(self, p):
 571     """Specials : Special Specials
 572                 | """
 573     if len(p) > 1:
 574       p[0] = ListFromConcat(p[1], p[2])
 575
 576   # [50]
 577   def p_Special(self, p):
 578     """Special : GETTER
 579                | SETTER
 580                | CREATOR
 581                | DELETER
 582                | LEGACYCALLER"""
 583     p[0] = self.BuildTrue(p[1].upper())
 584
 585   # [51]
 586   def p_OperationRest(self, p):
 587     """OperationRest : OptionalIdentifier '(' ArgumentList ')' ';'"""
 588     arguments = self.BuildProduction('Arguments', p, 2, p[3])
 589     p[0] = self.BuildNamed('Operation', p, 1, arguments)
 590
 591   # [52]
 592   def p_OptionalIdentifier(self, p):
 593     """OptionalIdentifier : identifier
 594                           |"""
 595     if len(p) > 1:
 596       p[0] = p[1]
 597     else:
 598       p[0] = '_unnamed_'
 599
 600   # [53]
 601   def p_ArgumentList(self, p):
 602     """ArgumentList : Argument Arguments
 603                     |"""
 604     if len(p) > 1:
 605       p[0] = ListFromConcat(p[1], p[2])
 606
 607   # [53.1] ArgumentList error recovery
 608   def p_ArgumentListError(self, p):
 609     """ArgumentList : error """
 610     p[0] = self.BuildError(p, 'ArgumentList')
 611
 612   # [54]
 613   def p_Arguments(self, p):
 614     """Arguments : ',' Argument Arguments
 615                  |"""
 616     if len(p) > 1:
 617       p[0] = ListFromConcat(p[2], p[3])
 618
 619   # [54.1] Arguments error recovery
 620   def p_ArgumentsError(self, p):
 621     """Arguments : ',' error"""
 622     p[0] = self.BuildError(p, 'Arguments')
 623
 624   # [55]
 625   def p_Argument(self, p):
 626     """Argument : ExtendedAttributeList OptionalOrRequiredArgument"""
 627     p[2].AddChildren(p[1])
 628     p[0] = p[2]
 629
 630   # [56]
 631   def p_OptionalOrRequiredArgument(self, p):
 632     """OptionalOrRequiredArgument : OPTIONAL Type ArgumentName Default
 633                                   | Type Ellipsis ArgumentName"""
 634     if len(p) > 4:
 635       arg = self.BuildNamed('Argument', p, 3, ListFromConcat(p[2], p[4]))
 636       arg.AddChildren(self.BuildTrue('OPTIONAL'))
 637     else:
 638       arg = self.BuildNamed('Argument', p, 3, ListFromConcat(p[1], p[2]))
 639     p[0] = arg
 640
 641   # [57]
 642   def p_ArgumentName(self, p):
 643     """ArgumentName : ArgumentNameKeyword
 644                     | identifier"""
 645     p[0] = p[1]
 646
 647   # [58]
 648   def p_Ellipsis(self, p):
 649     """Ellipsis : ELLIPSIS
 650                 |"""
 651     if len(p) > 1:
 652       p[0] = self.BuildNamed('Argument', p, 1)
 653       p[0].AddChildren(self.BuildTrue('ELLIPSIS'))
 654
 655   # [] Unspecified
 656   def p_ExceptionMember(self, p):
 657     """ExceptionMember : Const
 658                        | ExceptionField"""
 659     p[0] = p[1]
 660
 661   # [] Unspecified
 662   def p_ExceptionField(self, p):
 663     """ExceptionField : Type identifier ';'"""
 664     p[0] = self.BuildNamed('ExceptionField', p, 2, p[1])
 665
 666   # [] Error recovery for ExceptionMembers - Unspecified
 667   def p_ExceptionFieldError(self, p):
 668     """ExceptionField : error"""
 669     p[0] = self.BuildError(p, 'ExceptionField')
 670
 671   # [59]
 672   def p_Iterable(self, p):
 673     """Iterable : ITERABLE '<' Type OptionalType '>' ';'
 674                 | LEGACYITERABLE '<' Type '>' ';'"""
 675     if len(p) > 6:
 676       childlist = ListFromConcat(p[3], p[4])
 677       p[0] = self.BuildProduction('Iterable', p, 2, childlist)
 678     else:
 679       p[0] = self.BuildProduction('LegacyIterable', p, 2, p[3])
 680
 681   # [60]
 682   def p_OptionalType(self, p):
 683     """OptionalType : ',' Type
 684                     |"""
 685     if len(p) > 1:
 686       p[0] = p[2]
 687
 688   # [61]
 689   def p_ReadWriteMaplike(self, p):
 690     """ReadWriteMaplike : MaplikeRest"""
 691     p[0] = p[1]
 692
 693   # [62]
 694   def p_ReadWriteSetlike(self, p):
 695     """ReadWriteSetlike : SetlikeRest"""
 696     p[0] = p[1]
 697
 698   # [63]
 699   def p_MaplikeRest(self, p):
 700     """MaplikeRest : MAPLIKE '<' Type ',' Type '>' ';'"""
 701     childlist = ListFromConcat(p[3], p[5])
 702     p[0] = self.BuildProduction('Maplike', p, 2, childlist)
 703
 704   # [64]
 705   def p_SetlikeRest(self, p):
 706     """SetlikeRest : SETLIKE '<' Type '>' ';'"""
 707     p[0] = self.BuildProduction('Setlike', p, 2, p[3])
 708
 709   # [65] No comment version for mid statement attributes.
 710   def p_ExtendedAttributeListNoComments(self, p):
 711     """ExtendedAttributeListNoComments : '[' ExtendedAttribute ExtendedAttributes ']'
 712                                        | """
 713     if len(p) > 2:
 714       items = ListFromConcat(p[2], p[3])
 715       p[0] = self.BuildProduction('ExtAttributes', p, 1, items)
 716
 717   # [65.1] Add optional comment field for start of statements.
 718   def p_ExtendedAttributeList(self, p):
 719     """ExtendedAttributeList : Comments '[' ExtendedAttribute ExtendedAttributes ']'
 720                              | Comments """
 721     if len(p) > 2:
 722       items = ListFromConcat(p[3], p[4])
 723       attribs = self.BuildProduction('ExtAttributes', p, 2, items)
 724       p[0] = ListFromConcat(p[1], attribs)
 725     else:
 726       p[0] = p[1]
 727
 728   # [66]
 729   def p_ExtendedAttributes(self, p):
 730     """ExtendedAttributes : ',' ExtendedAttribute ExtendedAttributes
 731                           |"""
 732     if len(p) > 1:
 733       p[0] = ListFromConcat(p[2], p[3])
 734
 735   # We only support:
 736   #    [ identifier ]
 737   #    [ identifier ( ArgumentList ) ]
 738   #    [ identifier = identifier ]
 739   #    [ identifier = ( IdentifierList ) ]
 740   #    [ identifier = identifier ( ArgumentList ) ]
 741   # [66] map directly to [91-93, 95]
 742   # [67-69, 71] are unsupported
 743   def p_ExtendedAttribute(self, p):
 744     """ExtendedAttribute : ExtendedAttributeNoArgs
 745                          | ExtendedAttributeArgList
 746                          | ExtendedAttributeIdent
 747                          | ExtendedAttributeIdentList
 748                          | ExtendedAttributeNamedArgList"""
 749     p[0] = p[1]
 750
 751   # [71]
 752   def p_ArgumentNameKeyword(self, p):
 753     """ArgumentNameKeyword : ATTRIBUTE
 754                            | CALLBACK
 755                            | CONST
 756                            | CREATOR
 757                            | DELETER
 758                            | DICTIONARY
 759                            | ENUM
 760                            | EXCEPTION
 761                            | GETTER
 762                            | IMPLEMENTS
 763                            | INHERIT
 764                            | LEGACYCALLER
 765                            | PARTIAL
 766                            | SERIALIZER
 767                            | SETTER
 768                            | STATIC
 769                            | STRINGIFIER
 770                            | TYPEDEF
 771                            | UNRESTRICTED"""
 772     p[0] = p[1]
 773
 774   # [72] NOT IMPLEMENTED (OtherOrComma)
 775
 776   # [73]
 777   def p_Type(self, p):
 778     """Type : SingleType
 779             | UnionType TypeSuffix"""
 780     if len(p) == 2:
 781       p[0] = self.BuildProduction('Type', p, 1, p[1])
 782     else:
 783       p[0] = self.BuildProduction('Type', p, 1, ListFromConcat(p[1], p[2]))
 784
 785   # [74]
 786   def p_SingleType(self, p):
 787     """SingleType : NonAnyType
 788                   | ANY TypeSuffixStartingWithArray"""
 789     if len(p) == 2:
 790       p[0] = p[1]
 791     else:
 792       p[0] = ListFromConcat(self.BuildProduction('Any', p, 1), p[2])
 793
 794   # [75]
 795   def p_UnionType(self, p):
 796     """UnionType : '(' UnionMemberType OR UnionMemberType UnionMemberTypes ')'"""
 797
 798   # [76]
 799   def p_UnionMemberType(self, p):
 800     """UnionMemberType : NonAnyType
 801                        | UnionType TypeSuffix
 802                        | ANY '[' ']' TypeSuffix"""
 803   # [77]
 804   def p_UnionMemberTypes(self, p):
 805     """UnionMemberTypes : OR UnionMemberType UnionMemberTypes
 806                         |"""
 807
 808   # [78] Moved BYTESTRING, DOMSTRING, OBJECT, DATE, REGEXP to PrimitiveType
 809   # Moving all built-in types into PrimitiveType makes it easier to
 810   # differentiate between them and 'identifier', since p[1] would be a string in
 811   # both cases.
 812   def p_NonAnyType(self, p):
 813     """NonAnyType : PrimitiveType TypeSuffix
 814                   | PromiseType Null
 815                   | identifier TypeSuffix
 816                   | SEQUENCE '<' Type '>' Null"""
 817     if len(p) == 3:
 818       if type(p[1]) == str:
 819         typeref = self.BuildNamed('Typeref', p, 1)
 820       else:
 821         typeref = p[1]
 822       p[0] = ListFromConcat(typeref, p[2])
 823
 824     if len(p) == 6:
 825       p[0] = self.BuildProduction('Sequence', p, 1, ListFromConcat(p[3], p[5]))
 826
 827   # [79] NOT IMPLEMENTED (BufferRelatedType)
 828
 829   # [80]
 830   def p_ConstType(self,  p):
 831     """ConstType : PrimitiveType Null
 832                  | identifier Null"""
 833     if type(p[1]) == str:
 834       p[0] = self.BuildNamed('Typeref', p, 1, p[2])
 835     else:
 836       p[1].AddChildren(p[2])
 837       p[0] = p[1]
 838
 839
 840   # [81] Added BYTESTRING, DOMSTRING, OBJECT, DATE, REGEXP
 841   def p_PrimitiveType(self, p):
 842     """PrimitiveType : UnsignedIntegerType
 843                      | UnrestrictedFloatType
 844                      | BOOLEAN
 845                      | BYTE
 846                      | OCTET
 847                      | BYTESTRING
 848                      | DOMSTRING
 849                      | OBJECT
 850                      | DATE
 851                      | REGEXP"""
 852     if type(p[1]) == str:
 853       p[0] = self.BuildNamed('PrimitiveType', p, 1)
 854     else:
 855       p[0] = p[1]
 856
 857
 858   # [82]
 859   def p_UnrestrictedFloatType(self, p):
 860     """UnrestrictedFloatType : UNRESTRICTED FloatType
 861                              | FloatType"""
 862     if len(p) == 2:
 863       typeref = self.BuildNamed('PrimitiveType', p, 1)
 864     else:
 865       typeref = self.BuildNamed('PrimitiveType', p, 2)
 866       typeref.AddChildren(self.BuildTrue('UNRESTRICTED'))
 867     p[0] = typeref
 868
 869
 870   # [83]
 871   def p_FloatType(self, p):
 872     """FloatType : FLOAT
 873                  | DOUBLE"""
 874     p[0] = p[1]
 875
 876   # [84]
 877   def p_UnsignedIntegerType(self, p):
 878     """UnsignedIntegerType : UNSIGNED IntegerType
 879                            | IntegerType"""
 880     if len(p) == 2:
 881       p[0] = p[1]
 882     else:
 883       p[0] = 'unsigned ' + p[2]
 884
 885   # [85]
 886   def p_IntegerType(self, p):
 887     """IntegerType : SHORT
 888                    | LONG OptionalLong"""
 889     if len(p) == 2:
 890       p[0] = p[1]
 891     else:
 892       p[0] = p[1] + p[2]
 893
 894   # [86]
 895   def p_OptionalLong(self, p):
 896     """OptionalLong : LONG
 897                     | """
 898     if len(p) > 1:
 899       p[0] = ' ' + p[1]
 900     else:
 901       p[0] = ''
 902
 903   # [87] Add unqualified Promise
 904   def p_PromiseType(self, p):
 905     """PromiseType : PROMISE '<' ReturnType '>'
 906                    | PROMISE"""
 907     if len(p) == 2:
 908       # Promise without resolution type is not specified in the Web IDL spec.
 909       # As it is used in some specs and in the blink implementation,
 910       # we allow that here.
 911       resolution_type = self.BuildProduction('Type', p, 1,
 912                                              self.BuildProduction('Any', p, 1))
 913       p[0] = self.BuildNamed('Promise', p, 1, resolution_type)
 914     else:
 915       p[0] = self.BuildNamed('Promise', p, 1, p[3])
 916
 917   # [88] Add support for sized array
 918   def p_TypeSuffix(self, p):
 919     """TypeSuffix : '[' integer ']' TypeSuffix
 920                   | '[' ']' TypeSuffix
 921                   | '?' TypeSuffixStartingWithArray
 922                   | """
 923     if len(p) == 5:
 924       p[0] = self.BuildNamed('Array', p, 2, p[4])
 925
 926     if len(p) == 4:
 927       p[0] = self.BuildProduction('Array', p, 1, p[3])
 928
 929     if len(p) == 3:
 930       p[0] = ListFromConcat(self.BuildTrue('NULLABLE'), p[2])
 931
 932
 933   # [89]
 934   def p_TypeSuffixStartingWithArray(self, p):
 935     """TypeSuffixStartingWithArray : '[' ']' TypeSuffix
 936                                    | """
 937     if len(p) > 1:
 938       p[0] = self.BuildProduction('Array', p, 0, p[3])
 939
 940   # [90]
 941   def p_Null(self, p):
 942     """Null : '?'
 943             |"""
 944     if len(p) > 1:
 945       p[0] = self.BuildTrue('NULLABLE')
 946
 947   # [91]
 948   def p_ReturnType(self, p):
 949     """ReturnType : Type
 950                   | VOID"""
 951     if p[1] == 'void':
 952       p[0] = self.BuildProduction('Type', p, 1)
 953       p[0].AddChildren(self.BuildNamed('PrimitiveType', p, 1))
 954     else:
 955       p[0] = p[1]
 956
 957   # [92]
 958   def p_IdentifierList(self, p):
 959     """IdentifierList : identifier Identifiers"""
 960     p[0] = ListFromConcat(p[1], p[2])
 961
 962   # [93]
 963   def p_Identifiers(self, p):
 964     """Identifiers : ',' identifier Identifiers
 965                    |"""
 966     if len(p) > 1:
 967       p[0] = ListFromConcat(p[2], p[3])
 968
 969   # [94]
 970   def p_ExtendedAttributeNoArgs(self, p):
 971     """ExtendedAttributeNoArgs : identifier"""
 972     p[0] = self.BuildNamed('ExtAttribute', p, 1)
 973
 974   # [95]
 975   def p_ExtendedAttributeArgList(self, p):
 976     """ExtendedAttributeArgList : identifier '(' ArgumentList ')'"""
 977     arguments = self.BuildProduction('Arguments', p, 2, p[3])
 978     p[0] = self.BuildNamed('ExtAttribute', p, 1, arguments)
 979
 980   # [96]
 981   def p_ExtendedAttributeIdent(self, p):
 982     """ExtendedAttributeIdent : identifier '=' identifier"""
 983     value = self.BuildAttribute('VALUE', p[3])
 984     p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
 985
 986   # [97]
 987   def p_ExtendedAttributeIdentList(self, p):
 988     """ExtendedAttributeIdentList : identifier '=' '(' IdentifierList ')'"""
 989     value = self.BuildAttribute('VALUE', p[4])
 990     p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
 991
 992   # [98]
 993   def p_ExtendedAttributeNamedArgList(self, p):
 994     """ExtendedAttributeNamedArgList : identifier '=' identifier '(' ArgumentList ')'"""
 995     args = self.BuildProduction('Arguments', p, 4, p[5])
 996     value = self.BuildNamed('Call', p, 3, args)
 997     p[0] = self.BuildNamed('ExtAttribute', p, 1, value)
 998
 999 #
1000 # Parser Errors
1001 #
1002 # p_error is called whenever the parser can not find a pattern match for
1003 # a set of items from the current state.  The p_error function defined here
1004 # is triggered logging an error, and parsing recovery happens as the
1005 # p_<type>_error functions defined above are called.  This allows the parser
1006 # to continue so as to capture more than one error per file.
1007 #
1008   def p_error(self, t):
1009     if t:
1010       lineno = t.lineno
1011       pos = t.lexpos
1012       prev = self.yaccobj.symstack[-1]
1013       if type(prev) == lex.LexToken:
1014         msg = "Unexpected %s after %s." % (
1015             TokenTypeName(t), TokenTypeName(prev))
1016       else:
1017         msg = "Unexpected %s." % (t.value)
1018     else:
1019       last = self.LastToken()
1020       lineno = last.lineno
1021       pos = last.lexpos
1022       msg = "Unexpected end of file after %s." % TokenTypeName(last)
1023       self.yaccobj.restart()
1024
1025     # Attempt to remap the error to a friendlier form
1026     if msg in ERROR_REMAP:
1027       msg = ERROR_REMAP[msg]
1028
1029     self._last_error_msg = msg
1030     self._last_error_lineno = lineno
1031     self._last_error_pos = pos
1032
1033   def Warn(self, node, msg):
1034     sys.stdout.write(node.GetLogLine(msg))
1035     self.parse_warnings += 1
1036
1037   def LastToken(self):
1038     return self.lexer.last
1039
1040   def __init__(self, lexer, verbose=False, debug=False, mute_error=False):
1041     self.lexer = lexer
1042     self.tokens = lexer.KnownTokens()
1043     self.yaccobj = yacc.yacc(module=self, tabmodule=None, debug=debug,
1044                              optimize=0, write_tables=0)
1045     self.parse_debug = debug
1046     self.verbose = verbose
1047     self.mute_error = mute_error
1048     self._parse_errors = 0
1049     self._parse_warnings = 0
1050     self._last_error_msg = None
1051     self._last_error_lineno = 0
1052     self._last_error_pos = 0
1053
1054
1055 #
1056 # BuildProduction
1057 #
1058 # Production is the set of items sent to a grammar rule resulting in a new
1059 # item being returned.
1060 #
1061 # p - Is the Yacc production object containing the stack of items
1062 # index - Index into the production of the name for the item being produced.
1063 # cls - The type of item being producted
1064 # childlist - The children of the new item
1065   def BuildProduction(self, cls, p, index, childlist=None):
1066     try:
1067       if not childlist:
1068         childlist = []
1069
1070       filename = self.lexer.Lexer().filename
1071       lineno = p.lineno(index)
1072       pos = p.lexpos(index)
1073       out = IDLNode(cls, filename, lineno, pos, childlist)
1074       return out
1075     except:
1076       print 'Exception while parsing:'
1077       for num, item in enumerate(p):
1078         print '  [%d] %s' % (num, ExpandProduction(item))
1079       if self.LastToken():
1080         print 'Last token: %s' % str(self.LastToken())
1081       raise
1082
1083   def BuildNamed(self, cls, p, index, childlist=None):
1084     childlist = ListFromConcat(childlist)
1085     childlist.append(self.BuildAttribute('NAME', p[index]))
1086     return self.BuildProduction(cls, p, index, childlist)
1087
1088   def BuildComment(self, cls, p, index):
1089     name = p[index]
1090
1091     # Remove comment markers
1092     lines = []
1093     if name[:2] == '//':
1094       # For C++ style, remove any leading whitespace and the '//' marker from
1095       # each line.
1096       form = 'cc'
1097       for line in name.split('\n'):
1098         start = line.find('//')
1099         lines.append(line[start+2:])
1100     else:
1101       # For C style, remove ending '*/''
1102       form = 'c'
1103       for line in name[:-2].split('\n'):
1104         # Remove characters until start marker for this line '*' if found
1105         # otherwise it should be blank.
1106         offs = line.find('*')
1107         if offs >= 0:
1108           line = line[offs + 1:].rstrip()
1109         else:
1110           line = ''
1111         lines.append(line)
1112     name = '\n'.join(lines)
1113     childlist = [self.BuildAttribute('NAME', name),
1114                  self.BuildAttribute('FORM', form)]
1115     return self.BuildProduction(cls, p, index, childlist)
1116
1117 #
1118 # BuildError
1119 #
1120 # Build and Errror node as part of the recovery process.
1121 #
1122 #
1123   def BuildError(self, p, prod):
1124     self._parse_errors += 1
1125     name = self.BuildAttribute('NAME', self._last_error_msg)
1126     line = self.BuildAttribute('LINE', self._last_error_lineno)
1127     pos = self.BuildAttribute('POS', self._last_error_pos)
1128     prod = self.BuildAttribute('PROD', prod)
1129
1130     node = self.BuildProduction('Error', p, 1,
1131                                 ListFromConcat(name, line, pos, prod))
1132     if not self.mute_error:
1133       node.Error(self._last_error_msg)
1134
1135     return node
1136
1137 #
1138 # BuildAttribute
1139 #
1140 # An ExtendedAttribute is a special production that results in a property
1141 # which is applied to the adjacent item.  Attributes have no children and
1142 # instead represent key/value pairs.
1143 #
1144   def BuildAttribute(self, key, val):
1145     return IDLAttribute(key, val)
1146
1147   def BuildFalse(self, key):
1148     return IDLAttribute(key, Boolean(False))
1149
1150   def BuildTrue(self, key):
1151     return IDLAttribute(key, Boolean(True))
1152
1153   def GetErrors(self):
1154     # Access lexer errors, despite being private
1155     # pylint: disable=W0212
1156     return self._parse_errors + self.lexer._lex_errors
1157
1158 #
1159 # ParseData
1160 #
1161 # Attempts to parse the current data loaded in the lexer.
1162 #
1163   def ParseText(self, filename, data):
1164     self._parse_errors = 0
1165     self._parse_warnings = 0
1166     self._last_error_msg = None
1167     self._last_error_lineno = 0
1168     self._last_error_pos = 0
1169
1170     try:
1171       self.lexer.Tokenize(data, filename)
1172       nodes = self.yaccobj.parse(lexer=self.lexer) or []
1173       name = self.BuildAttribute('NAME', filename)
1174       return IDLNode('File', filename, 0, 0, nodes + [name])
1175
1176     except lex.LexError as lexError:
1177       sys.stderr.write('Error in token: %s\n' % str(lexError))
1178     return None
1179
1180
1181
1182 def ParseFile(parser, filename):
1183   """Parse a file and return a File type of node."""
1184   with open(filename) as fileobject:
1185     try:
1186       out = parser.ParseText(filename, fileobject.read())
1187       out.SetProperty('DATETIME', time.ctime(os.path.getmtime(filename)))
1188       out.SetProperty('ERRORS', parser.GetErrors())
1189       return out
1190
1191     except Exception as e:
1192       last = parser.LastToken()
1193       sys.stderr.write('%s(%d) : Internal parsing error\n\t%s.\n' % (
1194                        filename, last.lineno, str(e)))
1195
1196
1197 def main(argv):
1198   nodes = []
1199   parser = IDLParser(IDLLexer())
1200   errors = 0
1201   for filename in argv:
1202     filenode = ParseFile(parser, filename)
1203     if (filenode):
1204       errors += filenode.GetProperty('ERRORS')
1205       nodes.append(filenode)
1206
1207   ast = IDLNode('AST', '__AST__', 0, 0, nodes)
1208
1209   print '\n'.join(ast.Tree(accept_props=['PROD']))
1210   if errors:
1211     print '\nFound %d errors.\n' % errors
1212
1213   return errors
1214
1215
1216 if __name__ == '__main__':
1217   sys.exit(main(sys.argv[1:]))