Lib/sgmllib.py

   1 """A parser for SGML, using the derived class as a static DTD."""
   2
   3 # XXX This only supports those SGML features used by HTML.
   4
   5 # XXX There should be a way to distinguish between PCDATA (parsed
   6 # character data -- the normal case), RCDATA (replaceable character
   7 # data -- only char and entity references and end tags are special)
   8 # and CDATA (character data -- only end tags are special).  RCDATA is
   9 # not supported at all.
  10
  11
  12 from warnings import warnpy3k
  13 warnpy3k("the sgmllib module has been removed in Python 3.0",
  14          stacklevel=2)
  15 del warnpy3k
  16
  17 import markupbase
  18 import re
  19
  20 __all__ = ["SGMLParser", "SGMLParseError"]
  21
  22 # Regular expressions used for parsing
  23
  24 interesting = re.compile('[&<]')
  25 incomplete = re.compile('&([a-zA-Z][a-zA-Z0-9]*|#[0-9]*)?|'
  26                            '<([a-zA-Z][^<>]*|'
  27                               '/([a-zA-Z][^<>]*)?|'
  28                               '![^<>]*)?')
  29
  30 entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]')
  31 charref = re.compile('&#([0-9]+)[^0-9]')
  32
  33 starttagopen = re.compile('<[>a-zA-Z]')
  34 shorttagopen = re.compile('<[a-zA-Z][-.a-zA-Z0-9]*/')
  35 shorttag = re.compile('<([a-zA-Z][-.a-zA-Z0-9]*)/([^/]*)/')
  36 piclose = re.compile('>')
  37 endbracket = re.compile('[<>]')
  38 tagfind = re.compile('[a-zA-Z][-_.a-zA-Z0-9]*')
  39 attrfind = re.compile(
  40     r'\s*([a-zA-Z_][-:.a-zA-Z_0-9]*)(\s*=\s*'
  41     r'(\'[^\']*\'|"[^"]*"|[][\-a-zA-Z0-9./,:;+*%?!&$\(\)_#=~\'"@]*))?')
  42
  43
  44 class SGMLParseError(RuntimeError):
  45     """Exception raised for all parse errors."""
  46     pass
  47
  48
  49 # SGML parser base class -- find tags and call handler functions.
  50 # Usage: p = SGMLParser(); p.feed(data); ...; p.close().
  51 # The dtd is defined by deriving a class which defines methods
  52 # with special names to handle tags: start_foo and end_foo to handle
  53 # <foo> and </foo>, respectively, or do_foo to handle <foo> by itself.
  54 # (Tags are converted to lower case for this purpose.)  The data
  55 # between tags is passed to the parser by calling self.handle_data()
  56 # with some data as argument (the data may be split up in arbitrary
  57 # chunks).  Entity references are passed by calling
  58 # self.handle_entityref() with the entity reference as argument.
  59
  60 class SGMLParser(markupbase.ParserBase):
  61     # Definition of entities -- derived classes may override
  62     entity_or_charref = re.compile('&(?:'
  63       '([a-zA-Z][-.a-zA-Z0-9]*)|#([0-9]+)'
  64       ')(;?)')
  65
  66     def __init__(self, verbose=0):
  67         """Initialize and reset this instance."""
  68         self.verbose = verbose
  69         self.reset()
  70
  71     def reset(self):
  72         """Reset this instance. Loses all unprocessed data."""
  73         self.__starttag_text = None
  74         self.rawdata = ''
  75         self.stack = []
  76         self.lasttag = '???'
  77         self.nomoretags = 0
  78         self.literal = 0
  79         markupbase.ParserBase.reset(self)
  80
  81     def setnomoretags(self):
  82         """Enter literal mode (CDATA) till EOF.
  83
  84         Intended for derived classes only.
  85         """
  86         self.nomoretags = self.literal = 1
  87
  88     def setliteral(self, *args):
  89         """Enter literal mode (CDATA).
  90
  91         Intended for derived classes only.
  92         """
  93         self.literal = 1
  94
  95     def feed(self, data):
  96         """Feed some data to the parser.
  97
  98         Call this as often as you want, with as little or as much text
  99         as you want (may include '\n').  (This just saves the text,
 100         all the processing is done by goahead().)
 101         """
 102
 103         self.rawdata = self.rawdata + data
 104         self.goahead(0)
 105
 106     def close(self):
 107         """Handle the remaining data."""
 108         self.goahead(1)
 109
 110     def error(self, message):
 111         raise SGMLParseError(message)
 112
 113     # Internal -- handle data as far as reasonable.  May leave state
 114     # and data to be processed by a subsequent call.  If 'end' is
 115     # true, force handling all data as if followed by EOF marker.
 116     def goahead(self, end):
 117         rawdata = self.rawdata
 118         i = 0
 119         n = len(rawdata)
 120         while i < n:
 121             if self.nomoretags:
 122                 self.handle_data(rawdata[i:n])
 123                 i = n
 124                 break
 125             match = interesting.search(rawdata, i)
 126             if match: j = match.start()
 127             else: j = n
 128             if i < j:
 129                 self.handle_data(rawdata[i:j])
 130             i = j
 131             if i == n: break
 132             if rawdata[i] == '<':
 133                 if starttagopen.match(rawdata, i):
 134                     if self.literal:
 135                         self.handle_data(rawdata[i])
 136                         i = i+1
 137                         continue
 138                     k = self.parse_starttag(i)
 139                     if k < 0: break
 140                     i = k
 141                     continue
 142                 if rawdata.startswith("</", i):
 143                     k = self.parse_endtag(i)
 144                     if k < 0: break
 145                     i = k
 146                     self.literal = 0
 147                     continue
 148                 if self.literal:
 149                     if n > (i + 1):
 150                         self.handle_data("<")
 151                         i = i+1
 152                     else:
 153                         # incomplete
 154                         break
 155                     continue
 156                 if rawdata.startswith("<!--", i):
 157                         # Strictly speaking, a comment is --.*--
 158                         # within a declaration tag <!...>.
 159                         # This should be removed,
 160                         # and comments handled only in parse_declaration.
 161                     k = self.parse_comment(i)
 162                     if k < 0: break
 163                     i = k
 164                     continue
 165                 if rawdata.startswith("<?", i):
 166                     k = self.parse_pi(i)
 167                     if k < 0: break
 168                     i = i+k
 169                     continue
 170                 if rawdata.startswith("<!", i):
 171                     # This is some sort of declaration; in "HTML as
 172                     # deployed," this should only be the document type
 173                     # declaration ("<!DOCTYPE html...>").
 174                     k = self.parse_declaration(i)
 175                     if k < 0: break
 176                     i = k
 177                     continue
 178             elif rawdata[i] == '&':
 179                 if self.literal:
 180                     self.handle_data(rawdata[i])
 181                     i = i+1
 182                     continue
 183                 match = charref.match(rawdata, i)
 184                 if match:
 185                     name = match.group(1)
 186                     self.handle_charref(name)
 187                     i = match.end(0)
 188                     if rawdata[i-1] != ';': i = i-1
 189                     continue
 190                 match = entityref.match(rawdata, i)
 191                 if match:
 192                     name = match.group(1)
 193                     self.handle_entityref(name)
 194                     i = match.end(0)
 195                     if rawdata[i-1] != ';': i = i-1
 196                     continue
 197             else:
 198                 self.error('neither < nor & ??')
 199             # We get here only if incomplete matches but
 200             # nothing else
 201             match = incomplete.match(rawdata, i)
 202             if not match:
 203                 self.handle_data(rawdata[i])
 204                 i = i+1
 205                 continue
 206             j = match.end(0)
 207             if j == n:
 208                 break # Really incomplete
 209             self.handle_data(rawdata[i:j])
 210             i = j
 211         # end while
 212         if end and i < n:
 213             self.handle_data(rawdata[i:n])
 214             i = n
 215         self.rawdata = rawdata[i:]
 216         # XXX if end: check for empty stack
 217
 218     # Extensions for the DOCTYPE scanner:
 219     _decl_otherchars = '='
 220
 221     # Internal -- parse processing instr, return length or -1 if not terminated
 222     def parse_pi(self, i):
 223         rawdata = self.rawdata
 224         if rawdata[i:i+2] != '<?':
 225             self.error('unexpected call to parse_pi()')
 226         match = piclose.search(rawdata, i+2)
 227         if not match:
 228             return -1
 229         j = match.start(0)
 230         self.handle_pi(rawdata[i+2: j])
 231         j = match.end(0)
 232         return j-i
 233
 234     def get_starttag_text(self):
 235         return self.__starttag_text
 236
 237     # Internal -- handle starttag, return length or -1 if not terminated
 238     def parse_starttag(self, i):
 239         self.__starttag_text = None
 240         start_pos = i
 241         rawdata = self.rawdata
 242         if shorttagopen.match(rawdata, i):
 243             # SGML shorthand: <tag/data/ == <tag>data</tag>
 244             # XXX Can data contain &... (entity or char refs)?
 245             # XXX Can data contain < or > (tag characters)?
 246             # XXX Can there be whitespace before the first /?
 247             match = shorttag.match(rawdata, i)
 248             if not match:
 249                 return -1
 250             tag, data = match.group(1, 2)
 251             self.__starttag_text = '<%s/' % tag
 252             tag = tag.lower()
 253             k = match.end(0)
 254             self.finish_shorttag(tag, data)
 255             self.__starttag_text = rawdata[start_pos:match.end(1) + 1]
 256             return k
 257         # XXX The following should skip matching quotes (' or ")
 258         # As a shortcut way to exit, this isn't so bad, but shouldn't
 259         # be used to locate the actual end of the start tag since the
 260         # < or > characters may be embedded in an attribute value.
 261         match = endbracket.search(rawdata, i+1)
 262         if not match:
 263             return -1
 264         j = match.start(0)
 265         # Now parse the data between i+1 and j into a tag and attrs
 266         attrs = []
 267         if rawdata[i:i+2] == '<>':
 268             # SGML shorthand: <> == <last open tag seen>
 269             k = j
 270             tag = self.lasttag
 271         else:
 272             match = tagfind.match(rawdata, i+1)
 273             if not match:
 274                 self.error('unexpected call to parse_starttag')
 275             k = match.end(0)
 276             tag = rawdata[i+1:k].lower()
 277             self.lasttag = tag
 278         while k < j:
 279             match = attrfind.match(rawdata, k)
 280             if not match: break
 281             attrname, rest, attrvalue = match.group(1, 2, 3)
 282             if not rest:
 283                 attrvalue = attrname
 284             else:
 285                 if (attrvalue[:1] == "'" == attrvalue[-1:] or
 286                     attrvalue[:1] == '"' == attrvalue[-1:]):
 287                     # strip quotes
 288                     attrvalue = attrvalue[1:-1]
 289                 attrvalue = self.entity_or_charref.sub(
 290                     self._convert_ref, attrvalue)
 291             attrs.append((attrname.lower(), attrvalue))
 292             k = match.end(0)
 293         if rawdata[j] == '>':
 294             j = j+1
 295         self.__starttag_text = rawdata[start_pos:j]
 296         self.finish_starttag(tag, attrs)
 297         return j
 298
 299     # Internal -- convert entity or character reference
 300     def _convert_ref(self, match):
 301         if match.group(2):
 302             return self.convert_charref(match.group(2)) or \
 303                 '&#%s%s' % match.groups()[1:]
 304         elif match.group(3):
 305             return self.convert_entityref(match.group(1)) or \
 306                 '&%s;' % match.group(1)
 307         else:
 308             return '&%s' % match.group(1)
 309
 310     # Internal -- parse endtag
 311     def parse_endtag(self, i):
 312         rawdata = self.rawdata
 313         match = endbracket.search(rawdata, i+1)
 314         if not match:
 315             return -1
 316         j = match.start(0)
 317         tag = rawdata[i+2:j].strip().lower()
 318         if rawdata[j] == '>':
 319             j = j+1
 320         self.finish_endtag(tag)
 321         return j
 322
 323     # Internal -- finish parsing of <tag/data/ (same as <tag>data</tag>)
 324     def finish_shorttag(self, tag, data):
 325         self.finish_starttag(tag, [])
 326         self.handle_data(data)
 327         self.finish_endtag(tag)
 328
 329     # Internal -- finish processing of start tag
 330     # Return -1 for unknown tag, 0 for open-only tag, 1 for balanced tag
 331     def finish_starttag(self, tag, attrs):
 332         try:
 333             method = getattr(self, 'start_' + tag)
 334         except AttributeError:
 335             try:
 336                 method = getattr(self, 'do_' + tag)
 337             except AttributeError:
 338                 self.unknown_starttag(tag, attrs)
 339                 return -1
 340             else:
 341                 self.handle_starttag(tag, method, attrs)
 342                 return 0
 343         else:
 344             self.stack.append(tag)
 345             self.handle_starttag(tag, method, attrs)
 346             return 1
 347
 348     # Internal -- finish processing of end tag
 349     def finish_endtag(self, tag):
 350         if not tag:
 351             found = len(self.stack) - 1
 352             if found < 0:
 353                 self.unknown_endtag(tag)
 354                 return
 355         else:
 356             if tag not in self.stack:
 357                 try:
 358                     method = getattr(self, 'end_' + tag)
 359                 except AttributeError:
 360                     self.unknown_endtag(tag)
 361                 else:
 362                     self.report_unbalanced(tag)
 363                 return
 364             found = len(self.stack)
 365             for i in range(found):
 366                 if self.stack[i] == tag: found = i
 367         while len(self.stack) > found:
 368             tag = self.stack[-1]
 369             try:
 370                 method = getattr(self, 'end_' + tag)
 371             except AttributeError:
 372                 method = None
 373             if method:
 374                 self.handle_endtag(tag, method)
 375             else:
 376                 self.unknown_endtag(tag)
 377             del self.stack[-1]
 378
 379     # Overridable -- handle start tag
 380     def handle_starttag(self, tag, method, attrs):
 381         method(attrs)
 382
 383     # Overridable -- handle end tag
 384     def handle_endtag(self, tag, method):
 385         method()
 386
 387     # Example -- report an unbalanced </...> tag.
 388     def report_unbalanced(self, tag):
 389         if self.verbose:
 390             print '*** Unbalanced </' + tag + '>'
 391             print '*** Stack:', self.stack
 392
 393     def convert_charref(self, name):
 394         """Convert character reference, may be overridden."""
 395         try:
 396             n = int(name)
 397         except ValueError:
 398             return
 399         if not 0 <= n <= 127:
 400             return
 401         return self.convert_codepoint(n)
 402
 403     def convert_codepoint(self, codepoint):
 404         return chr(codepoint)
 405
 406     def handle_charref(self, name):
 407         """Handle character reference, no need to override."""
 408         replacement = self.convert_charref(name)
 409         if replacement is None:
 410             self.unknown_charref(name)
 411         else:
 412             self.handle_data(replacement)
 413
 414     # Definition of entities -- derived classes may override
 415     entitydefs = \
 416             {'lt': '<', 'gt': '>', 'amp': '&', 'quot': '"', 'apos': '\''}
 417
 418     def convert_entityref(self, name):
 419         """Convert entity references.
 420
 421         As an alternative to overriding this method; one can tailor the
 422         results by setting up the self.entitydefs mapping appropriately.
 423         """
 424         table = self.entitydefs
 425         if name in table:
 426             return table[name]
 427         else:
 428             return
 429
 430     def handle_entityref(self, name):
 431         """Handle entity references, no need to override."""
 432         replacement = self.convert_entityref(name)
 433         if replacement is None:
 434             self.unknown_entityref(name)
 435         else:
 436             self.handle_data(replacement)
 437
 438     # Example -- handle data, should be overridden
 439     def handle_data(self, data):
 440         pass
 441
 442     # Example -- handle comment, could be overridden
 443     def handle_comment(self, data):
 444         pass
 445
 446     # Example -- handle declaration, could be overridden
 447     def handle_decl(self, decl):
 448         pass
 449
 450     # Example -- handle processing instruction, could be overridden
 451     def handle_pi(self, data):
 452         pass
 453
 454     # To be overridden -- handlers for unknown objects
 455     def unknown_starttag(self, tag, attrs): pass
 456     def unknown_endtag(self, tag): pass
 457     def unknown_charref(self, ref): pass
 458     def unknown_entityref(self, ref): pass
 459
 460
 461 class TestSGMLParser(SGMLParser):
 462
 463     def __init__(self, verbose=0):
 464         self.testdata = ""
 465         SGMLParser.__init__(self, verbose)
 466
 467     def handle_data(self, data):
 468         self.testdata = self.testdata + data
 469         if len(repr(self.testdata)) >= 70:
 470             self.flush()
 471
 472     def flush(self):
 473         data = self.testdata
 474         if data:
 475             self.testdata = ""
 476             print 'data:', repr(data)
 477
 478     def handle_comment(self, data):
 479         self.flush()
 480         r = repr(data)
 481         if len(r) > 68:
 482             r = r[:32] + '...' + r[-32:]
 483         print 'comment:', r
 484
 485     def unknown_starttag(self, tag, attrs):
 486         self.flush()
 487         if not attrs:
 488             print 'start tag: <' + tag + '>'
 489         else:
 490             print 'start tag: <' + tag,
 491             for name, value in attrs:
 492                 print name + '=' + '"' + value + '"',
 493             print '>'
 494
 495     def unknown_endtag(self, tag):
 496         self.flush()
 497         print 'end tag: </' + tag + '>'
 498
 499     def unknown_entityref(self, ref):
 500         self.flush()
 501         print '*** unknown entity ref: &' + ref + ';'
 502
 503     def unknown_charref(self, ref):
 504         self.flush()
 505         print '*** unknown char ref: &#' + ref + ';'
 506
 507     def unknown_decl(self, data):
 508         self.flush()
 509         print '*** unknown decl: [' + data + ']'
 510
 511     def close(self):
 512         SGMLParser.close(self)
 513         self.flush()
 514
 515
 516 def test(args = None):
 517     import sys
 518
 519     if args is None:
 520         args = sys.argv[1:]
 521
 522     if args and args[0] == '-s':
 523         args = args[1:]
 524         klass = SGMLParser
 525     else:
 526         klass = TestSGMLParser
 527
 528     if args:
 529         file = args[0]
 530     else:
 531         file = 'test.html'
 532
 533     if file == '-':
 534         f = sys.stdin
 535     else:
 536         try:
 537             f = open(file, 'r')
 538         except IOError, msg:
 539             print file, ":", msg
 540             sys.exit(1)
 541
 542     data = f.read()
 543     if f is not sys.stdin:
 544         f.close()
 545
 546     x = klass()
 547     for c in data:
 548         x.feed(c)
 549     x.close()
 550
 551
 552 if __name__ == '__main__':
 553     test()