Lib/markupbase.py

   1 """Shared support for scanning document type declarations in HTML and XHTML.
   2
   3 This module is used as a foundation for the HTMLParser and sgmllib
   4 modules (indirectly, for htmllib as well).  It has no documented
   5 public API and should not be used directly.
   6
   7 """
   8
   9 import re
  10
  11 _declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match
  12 _declstringlit_match = re.compile(r'(\'[^\']*\'|"[^"]*")\s*').match
  13 _commentclose = re.compile(r'--\s*>')
  14 _markedsectionclose = re.compile(r']\s*]\s*>')
  15
  16 # An analysis of the MS-Word extensions is available at
  17 # http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf
  18
  19 _msmarkedsectionclose = re.compile(r']\s*>')
  20
  21 del re
  22
  23
  24 class ParserBase:
  25     """Parser base class which provides some common support methods used
  26     by the SGML/HTML and XHTML parsers."""
  27
  28     def __init__(self):
  29         if self.__class__ is ParserBase:
  30             raise RuntimeError(
  31                 "markupbase.ParserBase must be subclassed")
  32
  33     def error(self, message):
  34         raise NotImplementedError(
  35             "subclasses of ParserBase must override error()")
  36
  37     def reset(self):
  38         self.lineno = 1
  39         self.offset = 0
  40
  41     def getpos(self):
  42         """Return current line number and offset."""
  43         return self.lineno, self.offset
  44
  45     # Internal -- update line number and offset.  This should be
  46     # called for each piece of data exactly once, in order -- in other
  47     # words the concatenation of all the input strings to this
  48     # function should be exactly the entire input.
  49     def updatepos(self, i, j):
  50         if i >= j:
  51             return j
  52         rawdata = self.rawdata
  53         nlines = rawdata.count("\n", i, j)
  54         if nlines:
  55             self.lineno = self.lineno + nlines
  56             pos = rawdata.rindex("\n", i, j) # Should not fail
  57             self.offset = j-(pos+1)
  58         else:
  59             self.offset = self.offset + j-i
  60         return j
  61
  62     _decl_otherchars = ''
  63
  64     # Internal -- parse declaration (for use by subclasses).
  65     def parse_declaration(self, i):
  66         # This is some sort of declaration; in "HTML as
  67         # deployed," this should only be the document type
  68         # declaration ("<!DOCTYPE html...>").
  69         # ISO 8879:1986, however, has more complex
  70         # declaration syntax for elements in <!...>, including:
  71         # --comment--
  72         # [marked section]
  73         # name in the following list: ENTITY, DOCTYPE, ELEMENT,
  74         # ATTLIST, NOTATION, SHORTREF, USEMAP,
  75         # LINKTYPE, LINK, IDLINK, USELINK, SYSTEM
  76         rawdata = self.rawdata
  77         j = i + 2
  78         assert rawdata[i:j] == "<!", "unexpected call to parse_declaration"
  79         if rawdata[j:j+1] == ">":
  80             # the empty comment <!>
  81             return j + 1
  82         if rawdata[j:j+1] in ("-", ""):
  83             # Start of comment followed by buffer boundary,
  84             # or just a buffer boundary.
  85             return -1
  86         # A simple, practical version could look like: ((name|stringlit) S*) + '>'
  87         n = len(rawdata)
  88         if rawdata[j:j+2] == '--': #comment
  89             # Locate --.*-- as the body of the comment
  90             return self.parse_comment(i)
  91         elif rawdata[j] == '[': #marked section
  92             # Locate [statusWord [...arbitrary SGML...]] as the body of the marked section
  93             # Where statusWord is one of TEMP, CDATA, IGNORE, INCLUDE, RCDATA
  94             # Note that this is extended by Microsoft Office "Save as Web" function
  95             # to include [if...] and [endif].
  96             return self.parse_marked_section(i)
  97         else: #all other declaration elements
  98             decltype, j = self._scan_name(j, i)
  99         if j < 0:
 100             return j
 101         if decltype == "doctype":
 102             self._decl_otherchars = ''
 103         while j < n:
 104             c = rawdata[j]
 105             if c == ">":
 106                 # end of declaration syntax
 107                 data = rawdata[i+2:j]
 108                 if decltype == "doctype":
 109                     self.handle_decl(data)
 110                 else:
 111                     self.unknown_decl(data)
 112                 return j + 1
 113             if c in "\"'":
 114                 m = _declstringlit_match(rawdata, j)
 115                 if not m:
 116                     return -1 # incomplete
 117                 j = m.end()
 118             elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
 119                 name, j = self._scan_name(j, i)
 120             elif c in self._decl_otherchars:
 121                 j = j + 1
 122             elif c == "[":
 123                 # this could be handled in a separate doctype parser
 124                 if decltype == "doctype":
 125                     j = self._parse_doctype_subset(j + 1, i)
 126                 elif decltype in ("attlist", "linktype", "link", "element"):
 127                     # must tolerate []'d groups in a content model in an element declaration
 128                     # also in data attribute specifications of attlist declaration
 129                     # also link type declaration subsets in linktype declarations
 130                     # also link attribute specification lists in link declarations
 131                     self.error("unsupported '[' char in %s declaration" % decltype)
 132                 else:
 133                     self.error("unexpected '[' char in declaration")
 134             else:
 135                 self.error(
 136                     "unexpected %r char in declaration" % rawdata[j])
 137             if j < 0:
 138                 return j
 139         return -1 # incomplete
 140
 141     # Internal -- parse a marked section
 142     # Override this to handle MS-word extension syntax <![if word]>content<![endif]>
 143     def parse_marked_section(self, i, report=1):
 144         rawdata= self.rawdata
 145         assert rawdata[i:i+3] == '<![', "unexpected call to parse_marked_section()"
 146         sectName, j = self._scan_name( i+3, i )
 147         if j < 0:
 148             return j
 149         if sectName in ("temp", "cdata", "ignore", "include", "rcdata"):
 150             # look for standard ]]> ending
 151             match= _markedsectionclose.search(rawdata, i+3)
 152         elif sectName in ("if", "else", "endif"):
 153             # look for MS Office ]> ending
 154             match= _msmarkedsectionclose.search(rawdata, i+3)
 155         else:
 156             self.error('unknown status keyword %r in marked section' % rawdata[i+3:j])
 157         if not match:
 158             return -1
 159         if report:
 160             j = match.start(0)
 161             self.unknown_decl(rawdata[i+3: j])
 162         return match.end(0)
 163
 164     # Internal -- parse comment, return length or -1 if not terminated
 165     def parse_comment(self, i, report=1):
 166         rawdata = self.rawdata
 167         if rawdata[i:i+4] != '<!--':
 168             self.error('unexpected call to parse_comment()')
 169         match = _commentclose.search(rawdata, i+4)
 170         if not match:
 171             return -1
 172         if report:
 173             j = match.start(0)
 174             self.handle_comment(rawdata[i+4: j])
 175         return match.end(0)
 176
 177     # Internal -- scan past the internal subset in a <!DOCTYPE declaration,
 178     # returning the index just past any whitespace following the trailing ']'.
 179     def _parse_doctype_subset(self, i, declstartpos):
 180         rawdata = self.rawdata
 181         n = len(rawdata)
 182         j = i
 183         while j < n:
 184             c = rawdata[j]
 185             if c == "<":
 186                 s = rawdata[j:j+2]
 187                 if s == "<":
 188                     # end of buffer; incomplete
 189                     return -1
 190                 if s != "<!":
 191                     self.updatepos(declstartpos, j + 1)
 192                     self.error("unexpected char in internal subset (in %r)" % s)
 193                 if (j + 2) == n:
 194                     # end of buffer; incomplete
 195                     return -1
 196                 if (j + 4) > n:
 197                     # end of buffer; incomplete
 198                     return -1
 199                 if rawdata[j:j+4] == "<!--":
 200                     j = self.parse_comment(j, report=0)
 201                     if j < 0:
 202                         return j
 203                     continue
 204                 name, j = self._scan_name(j + 2, declstartpos)
 205                 if j == -1:
 206                     return -1
 207                 if name not in ("attlist", "element", "entity", "notation"):
 208                     self.updatepos(declstartpos, j + 2)
 209                     self.error(
 210                         "unknown declaration %r in internal subset" % name)
 211                 # handle the individual names
 212                 meth = getattr(self, "_parse_doctype_" + name)
 213                 j = meth(j, declstartpos)
 214                 if j < 0:
 215                     return j
 216             elif c == "%":
 217                 # parameter entity reference
 218                 if (j + 1) == n:
 219                     # end of buffer; incomplete
 220                     return -1
 221                 s, j = self._scan_name(j + 1, declstartpos)
 222                 if j < 0:
 223                     return j
 224                 if rawdata[j] == ";":
 225                     j = j + 1
 226             elif c == "]":
 227                 j = j + 1
 228                 while j < n and rawdata[j].isspace():
 229                     j = j + 1
 230                 if j < n:
 231                     if rawdata[j] == ">":
 232                         return j
 233                     self.updatepos(declstartpos, j)
 234                     self.error("unexpected char after internal subset")
 235                 else:
 236                     return -1
 237             elif c.isspace():
 238                 j = j + 1
 239             else:
 240                 self.updatepos(declstartpos, j)
 241                 self.error("unexpected char %r in internal subset" % c)
 242         # end of buffer reached
 243         return -1
 244
 245     # Internal -- scan past <!ELEMENT declarations
 246     def _parse_doctype_element(self, i, declstartpos):
 247         name, j = self._scan_name(i, declstartpos)
 248         if j == -1:
 249             return -1
 250         # style content model; just skip until '>'
 251         rawdata = self.rawdata
 252         if '>' in rawdata[j:]:
 253             return rawdata.find(">", j) + 1
 254         return -1
 255
 256     # Internal -- scan past <!ATTLIST declarations
 257     def _parse_doctype_attlist(self, i, declstartpos):
 258         rawdata = self.rawdata
 259         name, j = self._scan_name(i, declstartpos)
 260         c = rawdata[j:j+1]
 261         if c == "":
 262             return -1
 263         if c == ">":
 264             return j + 1
 265         while 1:
 266             # scan a series of attribute descriptions; simplified:
 267             #   name type [value] [#constraint]
 268             name, j = self._scan_name(j, declstartpos)
 269             if j < 0:
 270                 return j
 271             c = rawdata[j:j+1]
 272             if c == "":
 273                 return -1
 274             if c == "(":
 275                 # an enumerated type; look for ')'
 276                 if ")" in rawdata[j:]:
 277                     j = rawdata.find(")", j) + 1
 278                 else:
 279                     return -1
 280                 while rawdata[j:j+1].isspace():
 281                     j = j + 1
 282                 if not rawdata[j:]:
 283                     # end of buffer, incomplete
 284                     return -1
 285             else:
 286                 name, j = self._scan_name(j, declstartpos)
 287             c = rawdata[j:j+1]
 288             if not c:
 289                 return -1
 290             if c in "'\"":
 291                 m = _declstringlit_match(rawdata, j)
 292                 if m:
 293                     j = m.end()
 294                 else:
 295                     return -1
 296                 c = rawdata[j:j+1]
 297                 if not c:
 298                     return -1
 299             if c == "#":
 300                 if rawdata[j:] == "#":
 301                     # end of buffer
 302                     return -1
 303                 name, j = self._scan_name(j + 1, declstartpos)
 304                 if j < 0:
 305                     return j
 306                 c = rawdata[j:j+1]
 307                 if not c:
 308                     return -1
 309             if c == '>':
 310                 # all done
 311                 return j + 1
 312
 313     # Internal -- scan past <!NOTATION declarations
 314     def _parse_doctype_notation(self, i, declstartpos):
 315         name, j = self._scan_name(i, declstartpos)
 316         if j < 0:
 317             return j
 318         rawdata = self.rawdata
 319         while 1:
 320             c = rawdata[j:j+1]
 321             if not c:
 322                 # end of buffer; incomplete
 323                 return -1
 324             if c == '>':
 325                 return j + 1
 326             if c in "'\"":
 327                 m = _declstringlit_match(rawdata, j)
 328                 if not m:
 329                     return -1
 330                 j = m.end()
 331             else:
 332                 name, j = self._scan_name(j, declstartpos)
 333                 if j < 0:
 334                     return j
 335
 336     # Internal -- scan past <!ENTITY declarations
 337     def _parse_doctype_entity(self, i, declstartpos):
 338         rawdata = self.rawdata
 339         if rawdata[i:i+1] == "%":
 340             j = i + 1
 341             while 1:
 342                 c = rawdata[j:j+1]
 343                 if not c:
 344                     return -1
 345                 if c.isspace():
 346                     j = j + 1
 347                 else:
 348                     break
 349         else:
 350             j = i
 351         name, j = self._scan_name(j, declstartpos)
 352         if j < 0:
 353             return j
 354         while 1:
 355             c = self.rawdata[j:j+1]
 356             if not c:
 357                 return -1
 358             if c in "'\"":
 359                 m = _declstringlit_match(rawdata, j)
 360                 if m:
 361                     j = m.end()
 362                 else:
 363                     return -1    # incomplete
 364             elif c == ">":
 365                 return j + 1
 366             else:
 367                 name, j = self._scan_name(j, declstartpos)
 368                 if j < 0:
 369                     return j
 370
 371     # Internal -- scan a name token and the new position and the token, or
 372     # return -1 if we've reached the end of the buffer.
 373     def _scan_name(self, i, declstartpos):
 374         rawdata = self.rawdata
 375         n = len(rawdata)
 376         if i == n:
 377             return None, -1
 378         m = _declname_match(rawdata, i)
 379         if m:
 380             s = m.group()
 381             name = s.strip()
 382             if (i + len(s)) == n:
 383                 return None, -1  # end of buffer
 384             return name.lower(), m.end()
 385         else:
 386             self.updatepos(declstartpos, i)
 387             self.error("expected name token at %r"
 388                        % rawdata[declstartpos:declstartpos+20])
 389
 390     # To be overridden -- handlers for unknown objects
 391     def unknown_decl(self, data):
 392         pass