1 """Shared support for scanning document type declarations in HTML and XHTML.
3 This module is used as a foundation for the html.parser module. It has no
4 documented public API and should not be used directly.
10 _declname_match
= re
.compile(r
'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match
11 _declstringlit_match
= re
.compile(r
'(\'[^
\']*\'|
"[^"]*")\s*').match
12 _commentclose = re.compile(r'--\s*>')
13 _markedsectionclose = re.compile(r']\s*]\s*>')
15 # An analysis of the MS-Word extensions is available at
16 # http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf
18 _msmarkedsectionclose = re.compile(r']\s*>')
24 """Parser base class which provides some common support methods used
25 by the SGML/HTML and XHTML parsers."""
28 if self.__class__ is ParserBase:
30 "_markupbase
.ParserBase must be subclassed
")
32 def error(self, message):
33 raise NotImplementedError(
34 "subclasses of ParserBase must override
error()")
41 """Return current line number and offset."""
42 return self.lineno, self.offset
44 # Internal -- update line number and offset. This should be
45 # called for each piece of data exactly once, in order -- in other
46 # words the concatenation of all the input strings to this
47 # function should be exactly the entire input.
48 def updatepos(self, i, j):
51 rawdata = self.rawdata
52 nlines = rawdata.count("\n", i, j)
54 self.lineno = self.lineno + nlines
55 pos = rawdata.rindex("\n", i, j) # Should not fail
56 self.offset = j-(pos+1)
58 self.offset = self.offset + j-i
63 # Internal -- parse declaration (for use by subclasses).
64 def parse_declaration(self, i):
65 # This is some sort of declaration; in "HTML
as
66 # deployed," this should only be the document type
67 # declaration ("<!DOCTYPE html...>").
68 # ISO 8879:1986, however, has more complex
69 # declaration syntax for elements in <!...>, including:
72 # name in the following list: ENTITY, DOCTYPE, ELEMENT,
73 # ATTLIST, NOTATION, SHORTREF, USEMAP,
74 # LINKTYPE, LINK, IDLINK, USELINK, SYSTEM
75 rawdata
= self
.rawdata
77 assert rawdata
[i
:j
] == "<!", "unexpected call to parse_declaration"
78 if rawdata
[j
:j
+1] == ">":
79 # the empty comment <!>
81 if rawdata
[j
:j
+1] in ("-", ""):
82 # Start of comment followed by buffer boundary,
83 # or just a buffer boundary.
85 # A simple, practical version could look like: ((name|stringlit) S*) + '>'
87 if rawdata
[j
:j
+2] == '--': #comment
88 # Locate --.*-- as the body of the comment
89 return self
.parse_comment(i
)
90 elif rawdata
[j
] == '[': #marked section
91 # Locate [statusWord [...arbitrary SGML...]] as the body of the marked section
92 # Where statusWord is one of TEMP, CDATA, IGNORE, INCLUDE, RCDATA
93 # Note that this is extended by Microsoft Office "Save as Web" function
94 # to include [if...] and [endif].
95 return self
.parse_marked_section(i
)
96 else: #all other declaration elements
97 decltype
, j
= self
._scan
_name
(j
, i
)
100 if decltype
== "doctype":
101 self
._decl
_otherchars
= ''
105 # end of declaration syntax
106 data
= rawdata
[i
+2:j
]
107 if decltype
== "doctype":
108 self
.handle_decl(data
)
110 self
.unknown_decl(data
)
113 m
= _declstringlit_match(rawdata
, j
)
115 return -1 # incomplete
117 elif c
in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
118 name
, j
= self
._scan
_name
(j
, i
)
119 elif c
in self
._decl
_otherchars
:
122 # this could be handled in a separate doctype parser
123 if decltype
== "doctype":
124 j
= self
._parse
_doctype
_subset
(j
+ 1, i
)
125 elif decltype
in ("attlist", "linktype", "link", "element"):
126 # must tolerate []'d groups in a content model in an element declaration
127 # also in data attribute specifications of attlist declaration
128 # also link type declaration subsets in linktype declarations
129 # also link attribute specification lists in link declarations
130 self
.error("unsupported '[' char in %s declaration" % decltype
)
132 self
.error("unexpected '[' char in declaration")
135 "unexpected %r char in declaration" % rawdata
[j
])
138 return -1 # incomplete
140 # Internal -- parse a marked section
141 # Override this to handle MS-word extension syntax <![if word]>content<![endif]>
142 def parse_marked_section(self
, i
, report
=1):
143 rawdata
= self
.rawdata
144 assert rawdata
[i
:i
+3] == '<![', "unexpected call to parse_marked_section()"
145 sectName
, j
= self
._scan
_name
( i
+3, i
)
148 if sectName
in ("temp", "cdata", "ignore", "include", "rcdata"):
149 # look for standard ]]> ending
150 match
= _markedsectionclose
.search(rawdata
, i
+3)
151 elif sectName
in ("if", "else", "endif"):
152 # look for MS Office ]> ending
153 match
= _msmarkedsectionclose
.search(rawdata
, i
+3)
155 self
.error('unknown status keyword %r in marked section' % rawdata
[i
+3:j
])
160 self
.unknown_decl(rawdata
[i
+3: j
])
163 # Internal -- parse comment, return length or -1 if not terminated
164 def parse_comment(self
, i
, report
=1):
165 rawdata
= self
.rawdata
166 if rawdata
[i
:i
+4] != '<!--':
167 self
.error('unexpected call to parse_comment()')
168 match
= _commentclose
.search(rawdata
, i
+4)
173 self
.handle_comment(rawdata
[i
+4: j
])
176 # Internal -- scan past the internal subset in a <!DOCTYPE declaration,
177 # returning the index just past any whitespace following the trailing ']'.
178 def _parse_doctype_subset(self
, i
, declstartpos
):
179 rawdata
= self
.rawdata
187 # end of buffer; incomplete
190 self
.updatepos(declstartpos
, j
+ 1)
191 self
.error("unexpected char in internal subset (in %r)" % s
)
193 # end of buffer; incomplete
196 # end of buffer; incomplete
198 if rawdata
[j
:j
+4] == "<!--":
199 j
= self
.parse_comment(j
, report
=0)
203 name
, j
= self
._scan
_name
(j
+ 2, declstartpos
)
206 if name
not in ("attlist", "element", "entity", "notation"):
207 self
.updatepos(declstartpos
, j
+ 2)
209 "unknown declaration %r in internal subset" % name
)
210 # handle the individual names
211 meth
= getattr(self
, "_parse_doctype_" + name
)
212 j
= meth(j
, declstartpos
)
216 # parameter entity reference
218 # end of buffer; incomplete
220 s
, j
= self
._scan
_name
(j
+ 1, declstartpos
)
223 if rawdata
[j
] == ";":
227 while j
< n
and rawdata
[j
].isspace():
230 if rawdata
[j
] == ">":
232 self
.updatepos(declstartpos
, j
)
233 self
.error("unexpected char after internal subset")
239 self
.updatepos(declstartpos
, j
)
240 self
.error("unexpected char %r in internal subset" % c
)
241 # end of buffer reached
244 # Internal -- scan past <!ELEMENT declarations
245 def _parse_doctype_element(self
, i
, declstartpos
):
246 name
, j
= self
._scan
_name
(i
, declstartpos
)
249 # style content model; just skip until '>'
250 rawdata
= self
.rawdata
251 if '>' in rawdata
[j
:]:
252 return rawdata
.find(">", j
) + 1
255 # Internal -- scan past <!ATTLIST declarations
256 def _parse_doctype_attlist(self
, i
, declstartpos
):
257 rawdata
= self
.rawdata
258 name
, j
= self
._scan
_name
(i
, declstartpos
)
265 # scan a series of attribute descriptions; simplified:
266 # name type [value] [#constraint]
267 name
, j
= self
._scan
_name
(j
, declstartpos
)
274 # an enumerated type; look for ')'
275 if ")" in rawdata
[j
:]:
276 j
= rawdata
.find(")", j
) + 1
279 while rawdata
[j
:j
+1].isspace():
282 # end of buffer, incomplete
285 name
, j
= self
._scan
_name
(j
, declstartpos
)
290 m
= _declstringlit_match(rawdata
, j
)
299 if rawdata
[j
:] == "#":
302 name
, j
= self
._scan
_name
(j
+ 1, declstartpos
)
312 # Internal -- scan past <!NOTATION declarations
313 def _parse_doctype_notation(self
, i
, declstartpos
):
314 name
, j
= self
._scan
_name
(i
, declstartpos
)
317 rawdata
= self
.rawdata
321 # end of buffer; incomplete
326 m
= _declstringlit_match(rawdata
, j
)
331 name
, j
= self
._scan
_name
(j
, declstartpos
)
335 # Internal -- scan past <!ENTITY declarations
336 def _parse_doctype_entity(self
, i
, declstartpos
):
337 rawdata
= self
.rawdata
338 if rawdata
[i
:i
+1] == "%":
350 name
, j
= self
._scan
_name
(j
, declstartpos
)
354 c
= self
.rawdata
[j
:j
+1]
358 m
= _declstringlit_match(rawdata
, j
)
362 return -1 # incomplete
366 name
, j
= self
._scan
_name
(j
, declstartpos
)
370 # Internal -- scan a name token and the new position and the token, or
371 # return -1 if we've reached the end of the buffer.
372 def _scan_name(self
, i
, declstartpos
):
373 rawdata
= self
.rawdata
377 m
= _declname_match(rawdata
, i
)
381 if (i
+ len(s
)) == n
:
382 return None, -1 # end of buffer
383 return name
.lower(), m
.end()
385 self
.updatepos(declstartpos
, i
)
386 self
.error("expected name token at %r"
387 % rawdata
[declstartpos
:declstartpos
+20])
389 # To be overridden -- handlers for unknown objects
390 def unknown_decl(self
, data
):