1 """Shared support for scanning document type declarations in HTML and XHTML.
3 This module is used as a foundation for the HTMLParser and sgmllib
4 modules (indirectly, for htmllib as well). It has no documented
5 public API and should not be used directly.
11 _declname_match
= re
.compile(r
'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match
12 _declstringlit_match
= re
.compile(r
'(\'[^
\']*\'|
"[^"]*")\s*').match
13 _commentclose = re.compile(r'--\s*>')
14 _markedsectionclose = re.compile(r']\s*]\s*>')
16 # An analysis of the MS-Word extensions is available at
17 # http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf
19 _msmarkedsectionclose = re.compile(r']\s*>')
25 """Parser base class which provides some common support methods used
26 by the SGML/HTML and XHTML parsers."""
29 if self.__class__ is ParserBase:
31 "markupbase
.ParserBase must be subclassed
")
33 def error(self, message):
34 raise NotImplementedError(
35 "subclasses of ParserBase must override
error()")
42 """Return current line number and offset."""
43 return self.lineno, self.offset
45 # Internal -- update line number and offset. This should be
46 # called for each piece of data exactly once, in order -- in other
47 # words the concatenation of all the input strings to this
48 # function should be exactly the entire input.
49 def updatepos(self, i, j):
52 rawdata = self.rawdata
53 nlines = rawdata.count("\n", i, j)
55 self.lineno = self.lineno + nlines
56 pos = rawdata.rindex("\n", i, j) # Should not fail
57 self.offset = j-(pos+1)
59 self.offset = self.offset + j-i
64 # Internal -- parse declaration (for use by subclasses).
65 def parse_declaration(self, i):
66 # This is some sort of declaration; in "HTML
as
67 # deployed," this should only be the document type
68 # declaration ("<!DOCTYPE html...>").
69 # ISO 8879:1986, however, has more complex
70 # declaration syntax for elements in <!...>, including:
73 # name in the following list: ENTITY, DOCTYPE, ELEMENT,
74 # ATTLIST, NOTATION, SHORTREF, USEMAP,
75 # LINKTYPE, LINK, IDLINK, USELINK, SYSTEM
76 rawdata
= self
.rawdata
78 assert rawdata
[i
:j
] == "<!", "unexpected call to parse_declaration"
79 if rawdata
[j
:j
+1] in ("-", ""):
80 # Start of comment followed by buffer boundary,
81 # or just a buffer boundary.
83 # A simple, practical version could look like: ((name|stringlit) S*) + '>'
85 if rawdata
[j
:j
+1] == '--': #comment
86 # Locate --.*-- as the body of the comment
87 return self
.parse_comment(i
)
88 elif rawdata
[j
] == '[': #marked section
89 # Locate [statusWord [...arbitrary SGML...]] as the body of the marked section
90 # Where statusWord is one of TEMP, CDATA, IGNORE, INCLUDE, RCDATA
91 # Note that this is extended by Microsoft Office "Save as Web" function
92 # to include [if...] and [endif].
93 return self
.parse_marked_section(i
)
94 else: #all other declaration elements
95 decltype
, j
= self
._scan
_name
(j
, i
)
98 if decltype
== "doctype":
99 self
._decl
_otherchars
= ''
103 # end of declaration syntax
104 data
= rawdata
[i
+2:j
]
105 if decltype
== "doctype":
106 self
.handle_decl(data
)
108 self
.unknown_decl(data
)
111 m
= _declstringlit_match(rawdata
, j
)
113 return -1 # incomplete
115 elif c
in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
116 name
, j
= self
._scan
_name
(j
, i
)
117 elif c
in self
._decl
_otherchars
:
120 # this could be handled in a separate doctype parser
121 if decltype
== "doctype":
122 j
= self
._parse
_doctype
_subset
(j
+ 1, i
)
123 elif decltype
in ("attlist", "linktype", "link", "element"):
124 # must tolerate []'d groups in a content model in an element declaration
125 # also in data attribute specifications of attlist declaration
126 # also link type declaration subsets in linktype declarations
127 # also link attribute specification lists in link declarations
128 self
.error("unsupported '[' char in %s declaration" % decltype
)
130 self
.error("unexpected '[' char in declaration")
133 "unexpected %r char in declaration" % rawdata
[j
])
136 return -1 # incomplete
138 # Internal -- parse a marked section
139 # Override this to handle MS-word extension syntax <![if word]>content<![endif]>
140 def parse_marked_section( self
, i
, report
=1 ):
141 rawdata
= self
.rawdata
142 assert rawdata
[i
:i
+3] == '<![', "unexpected call to parse_marked_section()"
143 sectName
, j
= self
._scan
_name
( i
+3, i
)
146 if sectName
in ("temp", "cdata", "ignore", "include", "rcdata"):
147 # look for standard ]]> ending
148 match
= _markedsectionclose
.search(rawdata
, i
+3)
149 elif sectName
in ("if", "else", "endif"):
150 # look for MS Office ]> ending
151 match
= _msmarkedsectionclose
.search(rawdata
, i
+3)
153 self
.error('unknown status keyword %r in marked section' % rawdata
[i
+3:j
])
158 self
.unknown_decl(rawdata
[i
+3: j
])
161 # Internal -- parse comment, return length or -1 if not terminated
162 def parse_comment(self
, i
, report
=1):
163 rawdata
= self
.rawdata
164 if rawdata
[i
:i
+4] != '<!--':
165 self
.error('unexpected call to parse_comment()')
166 match
= _commentclose
.search(rawdata
, i
+4)
171 self
.handle_comment(rawdata
[i
+4: j
])
174 # Internal -- scan past the internal subset in a <!DOCTYPE declaration,
175 # returning the index just past any whitespace following the trailing ']'.
176 def _parse_doctype_subset(self
, i
, declstartpos
):
177 rawdata
= self
.rawdata
185 # end of buffer; incomplete
188 self
.updatepos(declstartpos
, j
+ 1)
189 self
.error("unexpected char in internal subset (in %r)" % s
)
191 # end of buffer; incomplete
194 # end of buffer; incomplete
196 if rawdata
[j
:j
+4] == "<!--":
197 j
= self
.parse_comment(j
, report
=0)
201 name
, j
= self
._scan
_name
(j
+ 2, declstartpos
)
204 if name
not in ("attlist", "element", "entity", "notation"):
205 self
.updatepos(declstartpos
, j
+ 2)
207 "unknown declaration %r in internal subset" % name
)
208 # handle the individual names
209 meth
= getattr(self
, "_parse_doctype_" + name
)
210 j
= meth(j
, declstartpos
)
214 # parameter entity reference
216 # end of buffer; incomplete
218 s
, j
= self
._scan
_name
(j
+ 1, declstartpos
)
221 if rawdata
[j
] == ";":
225 while j
< n
and rawdata
[j
].isspace():
228 if rawdata
[j
] == ">":
230 self
.updatepos(declstartpos
, j
)
231 self
.error("unexpected char after internal subset")
237 self
.updatepos(declstartpos
, j
)
238 self
.error("unexpected char %r in internal subset" % c
)
239 # end of buffer reached
242 # Internal -- scan past <!ELEMENT declarations
243 def _parse_doctype_element(self
, i
, declstartpos
):
244 name
, j
= self
._scan
_name
(i
, declstartpos
)
247 # style content model; just skip until '>'
248 rawdata
= self
.rawdata
249 if '>' in rawdata
[j
:]:
250 return rawdata
.find(">", j
) + 1
253 # Internal -- scan past <!ATTLIST declarations
254 def _parse_doctype_attlist(self
, i
, declstartpos
):
255 rawdata
= self
.rawdata
256 name
, j
= self
._scan
_name
(i
, declstartpos
)
263 # scan a series of attribute descriptions; simplified:
264 # name type [value] [#constraint]
265 name
, j
= self
._scan
_name
(j
, declstartpos
)
272 # an enumerated type; look for ')'
273 if ")" in rawdata
[j
:]:
274 j
= rawdata
.find(")", j
) + 1
277 while rawdata
[j
:j
+1].isspace():
280 # end of buffer, incomplete
283 name
, j
= self
._scan
_name
(j
, declstartpos
)
288 m
= _declstringlit_match(rawdata
, j
)
297 if rawdata
[j
:] == "#":
300 name
, j
= self
._scan
_name
(j
+ 1, declstartpos
)
310 # Internal -- scan past <!NOTATION declarations
311 def _parse_doctype_notation(self
, i
, declstartpos
):
312 name
, j
= self
._scan
_name
(i
, declstartpos
)
315 rawdata
= self
.rawdata
319 # end of buffer; incomplete
324 m
= _declstringlit_match(rawdata
, j
)
329 name
, j
= self
._scan
_name
(j
, declstartpos
)
333 # Internal -- scan past <!ENTITY declarations
334 def _parse_doctype_entity(self
, i
, declstartpos
):
335 rawdata
= self
.rawdata
336 if rawdata
[i
:i
+1] == "%":
348 name
, j
= self
._scan
_name
(j
, declstartpos
)
352 c
= self
.rawdata
[j
:j
+1]
356 m
= _declstringlit_match(rawdata
, j
)
360 return -1 # incomplete
364 name
, j
= self
._scan
_name
(j
, declstartpos
)
368 # Internal -- scan a name token and the new position and the token, or
369 # return -1 if we've reached the end of the buffer.
370 def _scan_name(self
, i
, declstartpos
):
371 rawdata
= self
.rawdata
375 m
= _declname_match(rawdata
, i
)
379 if (i
+ len(s
)) == n
:
380 return None, -1 # end of buffer
381 return name
.lower(), m
.end()
383 self
.updatepos(declstartpos
, i
)
384 self
.error("expected name token at %r"
385 % rawdata
[declstartpos
:declstartpos
+20])
387 # To be overridden -- handlers for unknown objects
388 def unknown_decl(self
, data
):