1 """Shared support for scanning document type declarations in HTML and XHTML.
3 This module is used as a foundation for the HTMLParser and sgmllib
4 modules (indirectly, for htmllib as well). It has no documented
5 public API and should not be used directly.
11 _declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9]*\s*').match
12 _declstringlit_match = re.compile(r'(\'[^\']*\'|"[^"]*")\s*').match
13 _commentclose = re.compile(r'--\s*>')
14 _markedsectionclose = re.compile(r']\s*]\s*>')
16 # An analysis of the MS-Word extensions is available at
17 # http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf
19 _msmarkedsectionclose = re.compile(r']\s*>')
25 """Parser base class which provides some common support methods used
26 by the SGML/HTML and XHTML parsers."""
29 if self.__class__ is ParserBase:
31 "markupbase.ParserBase must be subclassed")
33 def error(self, message):
34 raise NotImplementedError(
35 "subclasses of ParserBase must override error()")
42 """Return current line number and offset."""
43 return self.lineno, self.offset
45 # Internal -- update line number and offset. This should be
46 # called for each piece of data exactly once, in order -- in other
47 # words the concatenation of all the input strings to this
48 # function should be exactly the entire input.
49 def updatepos(self, i, j):
52 rawdata = self.rawdata
53 nlines = rawdata.count("\n", i, j)
55 self.lineno = self.lineno + nlines
56 pos = rawdata.rindex("\n", i, j) # Should not fail
57 self.offset = j-(pos+1)
59 self.offset = self.offset + j-i
64 # Internal -- parse declaration (for use by subclasses).
65 def parse_declaration(self, i):
66 # This is some sort of declaration; in "HTML as
67 # deployed," this should only be the document type
68 # declaration ("<!DOCTYPE html...>").
69 # ISO 8879:1986, however, has more complex
70 # declaration syntax for elements in <!...>, including:
73 # name in the following list: ENTITY, DOCTYPE, ELEMENT,
74 # ATTLIST, NOTATION, SHORTREF, USEMAP,
75 # LINKTYPE, LINK, IDLINK, USELINK, SYSTEM
76 rawdata = self.rawdata
78 assert rawdata[i:j] == "<!", "unexpected call to parse_declaration"
79 if rawdata[j:j+1] == ">":
80 # the empty comment <!>
82 if rawdata[j:j+1] in ("-", ""):
83 # Start of comment followed by buffer boundary,
84 # or just a buffer boundary.
86 # A simple, practical version could look like: ((name|stringlit) S*) + '>'
88 if rawdata[j:j+2] == '--': #comment
89 # Locate --.*-- as the body of the comment
90 return self.parse_comment(i)
91 elif rawdata[j] == '[': #marked section
92 # Locate [statusWord [...arbitrary SGML...]] as the body of the marked section
93 # Where statusWord is one of TEMP, CDATA, IGNORE, INCLUDE, RCDATA
94 # Note that this is extended by Microsoft Office "Save as Web" function
95 # to include [if...] and [endif].
96 return self.parse_marked_section(i)
97 else: #all other declaration elements
98 decltype, j = self._scan_name(j, i)
101 if decltype == "doctype":
102 self._decl_otherchars = ''
106 # end of declaration syntax
107 data = rawdata[i+2:j]
108 if decltype == "doctype":
109 self.handle_decl(data)
111 self.unknown_decl(data)
114 m = _declstringlit_match(rawdata, j)
116 return -1 # incomplete
118 elif c in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ":
119 name, j = self._scan_name(j, i)
120 elif c in self._decl_otherchars:
123 # this could be handled in a separate doctype parser
124 if decltype == "doctype":
125 j = self._parse_doctype_subset(j + 1, i)
126 elif decltype in ("attlist", "linktype", "link", "element"):
127 # must tolerate []'d groups in a content model in an element declaration
128 # also in data attribute specifications of attlist declaration
129 # also link type declaration subsets in linktype declarations
130 # also link attribute specification lists in link declarations
131 self.error("unsupported '[' char in %s declaration" % decltype)
133 self.error("unexpected '[' char in declaration")
136 "unexpected %r char in declaration" % rawdata[j])
139 return -1 # incomplete
141 # Internal -- parse a marked section
142 # Override this to handle MS-word extension syntax <![if word]>content<![endif]>
143 def parse_marked_section(self, i, report=1):
144 rawdata= self.rawdata
145 assert rawdata[i:i+3] == '<![', "unexpected call to parse_marked_section()"
146 sectName, j = self._scan_name( i+3, i )
149 if sectName in ("temp", "cdata", "ignore", "include", "rcdata"):
150 # look for standard ]]> ending
151 match= _markedsectionclose.search(rawdata, i+3)
152 elif sectName in ("if", "else", "endif"):
153 # look for MS Office ]> ending
154 match= _msmarkedsectionclose.search(rawdata, i+3)
156 self.error('unknown status keyword %r in marked section' % rawdata[i+3:j])
161 self.unknown_decl(rawdata[i+3: j])
164 # Internal -- parse comment, return length or -1 if not terminated
165 def parse_comment(self, i, report=1):
166 rawdata = self.rawdata
167 if rawdata[i:i+4] != '<!--':
168 self.error('unexpected call to parse_comment()')
169 match = _commentclose.search(rawdata, i+4)
174 self.handle_comment(rawdata[i+4: j])
177 # Internal -- scan past the internal subset in a <!DOCTYPE declaration,
178 # returning the index just past any whitespace following the trailing ']'.
179 def _parse_doctype_subset(self, i, declstartpos):
180 rawdata = self.rawdata
188 # end of buffer; incomplete
191 self.updatepos(declstartpos, j + 1)
192 self.error("unexpected char in internal subset (in %r)" % s)
194 # end of buffer; incomplete
197 # end of buffer; incomplete
199 if rawdata[j:j+4] == "<!--":
200 j = self.parse_comment(j, report=0)
204 name, j = self._scan_name(j + 2, declstartpos)
207 if name not in ("attlist", "element", "entity", "notation"):
208 self.updatepos(declstartpos, j + 2)
210 "unknown declaration %r in internal subset" % name)
211 # handle the individual names
212 meth = getattr(self, "_parse_doctype_" + name)
213 j = meth(j, declstartpos)
217 # parameter entity reference
219 # end of buffer; incomplete
221 s, j = self._scan_name(j + 1, declstartpos)
224 if rawdata[j] == ";":
228 while j < n and rawdata[j].isspace():
231 if rawdata[j] == ">":
233 self.updatepos(declstartpos, j)
234 self.error("unexpected char after internal subset")
240 self.updatepos(declstartpos, j)
241 self.error("unexpected char %r in internal subset" % c)
242 # end of buffer reached
245 # Internal -- scan past <!ELEMENT declarations
246 def _parse_doctype_element(self, i, declstartpos):
247 name, j = self._scan_name(i, declstartpos)
250 # style content model; just skip until '>'
251 rawdata = self.rawdata
252 if '>' in rawdata[j:]:
253 return rawdata.find(">", j) + 1
256 # Internal -- scan past <!ATTLIST declarations
257 def _parse_doctype_attlist(self, i, declstartpos):
258 rawdata = self.rawdata
259 name, j = self._scan_name(i, declstartpos)
266 # scan a series of attribute descriptions; simplified:
267 # name type [value] [#constraint]
268 name, j = self._scan_name(j, declstartpos)
275 # an enumerated type; look for ')'
276 if ")" in rawdata[j:]:
277 j = rawdata.find(")", j) + 1
280 while rawdata[j:j+1].isspace():
283 # end of buffer, incomplete
286 name, j = self._scan_name(j, declstartpos)
291 m = _declstringlit_match(rawdata, j)
300 if rawdata[j:] == "#":
303 name, j = self._scan_name(j + 1, declstartpos)
313 # Internal -- scan past <!NOTATION declarations
314 def _parse_doctype_notation(self, i, declstartpos):
315 name, j = self._scan_name(i, declstartpos)
318 rawdata = self.rawdata
322 # end of buffer; incomplete
327 m = _declstringlit_match(rawdata, j)
332 name, j = self._scan_name(j, declstartpos)
336 # Internal -- scan past <!ENTITY declarations
337 def _parse_doctype_entity(self, i, declstartpos):
338 rawdata = self.rawdata
339 if rawdata[i:i+1] == "%":
351 name, j = self._scan_name(j, declstartpos)
355 c = self.rawdata[j:j+1]
359 m = _declstringlit_match(rawdata, j)
363 return -1 # incomplete
367 name, j = self._scan_name(j, declstartpos)
371 # Internal -- scan a name token and the new position and the token, or
372 # return -1 if we've reached the end of the buffer.
373 def _scan_name(self, i, declstartpos):
374 rawdata = self.rawdata
378 m = _declname_match(rawdata, i)
382 if (i + len(s)) == n:
383 return None, -1 # end of buffer
384 return name.lower(), m.end()
386 self.updatepos(declstartpos, i)
387 self.error("expected name token at %r"
388 % rawdata[declstartpos:declstartpos+20])
390 # To be overridden -- handlers for unknown objects
391 def unknown_decl(self, data):