htmlparse.cc

   1 /* htmlparse.cc: simple HTML parser for omega indexer
   2  *
   3  * ----START-LICENCE----
   4  * Copyright 1999,2000,2001 BrightStation PLC
   5  * Copyright 2001 Ananova Ltd
   6  * Copyright 2002 Olly Betts
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License as
  10  * published by the Free Software Foundation; either version 2 of the
  11  * License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
  21  * USA
  22  * -----END-LICENCE-----
  23  */
  24
  25 #include <algorithm>
  26 using std::find;
  27 using std::find_if;
  28 #include "htmlparse.h"
  29 #include <stdio.h>
  30 #include <ctype.h>
  31 #include <iostream>
  32
  33 using namespace std;
  34
  35 map<string, unsigned int> HtmlParser::named_ents;
  36
  37 inline static bool
  38 p_alpha(char c)
  39 {
  40     return isalpha(c);
  41 }
  42
  43 inline static bool
  44 p_notdigit(char c)
  45 {
  46     return !isdigit(c);
  47 }
  48
  49 inline static bool
  50 p_notxdigit(char c)
  51 {
  52     return !isxdigit(c);
  53 }
  54
  55 inline static bool
  56 p_notalnum(char c)
  57 {
  58     return !isalnum(c);
  59 }
  60
  61 inline static bool
  62 p_notwhitespace(char c)
  63 {
  64     return !isspace(c);
  65 }
  66
  67 inline static bool
  68 p_nottag(char c)
  69 {
  70     return !isalnum(c) && c != '.' && c != '-';
  71 }
  72
  73 inline static bool
  74 p_whitespacegt(char c)
  75 {
  76     return isspace(c) || c == '>';
  77 }
  78
  79 inline static bool
  80 p_whitespaceeqgt(char c)
  81 {
  82     return isspace(c) || c == '=' || c == '>';
  83 }
  84
  85 HtmlParser::HtmlParser()
  86 {
  87     static struct ent { const char *n; unsigned int v; } ents[] = {
  88         { "quot", 34 },
  89         { "amp", 38 },
  90         { "lt", 60 },
  91         { "gt", 62 },
  92         { "AElig", 198 },
  93         { "Aacute", 193 },
  94         { "Acirc", 194 },
  95         { "Agrave", 192 },
  96         { "Aring", 197 },
  97         { "Atilde", 195 },
  98         { "Auml", 196 },
  99         { "Ccedil", 199 },
 100         { "ETH", 208 },
 101         { "Eacute", 201 },
 102         { "Ecirc", 202 },
 103         { "Egrave", 200 },
 104         { "Euml", 203 },
 105         { "Iacute", 205 },
 106         { "Icirc", 206 },
 107         { "Igrave", 204 },
 108         { "Iuml", 207 },
 109         { "Ntilde", 209 },
 110         { "Oacute", 211 },
 111         { "Ocirc", 212 },
 112         { "Ograve", 210 },
 113         { "Oslash", 216 },
 114         { "Otilde", 213 },
 115         { "Ouml", 214 },
 116         { "THORN", 222 },
 117         { "Uacute", 218 },
 118         { "Ucirc", 219 },
 119         { "Ugrave", 217 },
 120         { "Uuml", 220 },
 121         { "Yacute", 221 },
 122         { "aacute", 225 },
 123         { "acirc", 226 },
 124         { "acute", 180 },
 125         { "aelig", 230 },
 126         { "agrave", 224 },
 127         { "aring", 229 },
 128         { "atilde", 227 },
 129         { "auml", 228 },
 130         { "brvbar", 166 },
 131         { "ccedil", 231 },
 132         { "cedil", 184 },
 133         { "cent", 162 },
 134         { "copy", 169 },
 135         { "curren", 164 },
 136         { "deg", 176 },
 137         { "divide", 247 },
 138         { "eacute", 233 },
 139         { "ecirc", 234 },
 140         { "egrave", 232 },
 141         { "eth", 240 },
 142         { "euml", 235 },
 143         { "frac12", 189 },
 144         { "frac14", 188 },
 145         { "frac34", 190 },
 146         { "iacute", 237 },
 147         { "icirc", 238 },
 148         { "iexcl", 161 },
 149         { "igrave", 236 },
 150         { "iquest", 191 },
 151         { "iuml", 239 },
 152         { "laquo", 171 },
 153         { "macr", 175 },
 154         { "micro", 181 },
 155         { "middot", 183 },
 156         { "nbsp", 160 },
 157         { "not", 172 },
 158         { "ntilde", 241 },
 159         { "oacute", 243 },
 160         { "ocirc", 244 },
 161         { "ograve", 242 },
 162         { "ordf", 170 },
 163         { "ordm", 186 },
 164         { "oslash", 248 },
 165         { "otilde", 245 },
 166         { "ouml", 246 },
 167         { "para", 182 },
 168         { "plusmn", 177 },
 169         { "pound", 163 },
 170         { "raquo", 187 },
 171         { "reg", 174 },
 172         { "sect", 167 },
 173         { "shy", 173 },
 174         { "sup1", 185 },
 175         { "sup2", 178 },
 176         { "sup3", 179 },
 177         { "szlig", 223 },
 178         { "thorn", 254 },
 179         { "times", 215 },
 180         { "uacute", 250 },
 181         { "ucirc", 251 },
 182         { "ugrave", 249 },
 183         { "uml", 168 },
 184         { "uuml", 252 },
 185         { "yacute", 253 },
 186         { "yen", 165 },
 187         { "yuml", 255 },
 188 // iso8859-1 only for now       { "OElig", 338 },
 189 // ditto                        { "oelig", 339 },
 190         { NULL, 0 }
 191     };
 192     if (named_ents.empty()) {
 193         struct ent *i = ents;
 194         while (i->n) {
 195             named_ents[string(i->n)] = i->v;
 196             ++i;
 197         }
 198     }
 199 }
 200
 201 void
 202 HtmlParser::decode_entities(string &s)
 203 {
 204     // We need a const_iterator version of s.end() - otherwise the
 205     // find() and find_if() templates don't work...
 206     string::const_iterator amp = s.begin(), s_end = s.end();
 207     while ((amp = find(amp, s_end, '&')) != s_end) {
 208         unsigned int val = 0;
 209         string::const_iterator end, p = amp + 1;
 210         if (p != s_end && *p == '#') {
 211             p++;
 212             if (p != s_end && tolower(*p) == 'x') {
 213                 // hex
 214                 p++;
 215                 end = find_if(p, s_end, p_notxdigit);
 216                 sscanf(s.substr(p - s.begin(), end - p).c_str(), "%x", &val);
 217             } else {
 218                 // number
 219                 end = find_if(p, s_end, p_notdigit);
 220                 val = atoi(s.substr(p - s.begin(), end - p).c_str());
 221             }
 222         } else {
 223             end = find_if(p, s_end, p_notalnum);
 224             string code = s.substr(p - s.begin(), end - p);
 225             map<string, unsigned int>::const_iterator i;
 226             i = named_ents.find(code);
 227             if (i != named_ents.end()) val = i->second;
 228         }
 229         if (end < s_end && *end == ';') end++;
 230         if (val) {
 231             string::size_type amp_pos = amp - s.begin();
 232             s.replace(amp_pos, end - amp, 1u, char(val));
 233             s_end = s.end();
 234             // We've modified the string, so the iterators are no longer
 235             // valid...
 236             amp = s.begin() + amp_pos + 1;
 237         } else {
 238             amp = end;
 239         }
 240     }
 241 }
 242
 243 void
 244 HtmlParser::parse_html(const string &body)
 245 {
 246     map<string,string> Param;
 247     string::const_iterator start = body.begin();
 248
 249     while (1) {
 250
 251         // Skip through until we find an HTML tag, a comment, or the end of
 252         // document.  Ignore isolated occurences of `<' which don't start
 253         // a tag or comment
 254         string::const_iterator p = start;
 255         while (1) {
 256
 257             p = find(p, body.end(), '<');
 258             if (p == body.end()) break;
 259             char ch = *(p + 1);
 260             // tag, closing tag, comment (or SGML declaration), or PHP
 261             if (isalpha(ch) || ch == '/' || ch == '!' || ch == '?') break;
 262             p++;
 263         }
 264
 265
 266         // process text up to start of tag
 267         if (p > start) {
 268             string text = body.substr(start - body.begin(), p - start);
 269             decode_entities(text);
 270             process_text(text);
 271         }
 272
 273         if (p == body.end()) break;
 274
 275         start = p + 1;
 276
 277         if (start == body.end()) break;
 278
 279         if (*start == '!') {
 280             if (++start == body.end()) break;
 281             if (++start == body.end()) break;
 282             // comment or SGML declaration
 283             if (*(start - 1) == '-' && *start == '-') {
 284                 start = find(start + 1, body.end(), '>');
 285                 // unterminated comment swallows rest of document
 286                 // (like NS, but unlike MSIE iirc)
 287                 if (start == body.end()) break;
 288
 289                 p = start;
 290                 // look for -->
 291                 while (p != body.end() && (*(p - 1) != '-' || *(p - 2) != '-'))
 292                     p = find(p + 1, body.end(), '>');
 293
 294                 // If we found --> skip to there, otherwise
 295                 // skip to the first > we found (as Netscape does)
 296                 if (p != body.end()) start = p;
 297             } else {
 298                 // just an SGML declaration, perhaps giving the DTD - ignore it
 299                 start = find(start - 1, body.end(), '>');
 300                 if (start == body.end()) break;
 301             }
 302             ++start;
 303         } else if (*start == '?') {
 304             if (++start == body.end()) break;
 305             // PHP - swallow until ?> or EOF
 306             start = find(start + 1, body.end(), '>');
 307
 308             // look for ?>
 309             while (start != body.end() && *(start - 1) != '?')
 310                 start = find(start + 1, body.end(), '>');
 311
 312             // unterminated PHP swallows rest of document (rather arbitrarily
 313             // but it avoids polluting the database when things go wrong)
 314             if (start != body.end()) ++start;
 315         } else {
 316             // opening or closing tag
 317             int closing = 0;
 318
 319             if (*start == '/') {
 320                 closing = 1;
 321                 start = find_if(start + 1, body.end(), p_notwhitespace);
 322             }
 323
 324             p = start;
 325             start = find_if(start, body.end(), p_nottag);
 326
 327             string tag = body.substr(p - body.begin(), start - p);
 328
 329             // convert tagname to lowercase
 330             for (string::iterator i = tag.begin(); i != tag.end(); i++)
 331                 *i = tolower(*i);
 332
 333             if (closing) {
 334
 335                                  closing_tag(tag);
 336
 337                                  /* ignore any bogus parameters on closing tags */
 338                                  p = find(start, body.end(), '>');
 339                                  if (p == body.end()) break;
 340                                  start = p + 1;
 341             } else {
 342
 343                 while (start < body.end() && *start != '>') {
 344                     string name, value;
 345
 346                     p = find_if(start, body.end(), p_whitespaceeqgt);
 347
 348                     name = body.substr(start - body.begin(), p - start);
 349
 350                     p = find_if(p, body.end(), p_notwhitespace);
 351
 352                     start = p;
 353                     if (start != body.end() && *start == '=') {
 354                         int quote;
 355
 356                         start = find_if(start + 1, body.end(), p_notwhitespace);
 357
 358                         p = body.end();
 359
 360                         quote = *start;
 361                         if (quote == '"' || quote == '\'') {
 362                             start++;
 363                             p = find(start, body.end(), quote);
 364                         }
 365
 366                         if (p == body.end()) {
 367                             // unquoted or no closing quote
 368                             p = find_if(start, body.end(), p_whitespacegt);
 369
 370                             value = body.substr(start - body.begin(), p - start);
 371
 372                             start = find_if(p, body.end(), p_notwhitespace);
 373                         } else {
 374                             value = body.substr(start - body.begin(), p - start);
 375                         }
 376
 377                         if (name.size()) {
 378
 379                             // convert parameter name to lowercase
 380                             string::iterator i;
 381                             for (i = name.begin(); i != name.end(); i++)
 382                                 *i = tolower(*i);
 383                             // in case of multiple entries, use the first
 384                             // (as Netscape does)
 385                             if (Param.find(name) == Param.end())
 386                                 Param[name] = value;
 387                         }
 388                     }
 389                 }
 390                 opening_tag(tag, Param);
 391                 Param.clear();
 392
 393                 if (start != body.end() && *start == '>') ++start;
 394             }
 395         }
 396     }
 397 }