filter.c

   1 /*@ S-nail - a mail user agent derived from Berkeley Mail.
   2  *@ Filter objects.
   3  *
   4  * Copyright (c) 2013 - 2015 Steffen (Daode) Nurpmeso <sdaoden@users.sf.net>.
   5  *
   6  * Permission to use, copy, modify, and/or distribute this software for any
   7  * purpose with or without fee is hereby granted, provided that the above
   8  * copyright notice and this permission notice appear in all copies.
   9  *
  10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  17  */
  18 #undef n_FILE
  19 #define n_FILE filter
  20
  21 #ifndef HAVE_AMALGAMATION
  22 # include "nail.h"
  23 #endif
  24
  25 /*
  26  * Quotation filter
  27  */
  28
  29 /*
  30  * TODO quotation filter: anticipate in future data: don't break if only WS
  31  * TODO or a LF escaping \ follows on the line (simply reuse the latter).
  32  */
  33
  34 #ifdef HAVE_QUOTE_FOLD
  35 CTA(QUOTE_MAX > 3);
  36
  37 enum qf_state {
  38    _QF_CLEAN,
  39    _QF_PREFIX,
  40    _QF_DATA
  41 };
  42
  43 struct qf_vc {
  44    struct quoteflt   *self;
  45    char const        *buf;
  46    size_t            len;
  47 };
  48
  49 /* Print out prefix and current quote */
  50 static ssize_t _qf_dump_prefix(struct quoteflt *self);
  51
  52 /* Add one data character */
  53 static ssize_t _qf_add_data(struct quoteflt *self, wchar_t wc);
  54
  55 /* State machine handlers */
  56 static ssize_t _qf_state_prefix(struct qf_vc *vc);
  57 static ssize_t _qf_state_data(struct qf_vc *vc);
  58
  59 static ssize_t
  60 _qf_dump_prefix(struct quoteflt *self)
  61 {
  62    ssize_t rv;
  63    size_t i;
  64    NYD_ENTER;
  65
  66    if ((i = self->qf_pfix_len) > 0 && i != fwrite(self->qf_pfix, 1, i,
  67          self->qf_os))
  68       goto jerr;
  69    rv = i;
  70
  71    if ((i = self->qf_currq.l) > 0 && i != fwrite(self->qf_currq.s, 1, i,
  72          self->qf_os))
  73       goto jerr;
  74    rv += i;
  75 jleave:
  76    NYD_LEAVE;
  77    return rv;
  78 jerr:
  79    rv = -1;
  80    goto jleave;
  81 }
  82
  83 static ssize_t
  84 _qf_add_data(struct quoteflt *self, wchar_t wc)
  85 {
  86    char *save_b;
  87    ui32_t save_l, save_w;
  88    ssize_t rv = 0;
  89    int w, l;
  90    NYD_ENTER;
  91
  92    save_l = save_w = 0; /* silence cc */
  93    save_b = NULL;
  94    /* <newline> ends state */
  95    if (wc == L'\n')
  96       goto jflush;
  97    if (wc == L'\r') /* TODO CR should be stripped in lower level!! */
  98       goto jleave;
  99
 100    /* Unroll <tab> to spaces */
 101    if (wc == L'\t') {
 102       save_l = self->qf_datw;
 103       save_w = (save_l + QUOTE_TAB_SPACES) & ~(QUOTE_TAB_SPACES - 1);
 104       save_w -= save_l;
 105       while (save_w-- > 0) {
 106          ssize_t j = _qf_add_data(self, L' ');
 107          if (j < 0) {
 108             rv = j;
 109             break;
 110          }
 111          rv += j;
 112       }
 113       goto jleave;
 114    }
 115
 116    w = wcwidth(wc);
 117    if (w == -1) {
 118 jbad:
 119       ++self->qf_datw;
 120       self->qf_dat.s[self->qf_dat.l++] = '?';
 121    } else {
 122       l = wctomb(self->qf_dat.s + self->qf_dat.l, wc);
 123       if (l < 0)
 124          goto jbad;
 125       self->qf_datw += (ui32_t)w;
 126       self->qf_dat.l += (size_t)l;
 127    }
 128
 129    /* TODO The last visual may excess (adjusted!) *qfold-max* if it's a wide;
 130     * TODO place it on the next line, break before */
 131    if (self->qf_datw >= self->qf_qfold_max) {
 132       /* If we have seen a nice breakpoint during traversal, shuffle data
 133        * around a bit so as to restore the trailing part after flushing */
 134       if (self->qf_brkl > 0) {
 135          save_w = self->qf_datw - self->qf_brkw;
 136          save_l = self->qf_dat.l - self->qf_brkl;
 137          save_b = self->qf_dat.s + self->qf_brkl + 2;
 138          memmove(save_b, save_b - 2, save_l);
 139          self->qf_dat.l = self->qf_brkl;
 140       }
 141
 142       self->qf_dat.s[self->qf_dat.l++] = '\\';
 143 jflush:
 144       self->qf_dat.s[self->qf_dat.l++] = '\n';
 145       rv = quoteflt_flush(self);
 146
 147       /* Restore takeovers, if any */
 148       if (save_b != NULL) {
 149          self->qf_brk_isws = FAL0;
 150          self->qf_datw += save_w;
 151          self->qf_dat.l = save_l;
 152          memmove(self->qf_dat.s, save_b, save_l);
 153       }
 154    } else if (self->qf_datw >= self->qf_qfold_min && !self->qf_brk_isws) {
 155       bool_t isws = iswspace(wc);
 156
 157       if ((isws && !self->qf_brk_isws) || self->qf_brkl == 0) {
 158          self->qf_brkl = self->qf_dat.l;
 159          self->qf_brkw = self->qf_datw;
 160          self->qf_brk_isws = isws;
 161       }
 162    }
 163
 164    /* If state changed to prefix, perform full reset (note this implies that
 165     * quoteflt_flush() performs too much work..) */
 166    if (wc == '\n') {
 167       self->qf_state = _QF_PREFIX;
 168       self->qf_wscnt = self->qf_datw = 0;
 169       self->qf_currq.l = 0;
 170    }
 171 jleave:
 172    NYD_LEAVE;
 173    return rv;
 174 }
 175
 176 static ssize_t
 177 _qf_state_prefix(struct qf_vc *vc)
 178 {
 179    struct quoteflt *self;
 180    ssize_t rv;
 181    char const *buf;
 182    size_t len, i;
 183    wchar_t wc;
 184    NYD_ENTER;
 185
 186    self = vc->self;
 187    rv = 0;
 188
 189    for (buf = vc->buf, len = vc->len; len > 0;) {
 190       /* xxx NULL BYTE! */
 191       i = mbrtowc(&wc, buf, len, self->qf_mbps);
 192       if (i == (size_t)-1) {
 193          /* On hard error, don't modify mbstate_t and step one byte */
 194          self->qf_mbps[0] = self->qf_mbps[1];
 195          ++buf;
 196          --len;
 197          self->qf_wscnt = 0;
 198          continue;
 199       }
 200       self->qf_mbps[1] = self->qf_mbps[0];
 201       if (i == (size_t)-2) {
 202          /* Redundant shift sequence, out of buffer */
 203          len = 0;
 204          break;
 205       }
 206       buf += i;
 207       len -= i;
 208
 209       if (wc == L'\n')
 210          goto jfin;
 211       if (iswspace(wc)) {
 212          ++self->qf_wscnt;
 213          continue;
 214       }
 215       if (i == 1 && ISQUOTE(wc)) {
 216          self->qf_wscnt = 0;
 217          if (self->qf_currq.l >= QUOTE_MAX - 3) {
 218             self->qf_currq.s[QUOTE_MAX - 3] = '.';
 219             self->qf_currq.s[QUOTE_MAX - 2] = '.';
 220             self->qf_currq.s[QUOTE_MAX - 1] = '.';
 221             self->qf_currq.l = QUOTE_MAX;
 222          } else
 223             self->qf_currq.s[self->qf_currq.l++] = buf[-1];
 224          continue;
 225       }
 226
 227       /* The quote is parsed and compressed; dump it */
 228 jfin:
 229       self->qf_state = _QF_DATA;
 230       /* Overtake WS to the current quote in order to preserve it for eventual
 231        * necessary follow lines, too */
 232       /* TODO we de-facto "normalize" to ASCII SP here which MESSES tabs!! */
 233       while (self->qf_wscnt-- > 0 && self->qf_currq.l < QUOTE_MAX)
 234          self->qf_currq.s[self->qf_currq.l++] = ' ';
 235       self->qf_datw = self->qf_pfix_len + self->qf_currq.l;
 236       self->qf_wscnt = 0;
 237       rv = _qf_add_data(self, wc);
 238       break;
 239    }
 240
 241    vc->buf = buf;
 242    vc->len = len;
 243    NYD_LEAVE;
 244    return rv;
 245 }
 246
 247 static ssize_t
 248 _qf_state_data(struct qf_vc *vc)
 249 {
 250    struct quoteflt *self;
 251    ssize_t rv;
 252    char const *buf;
 253    size_t len, i;
 254    wchar_t wc;
 255    NYD_ENTER;
 256
 257    self = vc->self;
 258    rv = 0;
 259
 260    for (buf = vc->buf, len = vc->len; len > 0;) {
 261       /* xxx NULL BYTE! */
 262       i = mbrtowc(&wc, buf, len, self->qf_mbps);
 263       if (i == (size_t)-1) {
 264          /* On hard error, don't modify mbstate_t and step one byte */
 265          self->qf_mbps[0] = self->qf_mbps[1];
 266          ++buf;
 267          --len;
 268          continue;
 269       }
 270       self->qf_mbps[1] = self->qf_mbps[0];
 271       if (i == (size_t)-2) {
 272          /* Redundant shift sequence, out of buffer */
 273          len = 0;
 274          break;
 275       }
 276       buf += i;
 277       len -= i;
 278
 279       {  ssize_t j = _qf_add_data(self, wc);
 280          if (j < 0) {
 281             rv = j;
 282             break;
 283          }
 284          rv += j;
 285       }
 286
 287       if (self->qf_state != _QF_DATA)
 288          break;
 289    }
 290
 291    vc->buf = buf;
 292    vc->len = len;
 293    NYD_LEAVE;
 294    return rv;
 295 }
 296 #endif /* HAVE_QUOTE_FOLD */
 297
 298 FL struct quoteflt *
 299 quoteflt_dummy(void) /* TODO LEGACY (until filters are plugged when needed) */
 300 {
 301    static struct quoteflt qf_i;
 302
 303    return &qf_i;
 304 }
 305
 306 FL void
 307 quoteflt_init(struct quoteflt *self, char const *prefix)
 308 {
 309 #ifdef HAVE_QUOTE_FOLD
 310    char *xcp, *cp;
 311 #endif
 312    NYD_ENTER;
 313
 314    memset(self, 0, sizeof *self);
 315
 316    if ((self->qf_pfix = prefix) != NULL)
 317       self->qf_pfix_len = (ui32_t)strlen(prefix);
 318
 319    /* Check wether the user wants the more fancy quoting algorithm */
 320    /* TODO *quote-fold*: QUOTE_MAX may excess it! */
 321 #ifdef HAVE_QUOTE_FOLD
 322    if (self->qf_pfix_len > 0 && (cp = ok_vlook(quote_fold)) != NULL) {
 323       ui32_t qmin, qmax = (ui32_t)strtol(cp, &xcp, 10);
 324       /* These magic values ensure we don't bail :) */
 325       if (qmax < self->qf_pfix_len + 6)
 326          qmax = self->qf_pfix_len + 6;
 327       --qmax; /* The newline escape */
 328       if (cp == xcp || *xcp == '\0')
 329          qmin = (qmax >> 1) + (qmax >> 2) + (qmax >> 5);
 330       else {
 331          qmin = (ui32_t)strtol(xcp + 1, NULL, 10);
 332          if (qmin < qmax >> 1)
 333             qmin = qmax >> 1;
 334          else if (qmin > qmax - 2)
 335             qmin = qmax - 2;
 336       }
 337       self->qf_qfold_min = qmin;
 338       self->qf_qfold_max = qmax;
 339
 340       /* Add pad for takeover copies, backslash and newline */
 341       self->qf_dat.s = salloc((qmax + 3) * mb_cur_max);
 342       self->qf_currq.s = salloc((QUOTE_MAX + 1) * mb_cur_max);
 343    }
 344 #endif
 345    NYD_LEAVE;
 346 }
 347
 348 FL void
 349 quoteflt_destroy(struct quoteflt *self) /* xxx inline */
 350 {
 351    NYD_ENTER;
 352    UNUSED(self);
 353    NYD_LEAVE;
 354 }
 355
 356 FL void
 357 quoteflt_reset(struct quoteflt *self, FILE *f) /* xxx inline */
 358 {
 359    NYD_ENTER;
 360    self->qf_os = f;
 361 #ifdef HAVE_QUOTE_FOLD
 362    self->qf_state = _QF_CLEAN;
 363    self->qf_dat.l =
 364    self->qf_currq.l = 0;
 365    memset(self->qf_mbps, 0, sizeof self->qf_mbps);
 366 #endif
 367    NYD_LEAVE;
 368 }
 369
 370 FL ssize_t
 371 quoteflt_push(struct quoteflt *self, char const *dat, size_t len)
 372 {
 373    /* (xxx Ideally the actual push() [and flush()] would be functions on their
 374     * xxx own, via indirect vtbl call ..) */
 375    ssize_t rv = 0;
 376    NYD_ENTER;
 377
 378    if (len == 0)
 379       goto jleave;
 380
 381    /* Bypass? XXX Finally, this filter simply should not be used, then */
 382    if (self->qf_pfix_len == 0) {
 383       if (len != fwrite(dat, 1, len, self->qf_os))
 384          goto jerr;
 385       rv = len;
 386    }
 387    /* Normal: place *indentprefix* at every BOL */
 388    else
 389 #ifdef HAVE_QUOTE_FOLD
 390       if (self->qf_qfold_max == 0)
 391 #endif
 392    {
 393       void *vp;
 394       size_t ll;
 395       bool_t pxok = (self->qf_qfold_min != 0);
 396
 397       for (;;) {
 398          if (!pxok) {
 399             ll = self->qf_pfix_len;
 400             if (ll != fwrite(self->qf_pfix, 1, ll, self->qf_os))
 401                goto jerr;
 402             rv += ll;
 403             pxok = TRU1;
 404          }
 405
 406          /* xxx Strictly speaking this is invalid, because only `/' and `.' are
 407           * xxx mandated by POSIX.1-2008 as "invariant across all locales
 408           * xxx supported"; though there is no charset known which uses this
 409           * xxx control char as part of a multibyte character; note that S-nail
 410           * XXX (and the Mail codebase as such) do not support EBCDIC */
 411          if ((vp = memchr(dat, '\n', len)) == NULL)
 412             ll = len;
 413          else {
 414             pxok = FAL0;
 415             ll = PTR2SIZE((char*)vp - dat) + 1;
 416          }
 417
 418          if (ll != fwrite(dat, sizeof *dat, ll, self->qf_os))
 419             goto jerr;
 420          rv += ll;
 421          if ((len -= ll) == 0)
 422             break;
 423          dat += ll;
 424       }
 425
 426       self->qf_qfold_min = pxok;
 427    }
 428    /* Overly complicated, though still only line-per-line: *quote-fold*.
 429     * - If .qf_currq.l is 0, then we are in a clean state.  Reset .qf_mbps;
 430     *   TODO note this means we assume that lines start with reset escape seq,
 431     *   TODO but i don't think this is any worse than what we currently do;
 432     *   TODO in 15.0, with the value carrier, we should carry conversion states
 433     *   TODO all along, only resetting on error (or at words for header =???=);
 434     *   TODO this still is weird for error handling, but we need to act more
 435     *   TODO stream-alike (though in practice i don't think cross-line states
 436     *   TODO can be found, because of compatibility reasons; however, being
 437     *   TODO a problem rather than a solution is not a good thing (tm))
 438     * - Lookout for a newline */
 439 #ifdef HAVE_QUOTE_FOLD
 440    else {
 441       struct qf_vc vc;
 442       ssize_t i;
 443
 444       vc.self = self;
 445       vc.buf = dat;
 446       vc.len = len;
 447       while (vc.len > 0) {
 448          switch (self->qf_state) {
 449          case _QF_CLEAN:
 450          case _QF_PREFIX:
 451             i = _qf_state_prefix(&vc);
 452             break;
 453          default: /* silence cc (`i' unused) */
 454          case _QF_DATA:
 455             i = _qf_state_data(&vc);
 456             break;
 457          }
 458          if (i < 0)
 459             goto jerr;
 460          rv += i;
 461       }
 462    }
 463 #endif /* HAVE_QUOTE_FOLD */
 464
 465 jleave:
 466    NYD_LEAVE;
 467    return rv;
 468 jerr:
 469    rv = -1;
 470    goto jleave;
 471 }
 472
 473 FL ssize_t
 474 quoteflt_flush(struct quoteflt *self)
 475 {
 476    ssize_t rv = 0;
 477    NYD_ENTER;
 478    UNUSED(self);
 479
 480 #ifdef HAVE_QUOTE_FOLD
 481    if (self->qf_dat.l > 0) {
 482       rv = _qf_dump_prefix(self);
 483       if (rv >= 0) {
 484          size_t i = self->qf_dat.l;
 485          if (i == fwrite(self->qf_dat.s, 1, i, self->qf_os))
 486             rv += i;
 487          else
 488             rv = -1;
 489          self->qf_dat.l = 0;
 490          self->qf_brk_isws = FAL0;
 491          self->qf_wscnt = self->qf_brkl = self->qf_brkw = 0;
 492          self->qf_datw = self->qf_pfix_len + self->qf_currq.l;
 493       }
 494    }
 495 #endif
 496    NYD_LEAVE;
 497    return rv;
 498 }
 499
 500 /*
 501  * HTML tagsoup filter
 502  * TODO . Numeric &#NO; entities should also be treated by struct hf_ent
 503  * TODO . Yes, we COULD support CSS based quoting when we'd check type="quote"
 504  * TODO   (nonstandard) and watch out for style="gmail_quote" (or so, VERY
 505  * TODO   nonstandard) and tracking a stack of such elements (to be popped
 506  * TODO   once the closing element is seen).  Then, after writing a newline,
 507  * TODO   place sizeof(stack) ">"s first.  But aren't these HTML mails rude?
 508  * TODO Interlocking and non-well-formed data will break us down
 509  */
 510 #ifdef HAVE_FILTER_HTML_TAGSOUP
 511
 512 enum hf_limits {
 513    _HF_MINLEN  = 10,       /* Minimum line length (can't really be smaller) */
 514    _HF_BRKSUB  = 8         /* Start considering line break MAX - BRKSUB */
 515 };
 516
 517 enum hf_flags {
 518    _HF_UTF8    = 1<<0,     /* Data is in UTF-8 */
 519    _HF_ERROR   = 1<<1,     /* A hard error occurred, bail as soon as possible */
 520    _HF_NOPUT   = 1<<2,     /* (In a tag,) Don't generate output */
 521    _HF_IGN     = 1<<3,     /* Ignore mode on */
 522    _HF_ANY     = 1<<4,     /* Yet seen just any output */
 523    _HF_PRE     = 1<<5,     /* In <pre>formatted mode */
 524    _HF_ENT     = 1<<6,     /* Currently parsing an entity */
 525    _HF_BLANK   = 1<<7,     /* Whitespace last */
 526    _HF_HREF    = 1<<8,     /* External <a href=> was the last href seen */
 527
 528    _HF_NL_1    = 1<<9,     /* One \n seen */
 529    _HF_NL_2    = 2<<9,     /* We have produced an all empty line */
 530    _HF_NL_MASK = _HF_NL_1 | _HF_NL_2
 531 };
 532
 533 enum hf_special_actions {
 534    _HFSA_NEEDSEP  = -1,    /* Need an empty line (paragraph separator) */
 535    _HFSA_NEEDNL   = -2,    /* Need a new line start (table row) */
 536    _HFSA_IGN      = -3,    /* Things like <style>..</style>, <script>.. */
 537    _HFSA_PRE      = -4,    /* <pre>.. */
 538    _HFSA_PRE_END  = -5,
 539    _HFSA_IMG      = -6,    /* <img> */
 540    _HFSA_HREF     = -7,    /* <a>.. */
 541    _HFSA_HREF_END = -8
 542 };
 543
 544 enum hf_entity_flags {
 545    _HFE_HAVE_UNI  = 1<<6,  /* Have a Unicode replacement character */
 546    _HFE_HAVE_CSTR = 1<<7,  /* Have a string replacement */
 547    /* We store the length of the entity name in the flags, too */
 548    _HFE_LENGTH_MASK = (1<<6) - 1
 549 };
 550
 551 struct htmlflt_href {
 552    struct htmlflt_href *hfh_next;
 553    ui32_t      hfh_no;     /* Running sequence */
 554    ui32_t      hfh_len;    /* of .hfh_dat */
 555    char        hfh_dat[VFIELD_SIZE(0)];
 556 };
 557
 558 struct htmlflt_tag {
 559    si32_t      hft_act;    /* char or hf_special_actions */
 560    /* Not NUL: character to inject, with high bit set: place a space
 561     * afterwards.  Note: only recognized with _HFSA_NEEDSEP or _HFSA_NEEDNL */
 562    char        hft_injc;
 563    ui8_t       hft_len;    /* Useful bytes in (NUL terminated) .hft_tag */
 564    char const  hft_tag[10]; /* Tag less < and > surroundings (TR, /TR, ..) */
 565 };
 566 CTA(SIZEOF_FIELD(struct htmlflt_tag, hft_tag) < LINESIZE); /* .hf_ign_tag */
 567
 568 struct hf_ent {
 569    ui8_t       hfe_flags;  /* enum hf_entity_flags plus length of .hfe_ent */
 570    char        hfe_c;      /* Plain replacement character */
 571    ui16_t      hfe_uni;    /* Unicode codepoint if _HFE_HAVE_UNI */
 572    char        hfe_cstr[5]; /* _HFE_HAVE_CSTR (e.g., &hellip; -> ...) */
 573    char const  hfe_ent[7]; /* Entity less & and ; surroundings */
 574 };
 575
 576 /* Tag list; not binary searched :(, so try to take care a bit */
 577 static struct htmlflt_tag const  _hf_tags[] = {
 578 # undef _X
 579 # undef _XC
 580 # define _X(S,A)     {A, '\0', sizeof(S) -1, S "\0"}
 581 # define _XC(S,C,A)  {A, C, sizeof(S) -1, S "\0"}
 582
 583    _X("P", _HFSA_NEEDSEP),       _X("/P", _HFSA_NEEDNL),
 584    _X("DIV", _HFSA_NEEDSEP),     _X("/DIV", _HFSA_NEEDNL),
 585    _X("TR", _HFSA_NEEDNL),
 586                                  _X("/TH", '\t'),
 587                                  _X("/TD", '\t'),
 588    /* Let it stand out; also since we don't support implicit paragraphs after
 589     * block elements, plain running text after a list (seen in Unicode
 590     * announcement via Firefox) */
 591    _X("UL", _HFSA_NEEDSEP),      _X("/UL", _HFSA_NEEDSEP),
 592    _XC("LI", (char)0x80 | '*', _HFSA_NEEDSEP),
 593    _X("DL", _HFSA_NEEDSEP),
 594    _X("DT", _HFSA_NEEDNL),
 595
 596    _X("A", _HFSA_HREF),          _X("/A", _HFSA_HREF_END),
 597    _X("IMG", _HFSA_IMG),
 598    _X("BR", '\n'),
 599    _X("PRE", _HFSA_PRE),         _X("/PRE", _HFSA_PRE_END),
 600    _X("TITLE", _HFSA_NEEDSEP),   /*_X("/TITLE", '\n'),*/
 601    _X("H1", _HFSA_NEEDSEP),      /*_X("/H1", '\n'),*/
 602    _X("H2", _HFSA_NEEDSEP),      /*_X("/H2", '\n'),*/
 603    _X("H3", _HFSA_NEEDSEP),      /*_X("/H3", '\n'),*/
 604    _X("H4", _HFSA_NEEDSEP),      /*_X("/H4", '\n'),*/
 605    _X("H5", _HFSA_NEEDSEP),      /*_X("/H5", '\n'),*/
 606    _X("H6", _HFSA_NEEDSEP),      /*_X("/H6", '\n'),*/
 607
 608    _X("STYLE", _HFSA_IGN),
 609    _X("SCRIPT", _HFSA_IGN),
 610
 611 # undef _X
 612 };
 613
 614 /* Entity list; not binary searched.. */
 615 static struct hf_ent const       _hf_ents[] = {
 616 # undef _X
 617 # undef _XU
 618 # undef _XS
 619 # undef _XUS
 620 # define _X(E,C)     {(sizeof(E) -1), C, 0x0u, "", E "\0"}
 621 # define _XU(E,C,U)  {(sizeof(E) -1) | _HFE_HAVE_UNI, C, U, "", E "\0"}
 622 # define _XS(E,S)    {(sizeof(E) -1) | _HFE_HAVE_CSTR, '\0', 0x0u,S "\0",E "\0"}
 623 # define _XSU(E,S,U) \
 624    {(sizeof(E) -1) | _HFE_HAVE_UNI | _HFE_HAVE_CSTR, '\0', U, S "\0", E "\0"}
 625
 626    _X("quot", '"'),
 627    _X("amp", '&'),
 628    _X("lt", '<'),                _X("gt", '>'),
 629
 630    _XU("nbsp", ' ', 0x0020 /* Note: not 0x00A0 seems to be better for us */),
 631    _XU("middot", '.', 0x00B7),
 632    _XSU("hellip", "...", 0x2026),
 633    _XSU("mdash", "---", 0x2014), _XSU("ndash", "--", 0x2013),
 634    _XSU("laquo", "<<", 0x00AB),  _XSU("raquo", ">>", 0x00BB),
 635    _XSU("lsaquo", "<", 0x2039),  _XSU("rsaquo", ">", 0x203A),
 636    _XSU("lsquo", "'", 0x2018),   _XSU("rsquo", "'", 0x2019),
 637    _XSU("ldquo", "\"", 0x201C),  _XSU("rdquo", "\"", 0x201D),
 638    _XSU("uarr", "^|", 0x2191),   _XSU("darr", "|v", 0x2193),
 639
 640    _XSU("cent", "CENT", 0x00A2),
 641    _XSU("copy", "(C)", 0x00A9),
 642    _XSU("euro", "EUR", 0x20AC),
 643    _XSU("infin", "INFY", 0x221E),
 644    _XSU("pound", "GBP", 0x00A3),
 645    _XSU("reg", "(R)", 0x00AE),
 646    _XSU("sect", "S:", 0x00A7),
 647    _XSU("yen", "JPY", 0x00A5),
 648
 649    /* German umlauts */
 650    _XSU("Auml", "Ae", 0x00C4),   _XSU("auml", "ae", 0x00E4),
 651    _XSU("Ouml", "Oe", 0x00D6),   _XSU("ouml", "oe", 0x00F6),
 652    _XSU("Uuml", "Ue", 0x00DC),   _XSU("uuml", "ue", 0x00FC),
 653    _XSU("szlig", "ss", 0x00DF)
 654
 655 # undef _X
 656 # undef _XU
 657 # undef _XS
 658 # undef _XSU
 659 };
 660
 661 /* Real output */
 662 static struct htmlflt * _hf_dump_hrefs(struct htmlflt *self);
 663 static struct htmlflt * _hf_dump(struct htmlflt *self);
 664 static struct htmlflt * _hf_store(struct htmlflt *self, char c);
 665 # ifdef HAVE_NATCH_CHAR
 666 static struct htmlflt * __hf_sync_mbstuff(struct htmlflt *self);
 667 # endif
 668
 669 /* Virtual output */
 670 static struct htmlflt * _hf_nl(struct htmlflt *self);
 671 static struct htmlflt * _hf_nl_force(struct htmlflt *self);
 672 static struct htmlflt * _hf_putc(struct htmlflt *self, char c);
 673 static struct htmlflt * _hf_putc_premode(struct htmlflt *self, char c);
 674 static struct htmlflt * _hf_puts(struct htmlflt *self, char const *cp);
 675 static struct htmlflt * _hf_putbuf(struct htmlflt *self,
 676                            char const *cp, size_t len);
 677
 678 /* Try to locate a param'eter in >hf_bdat, store it (non-terminated!) or NULL */
 679 static struct htmlflt * _hf_param(struct htmlflt *self, struct str *store,
 680                            char const *param);
 681
 682 /* Expand all entities in the given parameter */
 683 static struct htmlflt * _hf_expand_all_ents(struct htmlflt *self,
 684                            struct str const *param);
 685
 686 /* Completely parsed over a tag / an entity, interpret that */
 687 static struct htmlflt * _hf_check_tag(struct htmlflt *self, char const *s);
 688 static struct htmlflt * _hf_check_ent(struct htmlflt *self, char const *s,
 689                            size_t l);
 690
 691 /* Input handler */
 692 static ssize_t          _hf_add_data(struct htmlflt *self,
 693                            char const *dat, size_t len);
 694
 695 static struct htmlflt *
 696 _hf_dump_hrefs(struct htmlflt *self)
 697 {
 698    struct htmlflt_href *hhp;
 699    NYD2_ENTER;
 700
 701    if (!(self->hf_flags & _HF_NL_2) && putc('\n', self->hf_os) == EOF) {
 702       self->hf_flags |= _HF_ERROR;
 703       goto jleave;
 704    }
 705
 706    /* Reverse the list */
 707    for (hhp = self->hf_hrefs, self->hf_hrefs = NULL; hhp != NULL;) {
 708       struct htmlflt_href *tmp = hhp->hfh_next;
 709       hhp->hfh_next = self->hf_hrefs;
 710       self->hf_hrefs = hhp;
 711       hhp = tmp;
 712    }
 713
 714    /* Then dump it */
 715    while ((hhp = self->hf_hrefs) != NULL) {
 716       self->hf_hrefs = hhp->hfh_next;
 717
 718       if (!(self->hf_flags & _HF_ERROR)) {
 719          int w = fprintf(self->hf_os, "  [%u] %.*s\n",
 720                hhp->hfh_no, (int)hhp->hfh_len, hhp->hfh_dat);
 721          if (w < 0)
 722             self->hf_flags |= _HF_ERROR;
 723       }
 724       free(hhp);
 725    }
 726
 727    self->hf_flags |= (putc('\n', self->hf_os) == EOF)
 728          ?  _HF_ERROR : _HF_NL_1 | _HF_NL_2;
 729    self->hf_href_dist = (ui32_t)realscreenheight >> 1;
 730 jleave:
 731    NYD2_LEAVE;
 732    return self;
 733 }
 734
 735 static struct htmlflt *
 736 _hf_dump(struct htmlflt *self)
 737 {
 738    ui32_t f, l;
 739    char c, *cp;
 740    NYD2_ENTER;
 741
 742    f = self->hf_flags & ~_HF_BLANK;
 743    l = self->hf_len;
 744    cp = self->hf_line;
 745    self->hf_mbwidth = self->hf_mboff = self->hf_last_ws = self->hf_len = 0;
 746
 747    for (c = '\0'; l > 0; --l) {
 748       c = *cp++;
 749 jput:
 750       if (putc(c, self->hf_os) == EOF) {
 751          self->hf_flags = (f |= _HF_ERROR);
 752          goto jleave;
 753       }
 754    }
 755
 756    if (c != '\n') {
 757       f |= (f & _HF_NL_1) ? _HF_NL_2 : _HF_NL_1;
 758       l = 1;
 759       c = '\n';
 760       goto jput;
 761    }
 762    self->hf_flags = f;
 763
 764    /* Check wether there are HREFs to dump; there is so much messy tagsoup out
 765     * there that it seems best not to simply dump HREFs in each _dump(), but
 766     * only with some gap, let's say half the real screen height */
 767    if (--self->hf_href_dist < 0 && (f & _HF_NL_2) && self->hf_hrefs != NULL)
 768       self = _hf_dump_hrefs(self);
 769 jleave:
 770    NYD2_LEAVE;
 771    return self;
 772 }
 773
 774 static struct htmlflt *
 775 _hf_store(struct htmlflt *self, char c)
 776 {
 777    ui32_t f, l, i;
 778    NYD2_ENTER;
 779
 780    assert(c != '\n');
 781
 782    f = self->hf_flags;
 783    l = self->hf_len;
 784    self->hf_line[l] = (c == '\t' ? ' ' : c);
 785    self->hf_len = ++l;
 786    if (blankspacechar(c)) {
 787       if (c == '\t') {
 788          i = 8 - ((l - 1) & 7); /* xxx magic tab width of 8 */
 789          if (i > 0) {
 790             do
 791                self = _hf_store(self, ' ');
 792             while (--i > 0);
 793             goto jleave;
 794          }
 795       }
 796       self->hf_last_ws = l;
 797    } else if (/*c == '.' ||*/ c == ',' || c == ';' || c == '-')
 798       self->hf_last_ws = l;
 799
 800    i = l;
 801 # ifdef HAVE_NATCH_CHAR /* XXX This code is really ridiculous! */
 802    if (mb_cur_max > 1) { /* XXX should mbrtowc() and THEN store, at least.. */
 803       wchar_t wc;
 804       int x = mbtowc(&wc, self->hf_line + self->hf_mboff, l - self->hf_mboff);
 805
 806       if (x > 0) {
 807          self->hf_mboff += x;
 808          if ((x = wcwidth(wc)) == -1)
 809             x = 1;
 810          else if (iswspace(wc))
 811             self->hf_last_ws = l;
 812          i = (self->hf_mbwidth += x);
 813       } else {
 814          if (x < 0) {
 815             mbtowc(&wc, NULL, mb_cur_max);
 816             if (UICMP(32, l - self->hf_mboff, >=, mb_cur_max)) { /* XXX */
 817                ++self->hf_mboff;
 818                ++self->hf_mbwidth;
 819             }
 820          }
 821          i = self->hf_mbwidth;
 822       }
 823    }
 824 # endif
 825
 826    /* Do we need to break the line? */
 827    if (i >= self->hf_lmax - _HF_BRKSUB) {
 828       ui32_t lim = self->hf_lmax >> 1;
 829
 830       /* Let's hope we saw a sane place to break this line! */
 831       if (self->hf_last_ws >= lim) {
 832 jput:
 833          i = self->hf_len = self->hf_last_ws;
 834          self = _hf_dump(self);
 835          if ((self->hf_len = (l -= i)) > 0) {
 836             self->hf_flags &= ~_HF_NL_MASK;
 837             memmove(self->hf_line, self->hf_line + i, l);
 838 # ifdef HAVE_NATCH_CHAR
 839             __hf_sync_mbstuff(self);
 840 # endif
 841          }
 842          goto jleave;
 843       }
 844
 845       /* Any 7-bit characters? */
 846       for (i = l; i-- >= lim;)
 847          if (asciichar((c = self->hf_line[i]))) {
 848             self->hf_last_ws = ++i;
 849             goto jput;
 850          } else if ((f & _HF_UTF8) && ((ui8_t)c & 0xC0) != 0x80) {
 851             self->hf_last_ws = i;
 852             goto jput;
 853          }
 854
 855       /* Hard break necessary!  xxx really badly done */
 856       if (l >= self->hf_lmax - 1)
 857          self = _hf_dump(self);
 858    }
 859 jleave:
 860    NYD2_LEAVE;
 861    return self;
 862 }
 863
 864 # ifdef HAVE_NATCH_CHAR
 865 static struct htmlflt *
 866 __hf_sync_mbstuff(struct htmlflt *self)
 867 {
 868    wchar_t wc;
 869    char const *b;
 870    ui32_t o, w, l;
 871    NYD2_ENTER;
 872
 873    b = self->hf_line;
 874    o = w = 0;
 875    l = self->hf_len;
 876    goto jumpin;
 877
 878    while (l > 0) {
 879       int x = mbtowc(&wc, b, l);
 880
 881       if (x == 0)
 882          break;
 883
 884       if (x > 0) {
 885          b += x;
 886          l -= x;
 887          o += x;
 888          if ((x = wcwidth(wc)) == -1)
 889             x = 1;
 890          w += x;
 891          continue;
 892       }
 893
 894       /* Bad, skip over a single character.. XXX very bad indeed */
 895       ++b;
 896       ++o;
 897       ++w;
 898       --l;
 899 jumpin:
 900       mbtowc(&wc, NULL, mb_cur_max);
 901    }
 902
 903    self->hf_mboff = o;
 904    self->hf_mbwidth = w;
 905
 906    NYD2_LEAVE;
 907    return self;
 908 }
 909 # endif /* HAVE_NATCH_CHAR */
 910
 911 static struct htmlflt *
 912 _hf_nl(struct htmlflt *self)
 913 {
 914    ui32_t f;
 915    NYD2_ENTER;
 916
 917    if (!((f = self->hf_flags) & _HF_ERROR)) {
 918       if (f & _HF_ANY) {
 919          if ((f & _HF_NL_MASK) != _HF_NL_MASK)
 920             self = _hf_dump(self);
 921       } else
 922          self->hf_flags = (f |= _HF_NL_MASK);
 923    }
 924    NYD2_LEAVE;
 925    return self;
 926 }
 927
 928 static struct htmlflt *
 929 _hf_nl_force(struct htmlflt *self)
 930 {
 931    NYD2_ENTER;
 932    if (!(self->hf_flags & _HF_ERROR))
 933       self = _hf_dump(self);
 934    NYD2_LEAVE;
 935    return self;
 936 }
 937
 938 static struct htmlflt *
 939 _hf_putc(struct htmlflt *self, char c)
 940 {
 941    ui32_t f;
 942    NYD2_ENTER;
 943
 944    if ((f = self->hf_flags) & _HF_ERROR)
 945       goto jleave;
 946
 947    if (c == '\n') {
 948       self = _hf_nl(self);
 949       goto jleave;
 950    } else if (c == ' ' || c == '\t') {
 951       if ((f & _HF_BLANK) || self->hf_len == 0)
 952          goto jleave;
 953       f |= _HF_BLANK;
 954    } else
 955       f &= ~_HF_BLANK;
 956    f &= ~_HF_NL_MASK;
 957    self->hf_flags = (f |= _HF_ANY);
 958    self = _hf_store(self, c);
 959 jleave:
 960    NYD2_LEAVE;
 961    return self;
 962 }
 963
 964 static struct htmlflt *
 965 _hf_putc_premode(struct htmlflt *self, char c)
 966 {
 967    ui32_t f;
 968    NYD2_ENTER;
 969
 970    if ((f = self->hf_flags) & _HF_ERROR) {
 971       ;
 972    } else if (c == '\n')
 973       self = _hf_nl_force(self);
 974    else {
 975       f &= ~_HF_NL_MASK;
 976       self->hf_flags = (f |= _HF_ANY);
 977       self = _hf_store(self, c);
 978    }
 979    NYD2_LEAVE;
 980    return self;
 981 }
 982
 983 static struct htmlflt *
 984 _hf_puts(struct htmlflt *self, char const *cp)
 985 {
 986    char c;
 987    NYD2_ENTER;
 988
 989    while ((c = *cp++) != '\0')
 990       self = _hf_putc(self, c);
 991    NYD2_LEAVE;
 992    return self;
 993 }
 994
 995 static struct htmlflt *
 996 _hf_putbuf(struct htmlflt *self, char const *cp, size_t len)
 997 {
 998    NYD2_ENTER;
 999
1000    while (len-- > 0)
1001       self = _hf_putc(self, *cp++);
1002    NYD2_LEAVE;
1003    return self;
1004 }
1005
1006 static struct htmlflt *
1007 _hf_param(struct htmlflt *self, struct str *store, char const *param)
1008 {
1009    char const *cp;
1010    char c, x, quote;
1011    size_t i;
1012    bool_t hot;
1013    NYD2_ENTER;
1014
1015    store->s = NULL;
1016    store->l = 0;
1017    cp = self->hf_bdat;
1018
1019    /* Skip over any non-WS first; be aware of soup, if it slipped through */
1020    for(;;){
1021       if((c = *cp++) == '\0' || c == '>')
1022          goto jleave;
1023       if(whitechar(c))
1024          break;
1025    }
1026
1027    /* Search for the parameter, take care of other quoting along the way */
1028    x = *param++;
1029    x = upperconv(x);
1030    quote = '\0';
1031    i = strlen(param);
1032
1033    for(hot = TRU1;;){
1034       if((c = *cp++) == '\0' || c == '>')
1035          goto jleave;
1036       if(whitechar(c)){
1037          hot = TRU1;
1038          continue;
1039       }
1040
1041       /* Could it be a parameter? */
1042       if(hot){
1043          hot = FAL0;
1044
1045          /* Is it the desired one? */
1046          if((c = upperconv(c)) == x && !ascncasecmp(param, cp, i)){
1047             char const *cp2 = cp + i;
1048
1049             if((quote = *cp2++) != '='){
1050                if(quote == '\0' || quote == '>')
1051                   goto jleave;
1052                while(whitechar(quote))
1053                   quote = *cp2++;
1054             }
1055             if(quote == '='){
1056                cp = cp2;
1057                break;
1058             }
1059             continue; /* XXX Optimize: i bytes or even cp2 can't be it! */
1060          }
1061       }
1062
1063       /* Not the desired one; but a parameter? */
1064       if(c != '=')
1065          continue;
1066       /* If so, properly skip over the value */
1067       if(c == '"' || c == '\''){
1068          /* TODO oops i have forgotten wether backslash quoting is allowed in
1069           * TODO quoted HTML parameter values?  not supporting that for now.. */
1070          for(quote = c; (c = *cp) != '\0' && c != quote; ++cp)
1071             ;
1072       }else
1073          while((c = *cp) != '\0' && !whitechar(c) && c != '>')
1074             ++cp;
1075       if(c == '\0')
1076          goto jleave;
1077    }
1078
1079    /* Skip further whitespace */
1080    for(;;){
1081       if((c = *cp++) == '\0' || c == '>')
1082          goto jleave;
1083       if(!whitechar(c))
1084          break;
1085    }
1086
1087    if(c == '"' || c == '\''){
1088       /* TODO oops i have forgotten wether backslash quoting is allowed in
1089        * TODO quoted HTML parameter values?  not supporting that for now.. */
1090       store->s = UNCONST(cp);
1091       for(quote = c; (c = *cp) != '\0' && c != quote; ++cp)
1092          ;
1093       /* XXX ... and we simply ignore a missing trailing " :> */
1094    }else{
1095       store->s = UNCONST(cp - 1);
1096       if(!whitechar(c))
1097          while((c = *cp) != '\0' && !whitechar(c) && c != '>')
1098             ++cp;
1099    }
1100    i = PTR2SIZE(cp - store->s);
1101
1102    /* Terrible tagsoup out there, e.g., groups.google.com produces href=""
1103     * parameter values prefixed and suffixed by newlines!  Therefore trim the
1104     * value content TODO join into the parse step above! */
1105    for (cp = store->s; i > 0 && spacechar(*cp); ++cp, --i)
1106       ;
1107    store->s = UNCONST(cp);
1108    for (cp += i - 1; i > 0 && spacechar(*cp); --cp, --i)
1109       ;
1110    if ((store->l = i) == 0)
1111       store->s = NULL;
1112 jleave:
1113    NYD2_LEAVE;
1114    return self;
1115 }
1116
1117 static struct htmlflt *
1118 _hf_expand_all_ents(struct htmlflt *self, struct str const *param)
1119 {
1120    char const *cp, *maxcp, *ep;
1121    char c;
1122    size_t i;
1123    NYD2_ENTER;
1124
1125    for (cp = param->s, maxcp = cp + param->l; cp < maxcp;)
1126       if ((c = *cp++) != '&')
1127          self = _hf_putc(self, c);
1128       else {
1129          for (ep = cp--; ep < maxcp && (c = *ep++) != ';';)
1130             if (c == '\0') {
1131                self = _hf_puts(self, cp);
1132                goto jleave;
1133             }
1134          if ((i = PTR2SIZE(ep - cp)) > 1)
1135             self = _hf_check_ent(self, cp, i);
1136          cp = ep;
1137       }
1138 jleave:
1139    NYD2_LEAVE;
1140    return self;
1141 }
1142
1143 static struct htmlflt *
1144 _hf_check_tag(struct htmlflt *self, char const *s)
1145 {
1146    char nobuf[32], c;
1147    struct str param;
1148    size_t i;
1149    struct htmlflt_tag const *hftp;
1150    ui32_t f;
1151    NYD2_ENTER;
1152
1153    /* Extra check only */
1154    assert(s != NULL);
1155    if (*s != '<') {
1156       DBG( n_alert("HTML tagsoup filter _hf_check_tag() called on soup!"); )
1157 jput_as_is:
1158       self = _hf_puts(self, self->hf_bdat);
1159       goto jleave;
1160    }
1161
1162    for (++s, i = 0; (c = s[i]) != '\0' && c != '>' && !whitechar(c); ++i)
1163       /* Special massage for things like <br/>: after the slash only whitespace
1164        * may separate us from the closing right angle! */
1165       if (c == '/') {
1166          size_t j = i + 1;
1167
1168          while ((c = s[j]) != '\0' && c != '>' && whitechar(c))
1169             ++j;
1170          if (c == '>')
1171             break;
1172       }
1173
1174    for (hftp = _hf_tags;;) {
1175       if (i == hftp->hft_len && !ascncasecmp(s, hftp->hft_tag, i)) {
1176          c = s[hftp->hft_len];
1177          if (c == '>' || c == '/' || whitechar(c))
1178             break;
1179       }
1180       if (PTRCMP(++hftp, >=, _hf_tags + NELEM(_hf_tags)))
1181          goto jnotknown;
1182    }
1183
1184    f = self->hf_flags;
1185    switch (hftp->hft_act) {
1186    case _HFSA_PRE_END:
1187       f &= ~_HF_PRE;
1188       if (0) {
1189          /* FALLTHRU */
1190    case _HFSA_PRE:
1191          f |= _HF_PRE;
1192       }
1193       self->hf_flags = f;
1194       /* FALLTHRU */
1195
1196    case _HFSA_NEEDSEP:
1197       if (!(self->hf_flags & _HF_NL_2))
1198          self = _hf_nl(self);
1199       /* FALLTHRU */
1200    case _HFSA_NEEDNL:
1201       if (!(f & _HF_NL_1))
1202          self = _hf_nl(self);
1203       if (hftp->hft_injc != '\0') {
1204          self = _hf_putc(self, hftp->hft_injc & 0x7F);
1205          if ((uc_i)hftp->hft_injc & 0x80)
1206             self = _hf_putc(self, ' ');
1207       }
1208       break;
1209
1210    case _HFSA_IGN:
1211       self->hf_ign_tag = hftp;
1212       self->hf_flags = (f |= _HF_IGN | _HF_NOPUT);
1213       break;
1214
1215    case _HFSA_IMG:
1216       self = _hf_param(self, &param, "alt");
1217       self = _hf_putc(self, '[');
1218       if (param.s == NULL) {
1219          param.s = UNCONST("IMG");
1220          param.l = 3;
1221          goto jimg_put;
1222       } /* else */ if (memchr(param.s, '&', param.l) != NULL)
1223          self = _hf_expand_all_ents(self, &param);
1224       else
1225 jimg_put:
1226          self = _hf_putbuf(self, param.s, param.l);
1227       self = _hf_putc(self, ']');
1228       break;
1229
1230    case _HFSA_HREF:
1231       self = _hf_param(self, &param, "href");
1232       /* Ignore non-external links */
1233       if (param.s != NULL && *param.s != '#') {
1234          struct htmlflt_href *hhp = smalloc(sizeof(*hhp) -
1235                VFIELD_SIZEOF(struct htmlflt_href, hfh_dat) + param.l +1);
1236
1237          hhp->hfh_next = self->hf_hrefs;
1238          hhp->hfh_no = ++self->hf_href_no;
1239          hhp->hfh_len = (ui32_t)param.l;
1240          memcpy(hhp->hfh_dat, param.s, param.l);
1241
1242          snprintf(nobuf, sizeof nobuf, "[%u]", hhp->hfh_no);
1243          self->hf_flags = (f |= _HF_HREF);
1244          self->hf_hrefs = hhp;
1245          self = _hf_puts(self, nobuf);
1246       } else
1247          self->hf_flags = (f &= ~_HF_HREF);
1248       break;
1249    case _HFSA_HREF_END:
1250       if (f & _HF_HREF) {
1251          snprintf(nobuf, sizeof nobuf, "[/%u]", self->hf_href_no);
1252          self = _hf_puts(self, nobuf);
1253       }
1254       break;
1255
1256    default:
1257       c = (char)(hftp->hft_act & 0xFF);
1258       self = _hf_putc(self, c);
1259       break;
1260    case '\0':
1261       break;
1262    }
1263 jleave:
1264    NYD2_LEAVE;
1265    return self;
1266
1267    /* The problem is that even invalid tagsoup is widely used, without real
1268     * searching i have seen e-mail address in <N@H.D> notation, and more.
1269     * To protect us a bit look around and possibly write the content as such */
1270 jnotknown:
1271    switch (*s) {
1272    case '!':
1273    case '?':
1274       /* Ignore <!DOCTYPE, <!-- comments, <? PIs.. */
1275       goto jleave;
1276    case '>':
1277       /* Print out an empty tag as such */
1278       if (s[1] == '\0') {
1279          --s;
1280          goto jput_as_is;
1281       }
1282       break;
1283    case '/':
1284       ++s;
1285       break;
1286    default:
1287       break;
1288    }
1289
1290    /* Also skip over : in order to suppress v:roundrect, w:anchorlock.. */
1291    while ((c = *s++) != '\0' && c != '>' && !whitechar(c) && c != ':')
1292       if (!asciichar(c) || punctchar(c)) {
1293          self = _hf_puts(self, self->hf_bdat);
1294          break;
1295       }
1296    goto jleave;
1297 }
1298
1299 static struct htmlflt *
1300 _hf_check_ent(struct htmlflt *self, char const *s, size_t l)
1301 {
1302    char nobuf[32];
1303    char const *s_save;
1304    size_t l_save;
1305    struct hf_ent const *hfep;
1306    long i;
1307    NYD2_ENTER;
1308
1309    s_save = s;
1310    l_save = l;
1311    assert(*s == '&');
1312    assert(l > 0);
1313    assert(s[l - 1] == ';');
1314    ++s;
1315    l -= 2;
1316
1317    /* Numeric entity, or try named search */
1318    if (*s == '#') {
1319       i = (*++s == 'x' ? 16 : 10);
1320
1321       if ((i != 16 || (++s, --l) > 0) && l < sizeof(nobuf)) {
1322          memcpy(nobuf, s, l);
1323          nobuf[l] = '\0';
1324          i = strtol(nobuf, NULL, i);
1325          if (i <= 0x7F)
1326             self = _hf_putc(self, (char)i);
1327          else if (self->hf_flags & _HF_UTF8) {
1328 jputuni:
1329             l = n_utf32_to_utf8((ui32_t)i, nobuf);
1330             self = _hf_putbuf(self, nobuf, l);
1331          } else
1332             goto jeent;
1333       } else
1334          goto jeent;
1335    } else {
1336       ui32_t f = self->hf_flags, hf;
1337
1338       for (hfep = _hf_ents; PTRCMP(hfep, <, _hf_ents + NELEM(_hf_ents)); ++hfep)
1339          if (l == ((hf = hfep->hfe_flags) & _HFE_LENGTH_MASK) &&
1340                !strncmp(s, hfep->hfe_ent, l)) {
1341             if ((hf & _HFE_HAVE_UNI) && (f & _HF_UTF8)) {
1342                i = hfep->hfe_uni;
1343                goto jputuni;
1344             } else if (hf & _HFE_HAVE_CSTR)
1345                self = _hf_puts(self, hfep->hfe_cstr);
1346             else
1347                self = _hf_putc(self, hfep->hfe_c);
1348             goto jleave;
1349          }
1350 jeent:
1351       self = _hf_putbuf(self, s_save, l_save);
1352    }
1353 jleave:
1354    NYD2_LEAVE;
1355    return self;
1356 }
1357
1358 static ssize_t
1359 _hf_add_data(struct htmlflt *self, char const *dat, size_t len)
1360 {
1361    char c, *cp, *cp_max;
1362    bool_t hot;
1363    ssize_t rv = 0;
1364    NYD_ENTER;
1365
1366    /* Final put request? */
1367    if (dat == NULL) {
1368       if (self->hf_len > 0 || self->hf_hrefs != NULL) {
1369          self = _hf_dump(self);
1370          if (self->hf_hrefs != NULL)
1371             self = _hf_dump_hrefs(self);
1372          rv = 1;
1373       }
1374       goto jleave;
1375    }
1376
1377    /* Always ensure some initial buffer */
1378    if ((cp = self->hf_curr) != NULL)
1379       cp_max = self->hf_bmax;
1380    else {
1381       cp = self->hf_curr = self->hf_bdat = smalloc(LINESIZE);
1382       cp_max = self->hf_bmax = cp + LINESIZE -1; /* (Always room for NUL!) */
1383    }
1384    hot = (cp != self->hf_bdat);
1385
1386    for (rv = (ssize_t)len; len > 0; --len) {
1387       ui32_t f = self->hf_flags;
1388
1389       if (f & _HF_ERROR)
1390          break;
1391       c = *dat++;
1392
1393       /* Soup is really weird, and scripts may contain almost anything (and
1394        * newer CSS standards are also cryptic): therefore prefix the _HF_IGN
1395        * test and walk until we see the required end tag */
1396       /* TODO For real safety _HF_IGN soup condome would also need to know
1397        * TODO about quoted strings so that 'var i = "</script>";' couldn't
1398        * TODO fool it!   We really want this mode also for _HF_NOPUT to be
1399        * TODO able to *gracefully* detect the tag-closing '>', but then if
1400        * TODO that is a single mechanism we should have made it! */
1401       if (f & _HF_IGN) {
1402          struct htmlflt_tag const *hftp = self->hf_ign_tag;
1403          size_t i;
1404
1405          if (c == '<') {
1406             hot = TRU1;
1407 jcp_reset:
1408             cp = self->hf_bdat;
1409          } else if (c == '>') {
1410             if (hot) {
1411                if ((i = PTR2SIZE(cp - self->hf_bdat)) > 1 &&
1412                      --i == hftp->hft_len &&
1413                      !ascncasecmp(self->hf_bdat + 1, hftp->hft_tag, i))
1414                   self->hf_flags = (f &= ~(_HF_IGN | _HF_NOPUT));
1415                hot = FAL0;
1416                goto jcp_reset;
1417             }
1418          } else if (hot) {
1419             *cp++ = c;
1420             i = PTR2SIZE(cp - self->hf_bdat);
1421             if ((i == 1 && c != '/') || --i > hftp->hft_len) {
1422                hot = FAL0;
1423                goto jcp_reset;
1424             }
1425          }
1426       } else switch (c) {
1427       case '<':
1428          /* People are using & without &amp;ing it, ditto <; be aware */
1429          if (f & (_HF_NOPUT | _HF_ENT)) {
1430             f &= ~_HF_ENT;
1431             /* Special case "<!--" buffer content to deal with really weird
1432              * things that can be done with "<!--[if gte mso 9]>" syntax */
1433             if (PTR2SIZE(cp - self->hf_bdat) != 4 ||
1434                   memcmp(self->hf_bdat, "<!--", 4)) {
1435                self->hf_flags = f;
1436                *cp = '\0';
1437                self = _hf_puts(self, self->hf_bdat);
1438                f = self->hf_flags;
1439             }
1440          }
1441          cp = self->hf_bdat;
1442          *cp++ = c;
1443          self->hf_flags = (f |= _HF_NOPUT);
1444          break;
1445       case '>':
1446          /* Weird tagsoup around, do we actually parse a tag? */
1447          if (!(f & _HF_NOPUT))
1448             goto jdo_c;
1449          cp[0] = c;
1450          cp[1] = '\0';
1451          f &= ~(_HF_NOPUT | _HF_ENT);
1452          self->hf_flags = f;
1453          self = _hf_check_tag(self, self->hf_bdat);
1454          *(cp = self->hf_bdat) = '\0'; /* xxx extra safety */
1455          /* Quick hack to get rid of redundant newline after <pre> XXX */
1456          if (!(f & _HF_PRE) && (self->hf_flags & _HF_PRE) &&
1457                len > 1 && *dat == '\n')
1458             ++dat, --len;
1459          break;
1460
1461       case '\r': /* TODO CR should be stripped in lower level!! (Only B64!?!) */
1462          break;
1463       case '\n':
1464          /* End of line is not considered unless we are in PRE section.
1465           * However, in _HF_NOPUT mode we must be aware of tagsoup which uses
1466           * newlines for separating parameters */
1467          if (f & _HF_NOPUT)
1468             goto jdo_c;
1469          self = (f & _HF_PRE) ? _hf_nl_force(self) : _hf_putc(self, ' ');
1470          break;
1471
1472       case '\t':
1473          if (!(f & _HF_PRE))
1474             c = ' ';
1475          /* FALLTHRU */
1476       default:
1477 jdo_c:
1478          /* If not currently parsing a tag and bypassing normal output.. */
1479          if (!(f & _HF_NOPUT)) {
1480             if (cntrlchar(c))
1481                break;
1482             if (c == '&') {
1483                cp = self->hf_bdat;
1484                *cp++ = c;
1485                self->hf_flags = (f |= _HF_NOPUT | _HF_ENT);
1486             } else if (f & _HF_PRE) {
1487                self = _hf_putc_premode(self, c);
1488                self->hf_flags &= ~_HF_BLANK;
1489             } else
1490               self = _hf_putc(self, c);
1491          } else if ((f & _HF_ENT) && c == ';') {
1492             cp[0] = c;
1493             cp[1] = '\0';
1494             f &= ~(_HF_NOPUT | _HF_ENT);
1495             self->hf_flags = f;
1496            self = _hf_check_ent(self, self->hf_bdat,
1497                PTR2SIZE(cp + 1 - self->hf_bdat));
1498          } else {
1499             /* We may need to grow the buffer */
1500             if (PTRCMP(cp + 42/2, >=, cp_max)) {
1501                size_t i = PTR2SIZE(cp - self->hf_bdat),
1502                   m = PTR2SIZE(self->hf_bmax - self->hf_bdat) + LINESIZE;
1503
1504                cp = self->hf_bdat = srealloc(self->hf_bdat, m);
1505                self->hf_bmax = cp + m -1;
1506                self->hf_curr = (cp += i);
1507             }
1508             *cp++ = c;
1509          }
1510       }
1511    }
1512    self->hf_curr = cp;
1513 jleave:
1514   NYD_LEAVE;
1515   return (self->hf_flags & _HF_ERROR) ? -1 : rv;
1516 }
1517
1518 /*
1519  * TODO Because we don't support filter chains yet this filter will be run
1520  * TODO in a dedicated subprocess, driven via a special Popen() mode
1521  */
1522 static bool_t __hf_hadpipesig;
1523 static void
1524 __hf_onpipe(int signo)
1525 {
1526    NYD_X; /* Signal handler */
1527    UNUSED(signo);
1528    __hf_hadpipesig = TRU1;
1529 }
1530
1531 FL int
1532 htmlflt_process_main(void)
1533 {
1534    char buf[BUFFER_SIZE];
1535    struct htmlflt hf;
1536    size_t i;
1537    int rv;
1538    NYD_ENTER;
1539
1540    __hf_hadpipesig = FAL0;
1541    safe_signal(SIGPIPE, &__hf_onpipe);
1542
1543    htmlflt_init(&hf);
1544    htmlflt_reset(&hf, stdout);
1545
1546    for (;;) {
1547       if ((i = fread(buf, sizeof(buf[0]), NELEM(buf), stdin)) == 0) {
1548          rv = !feof(stdin);
1549          break;
1550       }
1551
1552       if ((rv = __hf_hadpipesig))
1553          break;
1554       /* Just use this directly.. */
1555       if (htmlflt_push(&hf, buf, i) < 0) {
1556          rv = 1;
1557          break;
1558       }
1559    }
1560    if (rv == 0 && htmlflt_flush(&hf) < 0)
1561       rv = 1;
1562
1563    htmlflt_destroy(&hf);
1564
1565    rv |= __hf_hadpipesig;
1566    NYD_LEAVE;
1567    return rv;
1568 }
1569
1570 FL void
1571 htmlflt_init(struct htmlflt *self)
1572 {
1573    NYD_ENTER;
1574    /* (Rather redundant though) */
1575    memset(self, 0, sizeof *self);
1576    NYD_LEAVE;
1577 }
1578
1579 FL void
1580 htmlflt_destroy(struct htmlflt *self)
1581 {
1582    NYD_ENTER;
1583    htmlflt_reset(self, NULL);
1584    NYD_LEAVE;
1585 }
1586
1587 FL void
1588 htmlflt_reset(struct htmlflt *self, FILE *f)
1589 {
1590    struct htmlflt_href *hfhp;
1591    NYD_ENTER;
1592
1593    while ((hfhp = self->hf_hrefs) != NULL) {
1594       self->hf_hrefs = hfhp->hfh_next;
1595       free(hfhp);
1596    }
1597
1598    if (self->hf_bdat != NULL)
1599       free(self->hf_bdat);
1600    if (self->hf_line != NULL)
1601       free(self->hf_line);
1602
1603    memset(self, 0, sizeof *self);
1604
1605    if (f != NULL) {
1606       ui32_t sw = MAX(_HF_MINLEN, (ui32_t)scrnwidth);
1607
1608       self->hf_line = smalloc((size_t)sw * mb_cur_max +1);
1609       self->hf_lmax = sw;
1610
1611       if (options & OPT_UNICODE) /* TODO not truly generic */
1612          self->hf_flags = _HF_UTF8;
1613       self->hf_os = f;
1614    }
1615    NYD_LEAVE;
1616 }
1617
1618 FL ssize_t
1619 htmlflt_push(struct htmlflt *self, char const *dat, size_t len)
1620 {
1621    ssize_t rv;
1622    NYD_ENTER;
1623
1624    rv = _hf_add_data(self, dat, len);
1625    NYD_LEAVE;
1626    return rv;
1627 }
1628
1629 FL ssize_t
1630 htmlflt_flush(struct htmlflt *self)
1631 {
1632    ssize_t rv;
1633    NYD_ENTER;
1634
1635    rv = _hf_add_data(self, NULL, 0);
1636    rv |= !fflush(self->hf_os) ? 0 : -1;
1637    NYD_LEAVE;
1638    return rv;
1639 }
1640 #endif /* HAVE_FILTER_HTML_TAGSOUP */
1641
1642 /* s-it-mode */