filter.c

   1 /*@ S-nail - a mail user agent derived from Berkeley Mail.
   2  *@ Filter objects.
   3  *
   4  * Copyright (c) 2013 - 2017 Steffen (Daode) Nurpmeso <steffen@sdaoden.eu>.
   5  *
   6  * Permission to use, copy, modify, and/or distribute this software for any
   7  * purpose with or without fee is hereby granted, provided that the above
   8  * copyright notice and this permission notice appear in all copies.
   9  *
  10  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  11  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  12  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  13  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  14  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  15  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  16  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  17  */
  18 #undef n_FILE
  19 #define n_FILE filter
  20
  21 #ifndef HAVE_AMALGAMATION
  22 # include "nail.h"
  23 #endif
  24
  25 /*
  26  * Quotation filter
  27  */
  28
  29 /*
  30  * TODO quotation filter: anticipate in future data: don't break if only WS
  31  * TODO or a LF escaping \ follows on the line (simply reuse the latter).
  32  */
  33
  34 #ifdef HAVE_QUOTE_FOLD
  35 n_CTAV(n_QUOTE_MAX > 3);
  36
  37 enum qf_state {
  38    _QF_CLEAN,
  39    _QF_PREFIX,
  40    _QF_DATA
  41 };
  42
  43 struct qf_vc {
  44    struct quoteflt   *self;
  45    char const        *buf;
  46    size_t            len;
  47 };
  48
  49 /* Print out prefix and current quote */
  50 static ssize_t _qf_dump_prefix(struct quoteflt *self);
  51
  52 /* Add one data character */
  53 static ssize_t _qf_add_data(struct quoteflt *self, wchar_t wc);
  54
  55 /* State machine handlers */
  56 static ssize_t _qf_state_prefix(struct qf_vc *vc);
  57 static ssize_t _qf_state_data(struct qf_vc *vc);
  58
  59 static ssize_t
  60 _qf_dump_prefix(struct quoteflt *self)
  61 {
  62    ssize_t rv;
  63    size_t i;
  64    NYD_ENTER;
  65
  66    if ((i = self->qf_pfix_len) > 0 && i != fwrite(self->qf_pfix, 1, i,
  67          self->qf_os))
  68       goto jerr;
  69    rv = i;
  70
  71    if ((i = self->qf_currq.l) > 0 && i != fwrite(self->qf_currq.s, 1, i,
  72          self->qf_os))
  73       goto jerr;
  74    rv += i;
  75 jleave:
  76    NYD_LEAVE;
  77    return rv;
  78 jerr:
  79    rv = -1;
  80    goto jleave;
  81 }
  82
  83 static ssize_t
  84 _qf_add_data(struct quoteflt *self, wchar_t wc)
  85 {
  86    char *save_b;
  87    ui32_t save_l, save_w;
  88    ssize_t rv = 0;
  89    int w, l;
  90    NYD_ENTER;
  91
  92    save_l = save_w = 0; /* silence cc */
  93    save_b = NULL;
  94    /* <newline> ends state */
  95    if (wc == L'\n')
  96       goto jflush;
  97    if (wc == L'\r') /* TODO CR should be stripped in lower level!! */
  98       goto jleave;
  99
 100    /* Unroll <tab> to spaces */
 101    if (wc == L'\t') {
 102       save_l = self->qf_datw;
 103       save_w = (save_l + n_QUOTE_TAB_SPACES) & ~(n_QUOTE_TAB_SPACES - 1);
 104       save_w -= save_l;
 105       while (save_w-- > 0) {
 106          ssize_t j = _qf_add_data(self, L' ');
 107          if (j < 0) {
 108             rv = j;
 109             break;
 110          }
 111          rv += j;
 112       }
 113       goto jleave;
 114    }
 115
 116    w = wcwidth(wc);
 117    if (w == -1) {
 118 jbad:
 119       ++self->qf_datw;
 120       self->qf_dat.s[self->qf_dat.l++] = '?';
 121    } else {
 122       l = wctomb(self->qf_dat.s + self->qf_dat.l, wc);
 123       if (l < 0)
 124          goto jbad;
 125       self->qf_datw += (ui32_t)w;
 126       self->qf_dat.l += (size_t)l;
 127    }
 128
 129    /* TODO The last visual may excess (adjusted!) *qfold-max* if it's a wide;
 130     * TODO place it on the next line, break before */
 131    if (self->qf_datw >= self->qf_qfold_max) {
 132       /* If we have seen a nice breakpoint during traversal, shuffle data
 133        * around a bit so as to restore the trailing part after flushing */
 134       if (self->qf_brkl > 0) {
 135          save_w = self->qf_datw - self->qf_brkw;
 136          save_l = self->qf_dat.l - self->qf_brkl;
 137          save_b = self->qf_dat.s + self->qf_brkl + 2;
 138          memmove(save_b, save_b - 2, save_l);
 139          self->qf_dat.l = self->qf_brkl;
 140       }
 141
 142       self->qf_dat.s[self->qf_dat.l++] = '\\';
 143 jflush:
 144       self->qf_dat.s[self->qf_dat.l++] = '\n';
 145       rv = quoteflt_flush(self);
 146
 147       /* Restore takeovers, if any */
 148       if (save_b != NULL) {
 149          self->qf_brk_isws = FAL0;
 150          self->qf_datw += save_w;
 151          self->qf_dat.l = save_l;
 152          memmove(self->qf_dat.s, save_b, save_l);
 153       }
 154    } else if (self->qf_datw >= self->qf_qfold_min && !self->qf_brk_isws) {
 155       bool_t isws = (iswspace(wc) != 0);
 156
 157       if (isws || !self->qf_brk_isws || self->qf_brkl == 0) {
 158          self->qf_brkl = self->qf_dat.l;
 159          self->qf_brkw = self->qf_datw;
 160          self->qf_brk_isws = isws;
 161       }
 162    }
 163
 164    /* If state changed to prefix, perform full reset (note this implies that
 165     * quoteflt_flush() performs too much work..) */
 166    if (wc == '\n') {
 167       self->qf_state = _QF_PREFIX;
 168       self->qf_wscnt = self->qf_datw = 0;
 169       self->qf_currq.l = 0;
 170    }
 171 jleave:
 172    NYD_LEAVE;
 173    return rv;
 174 }
 175
 176 static ssize_t
 177 _qf_state_prefix(struct qf_vc *vc)
 178 {
 179    struct quoteflt *self;
 180    ssize_t rv;
 181    char const *buf;
 182    size_t len, i;
 183    wchar_t wc;
 184    NYD_ENTER;
 185
 186    self = vc->self;
 187    rv = 0;
 188
 189    for (buf = vc->buf, len = vc->len; len > 0;) {
 190       /* xxx NULL BYTE! */
 191       i = mbrtowc(&wc, buf, len, self->qf_mbps);
 192       if (i == (size_t)-1) {
 193          /* On hard error, don't modify mbstate_t and step one byte */
 194          self->qf_mbps[0] = self->qf_mbps[1];
 195          ++buf;
 196          --len;
 197          self->qf_wscnt = 0;
 198          continue;
 199       }
 200       self->qf_mbps[1] = self->qf_mbps[0];
 201       if (i == (size_t)-2) {
 202          /* Redundant shift sequence, out of buffer */
 203          len = 0;
 204          break;
 205       }
 206       buf += i;
 207       len -= i;
 208
 209       if (wc == L'\n')
 210          goto jfin;
 211       if (iswspace(wc)) {
 212          ++self->qf_wscnt;
 213          continue;
 214       }
 215       if (i == 1 && n_uasciichar(wc) &&
 216             strchr(self->qf_quote_chars, (char)wc) != NULL){
 217          self->qf_wscnt = 0;
 218          if (self->qf_currq.l >= n_QUOTE_MAX - 3) {
 219             self->qf_currq.s[n_QUOTE_MAX - 3] = '.';
 220             self->qf_currq.s[n_QUOTE_MAX - 2] = '.';
 221             self->qf_currq.s[n_QUOTE_MAX - 1] = '.';
 222             self->qf_currq.l = n_QUOTE_MAX;
 223          } else
 224             self->qf_currq.s[self->qf_currq.l++] = buf[-1];
 225          continue;
 226       }
 227
 228       /* The quote is parsed and compressed; dump it */
 229 jfin:
 230       self->qf_state = _QF_DATA;
 231       /* Overtake WS to the current quote in order to preserve it for eventual
 232        * necessary follow lines, too */
 233       /* TODO we de-facto "normalize" to ASCII SP here which MESSES tabs!! */
 234       while (self->qf_wscnt-- > 0 && self->qf_currq.l < n_QUOTE_MAX)
 235          self->qf_currq.s[self->qf_currq.l++] = ' ';
 236       self->qf_datw = self->qf_pfix_len + self->qf_currq.l;
 237       self->qf_wscnt = 0;
 238       rv = _qf_add_data(self, wc);
 239       break;
 240    }
 241
 242    vc->buf = buf;
 243    vc->len = len;
 244    NYD_LEAVE;
 245    return rv;
 246 }
 247
 248 static ssize_t
 249 _qf_state_data(struct qf_vc *vc)
 250 {
 251    struct quoteflt *self;
 252    ssize_t rv;
 253    char const *buf;
 254    size_t len, i;
 255    wchar_t wc;
 256    NYD_ENTER;
 257
 258    self = vc->self;
 259    rv = 0;
 260
 261    for (buf = vc->buf, len = vc->len; len > 0;) {
 262       /* xxx NULL BYTE! */
 263       i = mbrtowc(&wc, buf, len, self->qf_mbps);
 264       if (i == (size_t)-1) {
 265          /* On hard error, don't modify mbstate_t and step one byte */
 266          self->qf_mbps[0] = self->qf_mbps[1];
 267          ++buf;
 268          --len;
 269          continue;
 270       }
 271       self->qf_mbps[1] = self->qf_mbps[0];
 272       if (i == (size_t)-2) {
 273          /* Redundant shift sequence, out of buffer */
 274          len = 0;
 275          break;
 276       }
 277       buf += i;
 278       len -= i;
 279
 280       {  ssize_t j = _qf_add_data(self, wc);
 281          if (j < 0) {
 282             rv = j;
 283             break;
 284          }
 285          rv += j;
 286       }
 287
 288       if (self->qf_state != _QF_DATA)
 289          break;
 290    }
 291
 292    vc->buf = buf;
 293    vc->len = len;
 294    NYD_LEAVE;
 295    return rv;
 296 }
 297 #endif /* HAVE_QUOTE_FOLD */
 298
 299 FL struct quoteflt *
 300 quoteflt_dummy(void) /* TODO LEGACY (until filters are plugged when needed) */
 301 {
 302    static struct quoteflt qf_i;
 303
 304    return &qf_i;
 305 }
 306
 307 FL void
 308 quoteflt_init(struct quoteflt *self, char const *prefix)
 309 {
 310 #ifdef HAVE_QUOTE_FOLD
 311    char const *xcp, *cp;
 312 #endif
 313    NYD_ENTER;
 314
 315    memset(self, 0, sizeof *self);
 316
 317    if ((self->qf_pfix = prefix) != NULL)
 318       self->qf_pfix_len = (ui32_t)strlen(prefix);
 319
 320    /* Check whether the user wants the more fancy quoting algorithm */
 321    /* TODO *quote-fold*: n_QUOTE_MAX may excess it! */
 322 #ifdef HAVE_QUOTE_FOLD
 323    if (self->qf_pfix_len > 0 && (cp = ok_vlook(quote_fold)) != NULL) {
 324       ui32_t qmin, qmax;
 325
 326       /* These magic values ensure we don't bail */
 327       n_idec_ui32_cp(&qmax, cp, 10, &xcp);
 328       if (qmax < self->qf_pfix_len + 6)
 329          qmax = self->qf_pfix_len + 6;
 330       --qmax; /* The newline escape */
 331       if (cp == xcp || *xcp == '\0')
 332          qmin = (qmax >> 1) + (qmax >> 2) + (qmax >> 5);
 333       else {
 334          n_idec_ui32_cp(&qmin, &xcp[1], 10, NULL);
 335          if (qmin < qmax >> 1)
 336             qmin = qmax >> 1;
 337          else if (qmin > qmax - 2)
 338             qmin = qmax - 2;
 339       }
 340       self->qf_qfold_min = qmin;
 341       self->qf_qfold_max = qmax;
 342       self->qf_quote_chars = ok_vlook(quote_chars);
 343
 344       /* Add pad for takeover copies, reverse solidus and newline */
 345       self->qf_dat.s = salloc((qmax + 3) * n_mb_cur_max);
 346       self->qf_currq.s = salloc((n_QUOTE_MAX + 1) * n_mb_cur_max);
 347    }
 348 #endif
 349    NYD_LEAVE;
 350 }
 351
 352 FL void
 353 quoteflt_destroy(struct quoteflt *self) /* xxx inline */
 354 {
 355    NYD_ENTER;
 356    n_UNUSED(self);
 357    NYD_LEAVE;
 358 }
 359
 360 FL void
 361 quoteflt_reset(struct quoteflt *self, FILE *f) /* xxx inline */
 362 {
 363    NYD_ENTER;
 364    self->qf_os = f;
 365 #ifdef HAVE_QUOTE_FOLD
 366    self->qf_state = _QF_CLEAN;
 367    self->qf_dat.l =
 368    self->qf_currq.l = 0;
 369    memset(self->qf_mbps, 0, sizeof self->qf_mbps);
 370 #endif
 371    NYD_LEAVE;
 372 }
 373
 374 FL ssize_t
 375 quoteflt_push(struct quoteflt *self, char const *dat, size_t len)
 376 {
 377    /* (xxx Ideally the actual push() [and flush()] would be functions on their
 378     * xxx own, via indirect vtbl call ..) */
 379    ssize_t rv = 0;
 380    NYD_ENTER;
 381
 382    self->qf_nl_last = (len > 0 && dat[len - 1] == '\n'); /* TODO HACK */
 383
 384    if (len == 0)
 385       goto jleave;
 386
 387    /* Bypass? TODO Finally, this filter simply should not be used, then
 388     * (TODO It supercedes prefix_write() or something) */
 389    if (self->qf_pfix_len == 0) {
 390       if (len != fwrite(dat, 1, len, self->qf_os))
 391          goto jerr;
 392       rv = len;
 393    }
 394    /* Normal: place *indentprefix* at every BOL */
 395    else
 396 #ifdef HAVE_QUOTE_FOLD
 397       if (self->qf_qfold_max == 0)
 398 #endif
 399    {
 400       void *vp;
 401       size_t ll;
 402       bool_t pxok = (self->qf_qfold_min != 0);
 403
 404       for (;;) {
 405          if (!pxok) {
 406             ll = self->qf_pfix_len;
 407             if (ll != fwrite(self->qf_pfix, 1, ll, self->qf_os))
 408                goto jerr;
 409             rv += ll;
 410             pxok = TRU1;
 411          }
 412
 413          /* xxx Strictly speaking this is invalid, because only `/' and `.' are
 414           * xxx mandated by POSIX.1-2008 as "invariant across all locales
 415           * xxx supported"; though there is no charset known which uses this
 416           * xxx control char as part of a multibyte character; note that S-nail
 417           * XXX (and the Mail codebase as such) do not support EBCDIC */
 418          if ((vp = memchr(dat, '\n', len)) == NULL)
 419             ll = len;
 420          else {
 421             pxok = FAL0;
 422             ll = PTR2SIZE((char*)vp - dat) + 1;
 423          }
 424
 425          if (ll != fwrite(dat, sizeof *dat, ll, self->qf_os))
 426             goto jerr;
 427          rv += ll;
 428          if ((len -= ll) == 0)
 429             break;
 430          dat += ll;
 431       }
 432
 433       self->qf_qfold_min = pxok;
 434    }
 435    /* Overly complicated, though still only line-per-line: *quote-fold*.
 436     * - If .qf_currq.l is 0, then we are in a clean state.  Reset .qf_mbps;
 437     *   TODO note this means we assume that lines start with reset escape seq,
 438     *   TODO but i don't think this is any worse than what we currently do;
 439     *   TODO in 15.0, with the value carrier, we should carry conversion states
 440     *   TODO all along, only resetting on error (or at words for header =???=);
 441     *   TODO this still is weird for error handling, but we need to act more
 442     *   TODO stream-alike (though in practice i don't think cross-line states
 443     *   TODO can be found, because of compatibility reasons; however, being
 444     *   TODO a problem rather than a solution is not a good thing (tm))
 445     * - Lookout for a newline */
 446 #ifdef HAVE_QUOTE_FOLD
 447    else {
 448       struct qf_vc vc;
 449       ssize_t i;
 450
 451       vc.self = self;
 452       vc.buf = dat;
 453       vc.len = len;
 454       while (vc.len > 0) {
 455          switch (self->qf_state) {
 456          case _QF_CLEAN:
 457          case _QF_PREFIX:
 458             i = _qf_state_prefix(&vc);
 459             break;
 460          default: /* silence cc (`i' unused) */
 461          case _QF_DATA:
 462             i = _qf_state_data(&vc);
 463             break;
 464          }
 465          if (i < 0)
 466             goto jerr;
 467          rv += i;
 468       }
 469    }
 470 #endif /* HAVE_QUOTE_FOLD */
 471
 472 jleave:
 473    NYD_LEAVE;
 474    return rv;
 475 jerr:
 476    rv = -1;
 477    goto jleave;
 478 }
 479
 480 FL ssize_t
 481 quoteflt_flush(struct quoteflt *self)
 482 {
 483    ssize_t rv = 0;
 484    NYD_ENTER;
 485    n_UNUSED(self);
 486
 487 #ifdef HAVE_QUOTE_FOLD
 488    if (self->qf_dat.l > 0) {
 489       rv = _qf_dump_prefix(self);
 490       if (rv >= 0) {
 491          size_t i = self->qf_dat.l;
 492          if (i == fwrite(self->qf_dat.s, 1, i, self->qf_os))
 493             rv += i;
 494          else
 495             rv = -1;
 496          self->qf_dat.l = 0;
 497          self->qf_brk_isws = FAL0;
 498          self->qf_wscnt = self->qf_brkl = self->qf_brkw = 0;
 499          self->qf_datw = self->qf_pfix_len + self->qf_currq.l;
 500       }
 501    }
 502 #endif
 503    NYD_LEAVE;
 504    return rv;
 505 }
 506
 507 /*
 508  * HTML tagsoup filter TODO rewrite wchar_t based (require HAVE_C90AMEND1)
 509  * TODO . Numeric &#NO; entities should also be treated by struct hf_ent
 510  * TODO . Yes, we COULD support CSS based quoting when we'd check type="quote"
 511  * TODO   (nonstandard) and watch out for style="gmail_quote" (or so, VERY
 512  * TODO   nonstandard) and tracking a stack of such elements (to be popped
 513  * TODO   once the closing element is seen).  Then, after writing a newline,
 514  * TODO   place sizeof(stack) ">"s first.  But aren't these HTML mails rude?
 515  * TODO Interlocking and non-well-formed data will break us down
 516  */
 517 #ifdef HAVE_FILTER_HTML_TAGSOUP
 518
 519 enum hf_limits {
 520    _HF_MINLEN  = 10,       /* Minimum line length (can't really be smaller) */
 521    _HF_BRKSUB  = 8         /* Start considering line break MAX - BRKSUB */
 522 };
 523
 524 enum hf_flags {
 525    _HF_UTF8    = 1<<0,     /* Data is in UTF-8 */
 526    _HF_ERROR   = 1<<1,     /* A hard error occurred, bail as soon as possible */
 527    _HF_NOPUT   = 1<<2,     /* (In a tag,) Don't generate output */
 528    _HF_IGN     = 1<<3,     /* Ignore mode on */
 529    _HF_ANY     = 1<<4,     /* Yet seen just any output */
 530    _HF_PRE     = 1<<5,     /* In <pre>formatted mode */
 531    _HF_ENT     = 1<<6,     /* Currently parsing an entity */
 532    _HF_BLANK   = 1<<7,     /* Whitespace last */
 533    _HF_HREF    = 1<<8,     /* External <a href=> was the last href seen */
 534
 535    _HF_NL_1    = 1<<9,     /* One \n seen */
 536    _HF_NL_2    = 2<<9,     /* We have produced an all empty line */
 537    _HF_NL_MASK = _HF_NL_1 | _HF_NL_2
 538 };
 539
 540 enum hf_special_actions {
 541    _HFSA_NEEDSEP  = -1,    /* Need an empty line (paragraph separator) */
 542    _HFSA_NEEDNL   = -2,    /* Need a new line start (table row) */
 543    _HFSA_IGN      = -3,    /* Things like <style>..</style>, <script>.. */
 544    _HFSA_PRE      = -4,    /* <pre>.. */
 545    _HFSA_PRE_END  = -5,
 546    _HFSA_IMG      = -6,    /* <img> */
 547    _HFSA_HREF     = -7,    /* <a>.. */
 548    _HFSA_HREF_END = -8
 549 };
 550
 551 enum hf_entity_flags {
 552    _HFE_HAVE_UNI  = 1<<6,  /* Have a Unicode replacement character */
 553    _HFE_HAVE_CSTR = 1<<7,  /* Have a string replacement */
 554    /* We store the length of the entity name in the flags, too */
 555    _HFE_LENGTH_MASK = (1<<6) - 1
 556 };
 557
 558 struct htmlflt_href {
 559    struct htmlflt_href *hfh_next;
 560    ui32_t      hfh_no;     /* Running sequence */
 561    ui32_t      hfh_len;    /* of .hfh_dat */
 562    char        hfh_dat[n_VFIELD_SIZE(0)];
 563 };
 564
 565 struct htmlflt_tag {
 566    si32_t      hft_act;    /* char or hf_special_actions */
 567    /* Not NUL: character to inject, with high bit set: place a space
 568     * afterwards.  Note: only recognized with _HFSA_NEEDSEP or _HFSA_NEEDNL */
 569    char        hft_injc;
 570    ui8_t       hft_len;    /* Useful bytes in (NUL terminated) .hft_tag */
 571    char const  hft_tag[10]; /* Tag less < and > surroundings (TR, /TR, ..) */
 572 };
 573 n_CTA(n_SIZEOF_FIELD(struct htmlflt_tag, hft_tag) < LINESIZE,
 574    "Structure field too large a size"); /* .hf_ign_tag */
 575
 576 struct hf_ent {
 577    ui8_t       hfe_flags;  /* enum hf_entity_flags plus length of .hfe_ent */
 578    char        hfe_c;      /* Plain replacement character */
 579    ui16_t      hfe_uni;    /* Unicode codepoint if _HFE_HAVE_UNI */
 580    char        hfe_cstr[5]; /* _HFE_HAVE_CSTR (e.g., &hellip; -> ...) */
 581    char const  hfe_ent[7]; /* Entity less & and ; surroundings */
 582 };
 583
 584 /* Tag list; not binary searched :(, so try to take care a bit */
 585 static struct htmlflt_tag const  _hf_tags[] = {
 586 # undef _X
 587 # undef _XC
 588 # define _X(S,A)     {A, '\0', sizeof(S) -1, S "\0"}
 589 # define _XC(S,C,A)  {A, C, sizeof(S) -1, S "\0"}
 590
 591    _X("P", _HFSA_NEEDSEP),       _X("/P", _HFSA_NEEDNL),
 592    _X("DIV", _HFSA_NEEDSEP),     _X("/DIV", _HFSA_NEEDNL),
 593    _X("TR", _HFSA_NEEDNL),
 594                                  _X("/TH", '\t'),
 595                                  _X("/TD", '\t'),
 596    /* Let it stand out; also since we don't support implicit paragraphs after
 597     * block elements, plain running text after a list (seen in Unicode
 598     * announcement via Firefox) */
 599    _X("UL", _HFSA_NEEDSEP),      _X("/UL", _HFSA_NEEDSEP),
 600    _XC("LI", (char)0x80 | '*', _HFSA_NEEDSEP),
 601    _X("DL", _HFSA_NEEDSEP),
 602    _X("DT", _HFSA_NEEDNL),
 603
 604    _X("A", _HFSA_HREF),          _X("/A", _HFSA_HREF_END),
 605    _X("IMG", _HFSA_IMG),
 606    _X("BR", '\n'),
 607    _X("PRE", _HFSA_PRE),         _X("/PRE", _HFSA_PRE_END),
 608    _X("TITLE", _HFSA_NEEDSEP),   /*_X("/TITLE", '\n'),*/
 609    _X("H1", _HFSA_NEEDSEP),      /*_X("/H1", '\n'),*/
 610    _X("H2", _HFSA_NEEDSEP),      /*_X("/H2", '\n'),*/
 611    _X("H3", _HFSA_NEEDSEP),      /*_X("/H3", '\n'),*/
 612    _X("H4", _HFSA_NEEDSEP),      /*_X("/H4", '\n'),*/
 613    _X("H5", _HFSA_NEEDSEP),      /*_X("/H5", '\n'),*/
 614    _X("H6", _HFSA_NEEDSEP),      /*_X("/H6", '\n'),*/
 615
 616    _X("STYLE", _HFSA_IGN),
 617    _X("SCRIPT", _HFSA_IGN),
 618
 619 # undef _X
 620 };
 621
 622 /* Entity list; not binary searched.. */
 623 static struct hf_ent const       _hf_ents[] = {
 624 # undef _X
 625 # undef _XU
 626 # undef _XS
 627 # undef _XUS
 628 # define _X(E,C)     {(sizeof(E) -1), C, 0x0u, "", E "\0"}
 629 # define _XU(E,C,U)  {(sizeof(E) -1) | _HFE_HAVE_UNI, C, U, "", E "\0"}
 630 # define _XS(E,S)    {(sizeof(E) -1) | _HFE_HAVE_CSTR, '\0', 0x0u,S "\0",E "\0"}
 631 # define _XSU(E,S,U) \
 632    {(sizeof(E) -1) | _HFE_HAVE_UNI | _HFE_HAVE_CSTR, '\0', U, S "\0", E "\0"}
 633
 634    _X("quot", '"'),
 635    _X("amp", '&'),
 636    _X("lt", '<'),                _X("gt", '>'),
 637
 638    _XU("nbsp", ' ', 0x0020 /* Note: not 0x00A0 seems to be better for us */),
 639    _XU("middot", '.', 0x00B7),
 640    _XSU("hellip", "...", 0x2026),
 641    _XSU("mdash", "---", 0x2014), _XSU("ndash", "--", 0x2013),
 642    _XSU("laquo", "<<", 0x00AB),  _XSU("raquo", ">>", 0x00BB),
 643    _XSU("lsaquo", "<", 0x2039),  _XSU("rsaquo", ">", 0x203A),
 644    _XSU("lsquo", "'", 0x2018),   _XSU("rsquo", "'", 0x2019),
 645    _XSU("ldquo", "\"", 0x201C),  _XSU("rdquo", "\"", 0x201D),
 646    _XSU("uarr", "^|", 0x2191),   _XSU("darr", "|v", 0x2193),
 647
 648    _XSU("cent", "CENT", 0x00A2),
 649    _XSU("copy", "(C)", 0x00A9),
 650    _XSU("euro", "EUR", 0x20AC),
 651    _XSU("infin", "INFY", 0x221E),
 652    _XSU("pound", "GBP", 0x00A3),
 653    _XSU("reg", "(R)", 0x00AE),
 654    _XSU("sect", "S:", 0x00A7),
 655    _XSU("yen", "JPY", 0x00A5),
 656
 657    /* German umlauts */
 658    _XSU("Auml", "Ae", 0x00C4),   _XSU("auml", "ae", 0x00E4),
 659    _XSU("Ouml", "Oe", 0x00D6),   _XSU("ouml", "oe", 0x00F6),
 660    _XSU("Uuml", "Ue", 0x00DC),   _XSU("uuml", "ue", 0x00FC),
 661    _XSU("szlig", "ss", 0x00DF)
 662
 663 # undef _X
 664 # undef _XU
 665 # undef _XS
 666 # undef _XSU
 667 };
 668
 669 /* Real output */
 670 static struct htmlflt * _hf_dump_hrefs(struct htmlflt *self);
 671 static struct htmlflt * _hf_dump(struct htmlflt *self);
 672 static struct htmlflt * _hf_store(struct htmlflt *self, char c);
 673 # ifdef HAVE_NATCH_CHAR
 674 static struct htmlflt * __hf_sync_mbstuff(struct htmlflt *self);
 675 # endif
 676
 677 /* Virtual output */
 678 static struct htmlflt * _hf_nl(struct htmlflt *self);
 679 static struct htmlflt * _hf_nl_force(struct htmlflt *self);
 680 static struct htmlflt * _hf_putc(struct htmlflt *self, char c);
 681 static struct htmlflt * _hf_putc_premode(struct htmlflt *self, char c);
 682 static struct htmlflt * _hf_puts(struct htmlflt *self, char const *cp);
 683 static struct htmlflt * _hf_putbuf(struct htmlflt *self,
 684                            char const *cp, size_t len);
 685
 686 /* Try to locate a param'eter in >hf_bdat, store it (non-terminated!) or NULL */
 687 static struct htmlflt * _hf_param(struct htmlflt *self, struct str *store,
 688                            char const *param);
 689
 690 /* Expand all entities in the given parameter */
 691 static struct htmlflt * _hf_expand_all_ents(struct htmlflt *self,
 692                            struct str const *param);
 693
 694 /* Completely parsed over a tag / an entity, interpret that */
 695 static struct htmlflt * _hf_check_tag(struct htmlflt *self, char const *s);
 696 static struct htmlflt * _hf_check_ent(struct htmlflt *self, char const *s,
 697                            size_t l);
 698
 699 /* Input handler */
 700 static ssize_t          _hf_add_data(struct htmlflt *self,
 701                            char const *dat, size_t len);
 702
 703 static struct htmlflt *
 704 _hf_dump_hrefs(struct htmlflt *self)
 705 {
 706    struct htmlflt_href *hhp;
 707    NYD2_ENTER;
 708
 709    if (!(self->hf_flags & _HF_NL_2) && putc('\n', self->hf_os) == EOF) {
 710       self->hf_flags |= _HF_ERROR;
 711       goto jleave;
 712    }
 713
 714    /* Reverse the list */
 715    for (hhp = self->hf_hrefs, self->hf_hrefs = NULL; hhp != NULL;) {
 716       struct htmlflt_href *tmp = hhp->hfh_next;
 717       hhp->hfh_next = self->hf_hrefs;
 718       self->hf_hrefs = hhp;
 719       hhp = tmp;
 720    }
 721
 722    /* Then dump it */
 723    while ((hhp = self->hf_hrefs) != NULL) {
 724       self->hf_hrefs = hhp->hfh_next;
 725
 726       if (!(self->hf_flags & _HF_ERROR)) {
 727          int w = fprintf(self->hf_os, "  [%u] %.*s\n",
 728                hhp->hfh_no, (int)hhp->hfh_len, hhp->hfh_dat);
 729          if (w < 0)
 730             self->hf_flags |= _HF_ERROR;
 731       }
 732       free(hhp);
 733    }
 734
 735    self->hf_flags |= (putc('\n', self->hf_os) == EOF)
 736          ?  _HF_ERROR : _HF_NL_1 | _HF_NL_2;
 737    self->hf_href_dist = (ui32_t)n_realscreenheight >> 1;
 738 jleave:
 739    NYD2_LEAVE;
 740    return self;
 741 }
 742
 743 static struct htmlflt *
 744 _hf_dump(struct htmlflt *self)
 745 {
 746    ui32_t f, l;
 747    char c, *cp;
 748    NYD2_ENTER;
 749
 750    f = self->hf_flags & ~_HF_BLANK;
 751    l = self->hf_len;
 752    cp = self->hf_line;
 753    self->hf_mbwidth = self->hf_mboff = self->hf_last_ws = self->hf_len = 0;
 754
 755    for (c = '\0'; l > 0; --l) {
 756       c = *cp++;
 757 jput:
 758       if (putc(c, self->hf_os) == EOF) {
 759          self->hf_flags = (f |= _HF_ERROR);
 760          goto jleave;
 761       }
 762    }
 763
 764    if (c != '\n') {
 765       f |= (f & _HF_NL_1) ? _HF_NL_2 : _HF_NL_1;
 766       l = 1;
 767       c = '\n';
 768       goto jput;
 769    }
 770    self->hf_flags = f;
 771
 772    /* Check whether there are HREFs to dump; there is so much messy tagsoup out
 773     * there that it seems best not to simply dump HREFs in each _dump(), but
 774     * only with some gap, let's say half the real screen height */
 775    if (--self->hf_href_dist < 0 && (f & _HF_NL_2) && self->hf_hrefs != NULL)
 776       self = _hf_dump_hrefs(self);
 777 jleave:
 778    NYD2_LEAVE;
 779    return self;
 780 }
 781
 782 static struct htmlflt *
 783 _hf_store(struct htmlflt *self, char c)
 784 {
 785    ui32_t f, l, i;
 786    NYD2_ENTER;
 787
 788    assert(c != '\n');
 789
 790    f = self->hf_flags;
 791    l = self->hf_len;
 792    self->hf_line[l] = (c == '\t' ? ' ' : c);
 793    self->hf_len = ++l;
 794    if (blankspacechar(c)) {
 795       if (c == '\t') {
 796          i = 8 - ((l - 1) & 7); /* xxx magic tab width of 8 */
 797          if (i > 0) {
 798             do
 799                self = _hf_store(self, ' ');
 800             while (--i > 0);
 801             goto jleave;
 802          }
 803       }
 804       self->hf_last_ws = l;
 805    } else if (/*c == '.' ||*/ c == ',' || c == ';' || c == '-')
 806       self->hf_last_ws = l;
 807
 808    i = l;
 809 # ifdef HAVE_NATCH_CHAR /* XXX This code is really ridiculous! */
 810    if (n_mb_cur_max > 1) { /* XXX should mbrtowc() and THEN store, at least */
 811       wchar_t wc;
 812       int w, x;
 813
 814       if((x = mbtowc(&wc, self->hf_line + self->hf_mboff, l - self->hf_mboff)
 815             ) > 0){
 816          if ((w = wcwidth(wc)) == -1 ||
 817                /* Actively filter out L-TO-R and R-TO-R marks TODO ctext */
 818                (wc == 0x200E || wc == 0x200F ||
 819                   (wc >= 0x202A && wc <= 0x202E)) ||
 820                /* And some zero-width messes */
 821                wc == 0x00AD || (wc >= 0x200B && wc <= 0x200D) ||
 822                /* Oh about the ISO C wide character interfaces, baby! */
 823                (wc == 0xFEFF)){
 824             self->hf_len -= x;
 825             goto jleave;
 826          } else if (iswspace(wc))
 827             self->hf_last_ws = l;
 828          self->hf_mboff += x;
 829          i = (self->hf_mbwidth += w);
 830       } else {
 831          if (x < 0) {
 832             (void)mbtowc(&wc, NULL, n_mb_cur_max);
 833             if (UICMP(32, l - self->hf_mboff, >=, n_mb_cur_max)) { /* XXX */
 834                ++self->hf_mboff;
 835                ++self->hf_mbwidth;
 836             }
 837          }
 838          i = self->hf_mbwidth;
 839       }
 840    }
 841 # endif
 842
 843    /* Do we need to break the line? */
 844    if (i >= self->hf_lmax - _HF_BRKSUB) {
 845       ui32_t lim = self->hf_lmax >> 1;
 846
 847       /* Let's hope we saw a sane place to break this line! */
 848       if (self->hf_last_ws >= lim) {
 849 jput:
 850          i = self->hf_len = self->hf_last_ws;
 851          self = _hf_dump(self);
 852          if ((self->hf_len = (l -= i)) > 0) {
 853             self->hf_flags &= ~_HF_NL_MASK;
 854             memmove(self->hf_line, self->hf_line + i, l);
 855 # ifdef HAVE_NATCH_CHAR
 856             __hf_sync_mbstuff(self);
 857 # endif
 858          }
 859          goto jleave;
 860       }
 861
 862       /* Any 7-bit characters? */
 863       for (i = l; i-- >= lim;)
 864          if (asciichar((c = self->hf_line[i]))) {
 865             self->hf_last_ws = ++i;
 866             goto jput;
 867          } else if ((f & _HF_UTF8) && ((ui8_t)c & 0xC0) != 0x80) {
 868             self->hf_last_ws = i;
 869             goto jput;
 870          }
 871
 872       /* Hard break necessary!  xxx really badly done */
 873       if (l >= self->hf_lmax - 1)
 874          self = _hf_dump(self);
 875    }
 876 jleave:
 877    NYD2_LEAVE;
 878    return self;
 879 }
 880
 881 # ifdef HAVE_NATCH_CHAR
 882 static struct htmlflt *
 883 __hf_sync_mbstuff(struct htmlflt *self)
 884 {
 885    wchar_t wc;
 886    char const *b;
 887    ui32_t o, w, l;
 888    NYD2_ENTER;
 889
 890    b = self->hf_line;
 891    o = w = 0;
 892    l = self->hf_len;
 893    goto jumpin;
 894
 895    while (l > 0) {
 896       int x = mbtowc(&wc, b, l);
 897
 898       if (x == 0)
 899          break;
 900
 901       if (x > 0) {
 902          b += x;
 903          l -= x;
 904          o += x;
 905          if ((x = wcwidth(wc)) == -1)
 906             x = 1;
 907          w += x;
 908          continue;
 909       }
 910
 911       /* Bad, skip over a single character.. XXX very bad indeed */
 912       ++b;
 913       ++o;
 914       ++w;
 915       --l;
 916 jumpin:
 917       (void)mbtowc(&wc, NULL, n_mb_cur_max);
 918    }
 919
 920    self->hf_mboff = o;
 921    self->hf_mbwidth = w;
 922
 923    NYD2_LEAVE;
 924    return self;
 925 }
 926 # endif /* HAVE_NATCH_CHAR */
 927
 928 static struct htmlflt *
 929 _hf_nl(struct htmlflt *self)
 930 {
 931    ui32_t f;
 932    NYD2_ENTER;
 933
 934    if (!((f = self->hf_flags) & _HF_ERROR)) {
 935       if (f & _HF_ANY) {
 936          if ((f & _HF_NL_MASK) != _HF_NL_MASK)
 937             self = _hf_dump(self);
 938       } else
 939          self->hf_flags = (f |= _HF_NL_MASK);
 940    }
 941    NYD2_LEAVE;
 942    return self;
 943 }
 944
 945 static struct htmlflt *
 946 _hf_nl_force(struct htmlflt *self)
 947 {
 948    NYD2_ENTER;
 949    if (!(self->hf_flags & _HF_ERROR))
 950       self = _hf_dump(self);
 951    NYD2_LEAVE;
 952    return self;
 953 }
 954
 955 static struct htmlflt *
 956 _hf_putc(struct htmlflt *self, char c)
 957 {
 958    ui32_t f;
 959    NYD2_ENTER;
 960
 961    if ((f = self->hf_flags) & _HF_ERROR)
 962       goto jleave;
 963
 964    if (c == '\n') {
 965       self = _hf_nl(self);
 966       goto jleave;
 967    } else if (c == ' ' || c == '\t') {
 968       if ((f & _HF_BLANK) || self->hf_len == 0)
 969          goto jleave;
 970       f |= _HF_BLANK;
 971    } else
 972       f &= ~_HF_BLANK;
 973    f &= ~_HF_NL_MASK;
 974    self->hf_flags = (f |= _HF_ANY);
 975    self = _hf_store(self, c);
 976 jleave:
 977    NYD2_LEAVE;
 978    return self;
 979 }
 980
 981 static struct htmlflt *
 982 _hf_putc_premode(struct htmlflt *self, char c)
 983 {
 984    ui32_t f;
 985    NYD2_ENTER;
 986
 987    if ((f = self->hf_flags) & _HF_ERROR) {
 988       ;
 989    } else if (c == '\n')
 990       self = _hf_nl_force(self);
 991    else {
 992       f &= ~_HF_NL_MASK;
 993       self->hf_flags = (f |= _HF_ANY);
 994       self = _hf_store(self, c);
 995    }
 996    NYD2_LEAVE;
 997    return self;
 998 }
 999
1000 static struct htmlflt *
1001 _hf_puts(struct htmlflt *self, char const *cp)
1002 {
1003    char c;
1004    NYD2_ENTER;
1005
1006    while ((c = *cp++) != '\0')
1007       self = _hf_putc(self, c);
1008    NYD2_LEAVE;
1009    return self;
1010 }
1011
1012 static struct htmlflt *
1013 _hf_putbuf(struct htmlflt *self, char const *cp, size_t len)
1014 {
1015    NYD2_ENTER;
1016
1017    while (len-- > 0)
1018       self = _hf_putc(self, *cp++);
1019    NYD2_LEAVE;
1020    return self;
1021 }
1022
1023 static struct htmlflt *
1024 _hf_param(struct htmlflt *self, struct str *store, char const *param)
1025 {
1026    char const *cp;
1027    char c, x, quote;
1028    size_t i;
1029    bool_t hot;
1030    NYD2_ENTER;
1031
1032    store->s = NULL;
1033    store->l = 0;
1034    cp = self->hf_bdat;
1035
1036    /* Skip over any non-WS first; be aware of soup, if it slipped through */
1037    for(;;){
1038       if((c = *cp++) == '\0' || c == '>')
1039          goto jleave;
1040       if(whitechar(c))
1041          break;
1042    }
1043
1044    /* Search for the parameter, take care of other quoting along the way */
1045    x = *param++;
1046    x = upperconv(x);
1047    i = strlen(param);
1048
1049    for(hot = TRU1;;){
1050       if((c = *cp++) == '\0' || c == '>')
1051          goto jleave;
1052       if(whitechar(c)){
1053          hot = TRU1;
1054          continue;
1055       }
1056
1057       /* Could it be a parameter? */
1058       if(hot){
1059          hot = FAL0;
1060
1061          /* Is it the desired one? */
1062          if((c = upperconv(c)) == x && !ascncasecmp(param, cp, i)){
1063             char const *cp2 = cp + i;
1064
1065             if((quote = *cp2++) != '='){
1066                if(quote == '\0' || quote == '>')
1067                   goto jleave;
1068                while(whitechar(quote))
1069                   quote = *cp2++;
1070             }
1071             if(quote == '='){
1072                cp = cp2;
1073                break;
1074             }
1075             continue; /* XXX Optimize: i bytes or even cp2 can't be it! */
1076          }
1077       }
1078
1079       /* Not the desired one; but a parameter? */
1080       if(c != '=')
1081          continue;
1082       /* If so, properly skip over the value */
1083       if((c = *cp++) == '"' || c == '\''){
1084          /* TODO i have forgotten whether reverse solidus quoting is allowed
1085           * TODO quoted HTML parameter values?  not supporting that for now.. */
1086          for(quote = c; (c = *cp++) != '\0' && c != quote;)
1087             ;
1088       }else
1089          while(c != '\0' && !whitechar(c) && c != '>')
1090             c = *++cp;
1091       if(c == '\0')
1092          goto jleave;
1093    }
1094
1095    /* Skip further whitespace */
1096    for(;;){
1097       if((c = *cp++) == '\0' || c == '>')
1098          goto jleave;
1099       if(!whitechar(c))
1100          break;
1101    }
1102
1103    if(c == '"' || c == '\''){
1104       /* TODO i have forgotten whether reverse solisud quoting is allowed in
1105        * TODO quoted HTML parameter values?  not supporting that for now.. */
1106       store->s = n_UNCONST(cp);
1107       for(quote = c; (c = *cp) != '\0' && c != quote; ++cp)
1108          ;
1109       /* XXX ... and we simply ignore a missing trailing " :> */
1110    }else{
1111       store->s = n_UNCONST(cp - 1);
1112       if(!whitechar(c))
1113          while((c = *cp) != '\0' && !whitechar(c) && c != '>')
1114             ++cp;
1115    }
1116    i = PTR2SIZE(cp - store->s);
1117
1118    /* Terrible tagsoup out there, e.g., groups.google.com produces href=""
1119     * parameter values prefixed and suffixed by newlines!  Therefore trim the
1120     * value content TODO join into the parse step above! */
1121    for (cp = store->s; i > 0 && spacechar(*cp); ++cp, --i)
1122       ;
1123    store->s = n_UNCONST(cp);
1124    for (cp += i - 1; i > 0 && spacechar(*cp); --cp, --i)
1125       ;
1126    if ((store->l = i) == 0)
1127       store->s = NULL;
1128 jleave:
1129    NYD2_LEAVE;
1130    return self;
1131 }
1132
1133 static struct htmlflt *
1134 _hf_expand_all_ents(struct htmlflt *self, struct str const *param)
1135 {
1136    char const *cp, *maxcp, *ep;
1137    char c;
1138    size_t i;
1139    NYD2_ENTER;
1140
1141    for (cp = param->s, maxcp = cp + param->l; cp < maxcp;)
1142       if ((c = *cp++) != '&')
1143 jputc:
1144          self = _hf_putc(self, c);
1145       else {
1146          for (ep = cp--;;) {
1147             if (ep == maxcp || (c = *ep++) == '\0') {
1148                for (; cp < ep; ++cp)
1149                   self = _hf_putc(self, *cp);
1150                goto jleave;
1151             } else if (c == ';') {
1152                if ((i = PTR2SIZE(ep - cp)) > 1) {
1153                   self = _hf_check_ent(self, cp, i);
1154                   break;
1155                } else {
1156                   c = *cp++;
1157                   goto jputc;
1158                }
1159             }
1160          }
1161          cp = ep;
1162       }
1163 jleave:
1164    NYD2_LEAVE;
1165    return self;
1166 }
1167
1168 static struct htmlflt *
1169 _hf_check_tag(struct htmlflt *self, char const *s)
1170 {
1171    char nobuf[32], c;
1172    struct str param;
1173    size_t i;
1174    struct htmlflt_tag const *hftp;
1175    ui32_t f;
1176    NYD2_ENTER;
1177
1178    /* Extra check only */
1179    assert(s != NULL);
1180    if (*s != '<') {
1181       DBG( n_alert("HTML tagsoup filter _hf_check_tag() called on soup!"); )
1182 jput_as_is:
1183       self = _hf_puts(self, self->hf_bdat);
1184       goto jleave;
1185    }
1186
1187    for (++s, i = 0; (c = s[i]) != '\0' && c != '>' && !whitechar(c); ++i)
1188       /* Special massage for things like <br/>: after the slash only whitespace
1189        * may separate us from the closing right angle! */
1190       if (c == '/') {
1191          size_t j = i + 1;
1192
1193          while ((c = s[j]) != '\0' && c != '>' && whitechar(c))
1194             ++j;
1195          if (c == '>')
1196             break;
1197       }
1198
1199    for (hftp = _hf_tags;;) {
1200       if (i == hftp->hft_len && !ascncasecmp(s, hftp->hft_tag, i)) {
1201          c = s[hftp->hft_len];
1202          if (c == '>' || c == '/' || whitechar(c))
1203             break;
1204       }
1205       if (PTRCMP(++hftp, >=, _hf_tags + n_NELEM(_hf_tags)))
1206          goto jnotknown;
1207    }
1208
1209    f = self->hf_flags;
1210    switch (hftp->hft_act) {
1211    case _HFSA_PRE_END:
1212       f &= ~_HF_PRE;
1213       if (0) {
1214          /* FALLTHRU */
1215    case _HFSA_PRE:
1216          f |= _HF_PRE;
1217       }
1218       self->hf_flags = f;
1219       /* FALLTHRU */
1220
1221    case _HFSA_NEEDSEP:
1222       if (!(self->hf_flags & _HF_NL_2))
1223          self = _hf_nl(self);
1224       /* FALLTHRU */
1225    case _HFSA_NEEDNL:
1226       if (!(f & _HF_NL_1))
1227          self = _hf_nl(self);
1228       if (hftp->hft_injc != '\0') {
1229          self = _hf_putc(self, hftp->hft_injc & 0x7F);
1230          if ((uc_i)hftp->hft_injc & 0x80)
1231             self = _hf_putc(self, ' ');
1232       }
1233       break;
1234
1235    case _HFSA_IGN:
1236       self->hf_ign_tag = hftp;
1237       self->hf_flags = (f |= _HF_IGN | _HF_NOPUT);
1238       break;
1239
1240    case _HFSA_IMG:
1241       self = _hf_param(self, &param, "alt");
1242       self = _hf_putc(self, '[');
1243       if (param.s == NULL) {
1244          param.s = n_UNCONST("IMG");
1245          param.l = 3;
1246          goto jimg_put;
1247       } /* else */ if (memchr(param.s, '&', param.l) != NULL)
1248          self = _hf_expand_all_ents(self, &param);
1249       else
1250 jimg_put:
1251          self = _hf_putbuf(self, param.s, param.l);
1252       self = _hf_putc(self, ']');
1253       break;
1254
1255    case _HFSA_HREF:
1256       self = _hf_param(self, &param, "href");
1257       /* Ignore non-external links */
1258       if (param.s != NULL && *param.s != '#') {
1259          struct htmlflt_href *hhp = smalloc(
1260                n_VSTRUCT_SIZEOF(struct htmlflt_href, hfh_dat) + param.l +1);
1261
1262          hhp->hfh_next = self->hf_hrefs;
1263          hhp->hfh_no = ++self->hf_href_no;
1264          hhp->hfh_len = (ui32_t)param.l;
1265          memcpy(hhp->hfh_dat, param.s, param.l);
1266
1267          snprintf(nobuf, sizeof nobuf, "[%u]", hhp->hfh_no);
1268          self->hf_flags = (f |= _HF_HREF);
1269          self->hf_hrefs = hhp;
1270          self = _hf_puts(self, nobuf);
1271       } else
1272          self->hf_flags = (f &= ~_HF_HREF);
1273       break;
1274    case _HFSA_HREF_END:
1275       if (f & _HF_HREF) {
1276          snprintf(nobuf, sizeof nobuf, "[/%u]", self->hf_href_no);
1277          self = _hf_puts(self, nobuf);
1278       }
1279       break;
1280
1281    default:
1282       c = (char)(hftp->hft_act & 0xFF);
1283       self = _hf_putc(self, c);
1284       break;
1285    case '\0':
1286       break;
1287    }
1288 jleave:
1289    NYD2_LEAVE;
1290    return self;
1291
1292    /* The problem is that even invalid tagsoup is widely used, without real
1293     * searching i have seen e-mail address in <N@H.D> notation, and more.
1294     * To protect us a bit look around and possibly write the content as such */
1295 jnotknown:
1296    switch (*s) {
1297    case '!':
1298    case '?':
1299       /* Ignore <!DOCTYPE, <!-- comments, <? PIs.. */
1300       goto jleave;
1301    case '>':
1302       /* Print out an empty tag as such */
1303       if (s[1] == '\0') {
1304          --s;
1305          goto jput_as_is;
1306       }
1307       break;
1308    case '/':
1309       ++s;
1310       break;
1311    default:
1312       break;
1313    }
1314
1315    /* Also skip over : in order to suppress v:roundrect, w:anchorlock.. */
1316    while ((c = *s++) != '\0' && c != '>' && !whitechar(c) && c != ':')
1317       if (!asciichar(c) || punctchar(c)) {
1318          self = _hf_puts(self, self->hf_bdat);
1319          break;
1320       }
1321    goto jleave;
1322 }
1323
1324 static struct htmlflt *
1325 _hf_check_ent(struct htmlflt *self, char const *s, size_t l)
1326 {
1327    char nobuf[32];
1328    char const *s_save;
1329    size_t l_save;
1330    struct hf_ent const *hfep;
1331    size_t i;
1332    NYD2_ENTER;
1333
1334    s_save = s;
1335    l_save = l;
1336    assert(*s == '&');
1337    assert(l > 0);
1338    /* False entities seen in the wild assert(s[l - 1] == ';'); */
1339    ++s;
1340    l -= 2;
1341
1342    /* Numeric entity, or try named search */
1343    if (*s == '#') {
1344       i = (*++s == 'x' ? 16 : 10);
1345
1346       if ((i != 16 || (++s, --l) > 0) && l < sizeof(nobuf)) {
1347          memcpy(nobuf, s, l);
1348          nobuf[l] = '\0';
1349          n_idec_uiz_cp(&i, nobuf, i, NULL);
1350          if (i <= 0x7F)
1351             self = _hf_putc(self, (char)i);
1352          else if (self->hf_flags & _HF_UTF8) {
1353 jputuni:
1354             l = n_utf32_to_utf8((ui32_t)i, nobuf);
1355             self = _hf_putbuf(self, nobuf, l);
1356          } else
1357             goto jeent;
1358       } else
1359          goto jeent;
1360    } else {
1361       ui32_t f = self->hf_flags, hf;
1362
1363       for (hfep = _hf_ents; PTRCMP(hfep, <, _hf_ents + n_NELEM(_hf_ents));
1364             ++hfep)
1365          if (l == ((hf = hfep->hfe_flags) & _HFE_LENGTH_MASK) &&
1366                !strncmp(s, hfep->hfe_ent, l)) {
1367             if ((hf & _HFE_HAVE_UNI) && (f & _HF_UTF8)) {
1368                i = hfep->hfe_uni;
1369                goto jputuni;
1370             } else if (hf & _HFE_HAVE_CSTR)
1371                self = _hf_puts(self, hfep->hfe_cstr);
1372             else
1373                self = _hf_putc(self, hfep->hfe_c);
1374             goto jleave;
1375          }
1376 jeent:
1377       self = _hf_putbuf(self, s_save, l_save);
1378    }
1379 jleave:
1380    NYD2_LEAVE;
1381    return self;
1382 }
1383
1384 static ssize_t
1385 _hf_add_data(struct htmlflt *self, char const *dat, size_t len)
1386 {
1387    char c, *cp, *cp_max;
1388    bool_t hot;
1389    ssize_t rv = 0;
1390    NYD_ENTER;
1391
1392    /* Final put request? */
1393    if (dat == NULL) {
1394       if (self->hf_len > 0 || self->hf_hrefs != NULL) {
1395          self = _hf_dump(self);
1396          if (self->hf_hrefs != NULL)
1397             self = _hf_dump_hrefs(self);
1398          rv = 1;
1399       }
1400       goto jleave;
1401    }
1402
1403    /* Always ensure some initial buffer */
1404    if ((cp = self->hf_curr) != NULL)
1405       cp_max = self->hf_bmax;
1406    else {
1407       cp = self->hf_curr = self->hf_bdat = smalloc(LINESIZE);
1408       cp_max = self->hf_bmax = cp + LINESIZE -1; /* (Always room for NUL!) */
1409    }
1410    hot = (cp != self->hf_bdat);
1411
1412    for (rv = (ssize_t)len; len > 0; --len) {
1413       ui32_t f = self->hf_flags;
1414
1415       if (f & _HF_ERROR)
1416          break;
1417       c = *dat++;
1418
1419       /* Soup is really weird, and scripts may contain almost anything (and
1420        * newer CSS standards are also cryptic): therefore prefix the _HF_IGN
1421        * test and walk until we see the required end tag */
1422       /* TODO For real safety _HF_IGN soup condome would also need to know
1423        * TODO about quoted strings so that 'var i = "</script>";' couldn't
1424        * TODO fool it!   We really want this mode also for _HF_NOPUT to be
1425        * TODO able to *gracefully* detect the tag-closing '>', but then if
1426        * TODO that is a single mechanism we should have made it! */
1427       if (f & _HF_IGN) {
1428          struct htmlflt_tag const *hftp = self->hf_ign_tag;
1429          size_t i;
1430
1431          if (c == '<') {
1432             hot = TRU1;
1433 jcp_reset:
1434             cp = self->hf_bdat;
1435          } else if (c == '>') {
1436             if (hot) {
1437                if ((i = PTR2SIZE(cp - self->hf_bdat)) > 1 &&
1438                      --i == hftp->hft_len &&
1439                      !ascncasecmp(self->hf_bdat + 1, hftp->hft_tag, i))
1440                   self->hf_flags = (f &= ~(_HF_IGN | _HF_NOPUT));
1441                hot = FAL0;
1442                goto jcp_reset;
1443             }
1444          } else if (hot) {
1445             *cp++ = c;
1446             i = PTR2SIZE(cp - self->hf_bdat);
1447             if ((i == 1 && c != '/') || --i > hftp->hft_len) {
1448                hot = FAL0;
1449                goto jcp_reset;
1450             }
1451          }
1452       } else switch (c) {
1453       case '<':
1454          /* People are using & without &amp;ing it, ditto <; be aware */
1455          if (f & (_HF_NOPUT | _HF_ENT)) {
1456             f &= ~_HF_ENT;
1457             /* Special case "<!--" buffer content to deal with really weird
1458              * things that can be done with "<!--[if gte mso 9]>" syntax */
1459             if (PTR2SIZE(cp - self->hf_bdat) != 4 ||
1460                   memcmp(self->hf_bdat, "<!--", 4)) {
1461                self->hf_flags = f;
1462                *cp = '\0';
1463                self = _hf_puts(self, self->hf_bdat);
1464                f = self->hf_flags;
1465             }
1466          }
1467          cp = self->hf_bdat;
1468          *cp++ = c;
1469          self->hf_flags = (f |= _HF_NOPUT);
1470          break;
1471       case '>':
1472          /* Weird tagsoup around, do we actually parse a tag? */
1473          if (!(f & _HF_NOPUT))
1474             goto jdo_c;
1475          cp[0] = c;
1476          cp[1] = '\0';
1477          f &= ~(_HF_NOPUT | _HF_ENT);
1478          self->hf_flags = f;
1479          self = _hf_check_tag(self, self->hf_bdat);
1480          *(cp = self->hf_bdat) = '\0'; /* xxx extra safety */
1481          /* Quick hack to get rid of redundant newline after <pre> XXX */
1482          if (!(f & _HF_PRE) && (self->hf_flags & _HF_PRE) &&
1483                len > 1 && *dat == '\n')
1484             ++dat, --len;
1485          break;
1486
1487       case '\r': /* TODO CR should be stripped in lower level!! (Only B64!?!) */
1488          break;
1489       case '\n':
1490          /* End of line is not considered unless we are in PRE section.
1491           * However, in _HF_NOPUT mode we must be aware of tagsoup which uses
1492           * newlines for separating parameters */
1493          if (f & _HF_NOPUT)
1494             goto jdo_c;
1495          self = (f & _HF_PRE) ? _hf_nl_force(self) : _hf_putc(self, ' ');
1496          break;
1497
1498       case '\t':
1499          if (!(f & _HF_PRE))
1500             c = ' ';
1501          /* FALLTHRU */
1502       default:
1503 jdo_c:
1504          /* If not currently parsing a tag and bypassing normal output.. */
1505          if (!(f & _HF_NOPUT)) {
1506             if (cntrlchar(c))
1507                break;
1508             if (c == '&') {
1509                cp = self->hf_bdat;
1510                *cp++ = c;
1511                self->hf_flags = (f |= _HF_NOPUT | _HF_ENT);
1512             } else if (f & _HF_PRE) {
1513                self = _hf_putc_premode(self, c);
1514                self->hf_flags &= ~_HF_BLANK;
1515             } else
1516               self = _hf_putc(self, c);
1517          } else if ((f & _HF_ENT) && c == ';') {
1518             cp[0] = c;
1519             cp[1] = '\0';
1520             f &= ~(_HF_NOPUT | _HF_ENT);
1521             self->hf_flags = f;
1522            self = _hf_check_ent(self, self->hf_bdat,
1523                PTR2SIZE(cp + 1 - self->hf_bdat));
1524          } else {
1525             /* We may need to grow the buffer */
1526             if (PTRCMP(cp + 42/2, >=, cp_max)) {
1527                size_t i = PTR2SIZE(cp - self->hf_bdat),
1528                   m = PTR2SIZE(self->hf_bmax - self->hf_bdat) + LINESIZE;
1529
1530                cp = self->hf_bdat = srealloc(self->hf_bdat, m);
1531                self->hf_bmax = cp + m -1;
1532                self->hf_curr = (cp += i);
1533             }
1534             *cp++ = c;
1535          }
1536       }
1537    }
1538    self->hf_curr = cp;
1539 jleave:
1540   NYD_LEAVE;
1541   return (self->hf_flags & _HF_ERROR) ? -1 : rv;
1542 }
1543
1544 /*
1545  * TODO Because we don't support filter chains yet this filter will be run
1546  * TODO in a dedicated subprocess, driven via a special Popen() mode
1547  */
1548 static bool_t __hf_hadpipesig;
1549 static void
1550 __hf_onpipe(int signo)
1551 {
1552    NYD_X; /* Signal handler */
1553    n_UNUSED(signo);
1554    __hf_hadpipesig = TRU1;
1555 }
1556
1557 FL int
1558 htmlflt_process_main(void)
1559 {
1560    char buf[BUFFER_SIZE];
1561    struct htmlflt hf;
1562    size_t i;
1563    int rv;
1564    NYD_ENTER;
1565
1566    __hf_hadpipesig = FAL0;
1567    safe_signal(SIGPIPE, &__hf_onpipe);
1568
1569    htmlflt_init(&hf);
1570    htmlflt_reset(&hf, n_stdout);
1571
1572    for (;;) {
1573       if ((i = fread(buf, sizeof(buf[0]), n_NELEM(buf), n_stdin)) == 0) {
1574          rv = !feof(n_stdin);
1575          break;
1576       }
1577
1578       if ((rv = __hf_hadpipesig))
1579          break;
1580       /* Just use this directly.. */
1581       if (htmlflt_push(&hf, buf, i) < 0) {
1582          rv = 1;
1583          break;
1584       }
1585    }
1586    if (rv == 0 && htmlflt_flush(&hf) < 0)
1587       rv = 1;
1588
1589    htmlflt_destroy(&hf);
1590
1591    rv |= __hf_hadpipesig;
1592    NYD_LEAVE;
1593    return rv;
1594 }
1595
1596 FL void
1597 htmlflt_init(struct htmlflt *self)
1598 {
1599    NYD_ENTER;
1600    /* (Rather redundant though) */
1601    memset(self, 0, sizeof *self);
1602    NYD_LEAVE;
1603 }
1604
1605 FL void
1606 htmlflt_destroy(struct htmlflt *self)
1607 {
1608    NYD_ENTER;
1609    htmlflt_reset(self, NULL);
1610    NYD_LEAVE;
1611 }
1612
1613 FL void
1614 htmlflt_reset(struct htmlflt *self, FILE *f)
1615 {
1616    struct htmlflt_href *hfhp;
1617    NYD_ENTER;
1618
1619    while ((hfhp = self->hf_hrefs) != NULL) {
1620       self->hf_hrefs = hfhp->hfh_next;
1621       free(hfhp);
1622    }
1623
1624    if (self->hf_bdat != NULL)
1625       free(self->hf_bdat);
1626    if (self->hf_line != NULL)
1627       free(self->hf_line);
1628
1629    memset(self, 0, sizeof *self);
1630
1631    if (f != NULL) {
1632       ui32_t sw = n_MAX(_HF_MINLEN, (ui32_t)n_scrnwidth);
1633
1634       self->hf_line = smalloc((size_t)sw * n_mb_cur_max +1);
1635       self->hf_lmax = sw;
1636
1637       if (n_psonce & n_PSO_UNICODE) /* TODO not truly generic */
1638          self->hf_flags = _HF_UTF8;
1639       self->hf_os = f;
1640    }
1641    NYD_LEAVE;
1642 }
1643
1644 FL ssize_t
1645 htmlflt_push(struct htmlflt *self, char const *dat, size_t len)
1646 {
1647    ssize_t rv;
1648    NYD_ENTER;
1649
1650    rv = _hf_add_data(self, dat, len);
1651    NYD_LEAVE;
1652    return rv;
1653 }
1654
1655 FL ssize_t
1656 htmlflt_flush(struct htmlflt *self)
1657 {
1658    ssize_t rv;
1659    NYD_ENTER;
1660
1661    rv = _hf_add_data(self, NULL, 0);
1662    rv |= !fflush(self->hf_os) ? 0 : -1;
1663    NYD_LEAVE;
1664    return rv;
1665 }
1666 #endif /* HAVE_FILTER_HTML_TAGSOUP */
1667
1668 /* s-it-mode */