pith/text.c

   1 /*
   2  * ========================================================================
   3  * Copyright 2013-2022 Eduardo Chappa
   4  * Copyright 2006-2008 University of Washington
   5  *
   6  * Licensed under the Apache License, Version 2.0 (the "License");
   7  * you may not use this file except in compliance with the License.
   8  * You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * ========================================================================
  13  */
  14
  15 /*======================================================================
  16
  17      text.c
  18      Implements message text fetching and decoding
  19
  20   ====*/
  21
  22
  23 #include "../pith/headers.h"
  24 #include "../pith/text.h"
  25 #include "../pith/state.h"
  26 #include "../pith/conf.h"
  27 #include "../pith/handle.h"
  28 #include "../pith/margin.h"
  29 #include "../pith/editorial.h"
  30 #include "../pith/color.h"
  31 #include "../pith/filter.h"
  32 #include "../pith/charset.h"
  33 #include "../pith/status.h"
  34 #include "../pith/mailview.h"
  35 #include "../pith/mimedesc.h"
  36 #include "../pith/detach.h"
  37
  38
  39 /* internal prototypes */
  40 int      charset_editorial(char *, long, HANDLE_S **, int, int, int, gf_io_t);
  41 int      replace_quotes(long, char *, LT_INS_S **, void *);
  42 void     mark_handles_in_line(char *, HANDLE_S **, int);
  43 void     clear_html_risk(void);
  44 void     set_html_risk(void);
  45 int      get_html_risk(void);
  46
  47
  48 #define CHARSET_DISCLAIMER_1    \
  49         "The following text is in the \"%.40s\" character set.\015\012Your "
  50 #define CHARSET_DISCLAIMER_2    "display is set"
  51 #define CHARSET_DISCLAIMER_3    \
  52        " for the \"%.40s\" character set. \015\012Some %.40scharacters may be displayed incorrectly."
  53 #define ENCODING_DISCLAIMER     \
  54         "The following text contains the unknown encoding type \"%.20s\". \015\012Some or all of the text may be displayed incorrectly."
  55 #define HTML_WARNING            \
  56         "The following HTML text may contain deceptive links.  Carefully note the destination URL before visiting any links."
  57
  58
  59 /*
  60  * HTML risk state holder.
  61  */
  62 static int _html_risk;
  63
  64
  65
  66 /*----------------------------------------------------------------------
  67    Handle fetching and filtering a text message segment to be displayed
  68    by scrolltool or printed or exported or piped.
  69
  70 Args: att   -- segment to fetch
  71       msgno -- message number segment is a part of
  72       pc    -- function to write characters from segment with
  73       style -- Indicates special handling for error messages
  74       flags -- Indicates special necessary handling
  75
  76 Returns: 1 if errors encountered, 0 if everything went A-OK
  77
  78  ----*/
  79 int
  80 decode_text(ATTACH_S        *att,
  81             long int         msgno,
  82             gf_io_t          pc,
  83             HANDLE_S       **handlesp,
  84             DetachErrStyle   style,
  85             int              flags)
  86 {
  87     FILTLIST_S  filters[14];
  88     char       *err, *charset;
  89     int         filtcnt = 0, error_found = 0, column, wrapit;
  90     int         is_in_sig = OUT_SIG_BLOCK;
  91     int         is_flowed_msg = 0;
  92     int         is_delsp_yes = 0;
  93     int         filt_only_c0 = 0;
  94     char       *parmval;
  95     char       *free_this = NULL;
  96     STORE_S    *warn_so = NULL;
  97     DELQ_S      dq;
  98     URL_HILITE_S uh;
  99
 100     column = (flags & FM_DISPLAY) ? ps_global->ttyo->screen_cols : 80;
 101
 102     if(!(flags & FM_DISPLAY))
 103       flags |= FM_NOINDENT;
 104
 105     wrapit = column;
 106
 107     memset(filters, 0, sizeof(filters));
 108
 109     /* charset the body part is in */
 110     charset = parameter_val(att->body->parameter, "charset");
 111
 112     /* determined if it's flowed, affects wrapping and quote coloring */
 113     if(att->body->type == TYPETEXT
 114        && !strucmp(att->body->subtype, "plain")
 115        && (parmval = parameter_val(att->body->parameter, "format"))){
 116         if(!strucmp(parmval, "flowed"))
 117           is_flowed_msg = 1;
 118         fs_give((void **) &parmval);
 119
 120         if(is_flowed_msg){
 121             if((parmval = parameter_val(att->body->parameter, "delsp")) != NULL){
 122                 if(!strucmp(parmval, "yes"))
 123                   is_delsp_yes = 1;
 124
 125                 fs_give((void **) &parmval);
 126             }
 127         }
 128     }
 129
 130     if(!ps_global->pass_ctrl_chars){
 131         filters[filtcnt++].filter = gf_escape_filter;
 132         filters[filtcnt].filter = gf_control_filter;
 133
 134         filt_only_c0 = 1;
 135         filters[filtcnt++].data = gf_control_filter_opt(&filt_only_c0);
 136     }
 137
 138     if(flags & FM_DISPLAY)
 139       filters[filtcnt++].filter = gf_tag_filter;
 140
 141     /*
 142      * if it's just plain old text, look for url's
 143      */
 144     if(!(att->body->subtype && strucmp(att->body->subtype, "plain"))){
 145         struct variable *vars = ps_global->vars;
 146
 147         if((F_ON(F_VIEW_SEL_URL, ps_global)
 148             || F_ON(F_VIEW_SEL_URL_HOST, ps_global)
 149             || F_ON(F_SCAN_ADDR, ps_global))
 150            && handlesp){
 151
 152             /*
 153              * The url_hilite filter really ought to come
 154              * after flowing, because flowing with the DelSp=yes parameter
 155              * can reassemble broken urls back into identifiable urls.
 156              * We add the preflow filter to do only the reassembly part
 157              * of the flowing so that we can spot the urls.
 158              * At this time (2005-03-29) we know that Apple Mail does
 159              * send mail like this sometimes. This filter removes the
 160              * sequence  SP CRLF  if that seems safe.
 161              */
 162             if(ps_global->full_header != 2 && is_delsp_yes)
 163               filters[filtcnt++].filter = gf_preflow;
 164
 165             filters[filtcnt].filter = gf_line_test;
 166             filters[filtcnt++].data = gf_line_test_opt(url_hilite,
 167                                                        gf_url_hilite_opt(&uh,handlesp,0));
 168         }
 169
 170         /*
 171          * First, paint the signature.
 172          * Disclaimers noted below for coloring quotes apply here as well.
 173          */
 174         if((flags & FM_DISPLAY)
 175            && !(flags & FM_NOCOLOR)
 176            && pico_usingcolor()
 177            && VAR_SIGNATURE_FORE_COLOR
 178            && VAR_SIGNATURE_BACK_COLOR){
 179             filters[filtcnt].filter = gf_line_test;
 180             filters[filtcnt++].data = gf_line_test_opt(color_signature,
 181                                                        &is_in_sig);
 182         }
 183
 184         /*
 185          * Gotta be careful with this. The color_a_quote filter adds color
 186          * to the beginning and end of the line. This will break some
 187          * line_test-style filters which come after it. For example, if they
 188          * are looking for something at the start of a line (like color_a_quote
 189          * itself). I guess we could fix that by ignoring tags at the
 190          * beginning of the line when doing the search.
 191          */
 192         if((flags & FM_DISPLAY)
 193            && !(flags & FM_NOCOLOR)
 194            && pico_usingcolor()
 195            && VAR_QUOTE1_FORE_COLOR
 196            && VAR_QUOTE1_BACK_COLOR){
 197             filters[filtcnt].filter = gf_line_test;
 198             filters[filtcnt++].data = gf_line_test_opt(color_a_quote,
 199                                                        &is_flowed_msg);
 200         }
 201     }
 202     else if(!strucmp(att->body->subtype, "richtext")){
 203         int plain_opt;
 204
 205         plain_opt = !(flags&FM_DISPLAY);
 206
 207         /* maybe strip everything! */
 208         filters[filtcnt].filter = gf_rich2plain;
 209         filters[filtcnt++].data = gf_rich2plain_opt(&plain_opt);
 210         /* width to use for file or printer */
 211         if(wrapit - 5 > 0)
 212           wrapit -= 5;
 213     }
 214     else if(!strucmp(att->body->subtype, "enriched")){
 215         int plain_opt;
 216
 217         plain_opt = !(flags&FM_DISPLAY);
 218
 219         filters[filtcnt].filter = gf_enriched2plain;
 220         filters[filtcnt++].data = gf_enriched2plain_opt(&plain_opt);
 221         /* width to use for file or printer */
 222         if(wrapit - 5 > 0)
 223           wrapit -= 5;
 224     }
 225     else if(!strucmp(att->body->subtype, "html") && ps_global->full_header < 2){
 226 /*BUG:      sniff the params for "version=2.0" ala draft-ietf-html-spec-01 */
 227         int      opts = 0;
 228
 229         clear_html_risk();
 230
 231         if(flags & FM_DISPLAY){
 232             gf_io_t      warn_pc;
 233
 234             if(handlesp){               /* pass on handles awareness */
 235                 opts |= GFHP_HANDLES;
 236
 237                 if(F_OFF(F_QUELL_HOST_AFTER_URL, ps_global) && !(flags & FM_HIDESERVER))
 238                   opts |= GFHP_SHOW_SERVER;
 239             }
 240
 241             if(!(flags & FM_NOEDITORIAL)){
 242                 warn_so = so_get(CharStar, NULL, EDIT_ACCESS);
 243                 gf_set_so_writec(&warn_pc, warn_so);
 244                 format_editorial(HTML_WARNING, column, flags, handlesp, warn_pc);
 245                 gf_clear_so_writec(warn_so);
 246                 so_puts(warn_so, "\015\012");
 247                 so_writec('\0', warn_so);
 248             }
 249         }
 250         else
 251           opts |= GFHP_STRIPPED;        /* don't embed anything! */
 252
 253         if(flags & FM_NOHTMLREL)
 254           opts |= GFHP_NO_RELATIVE;
 255
 256         if(flags & FM_HTMLRELATED)
 257           opts |= GFHP_RELATED_CONTENT;
 258
 259         if(flags & FM_HTML)
 260           opts |= GFHP_HTML;
 261
 262         if(flags & FM_HTMLIMAGES)
 263           opts |= GFHP_HTML_IMAGES;
 264
 265         wrapit = 0;             /* wrap already handled! */
 266         filters[filtcnt].filter = gf_html2plain;
 267         filters[filtcnt++].data = gf_html2plain_opt(NULL, column,
 268                                                     (flags & (FM_NOINDENT | FM_HTML))
 269                                                       ? 0
 270                                                       : format_view_margin(),
 271                                                     handlesp, set_html_risk, opts);
 272
 273         if(warn_so){
 274             filters[filtcnt].filter = gf_prepend_editorial;
 275             filters[filtcnt++].data = gf_prepend_editorial_opt(get_html_risk,
 276                                                                (char *) so_text(warn_so));
 277         }
 278     }
 279
 280     /*
 281      * If the message is not flowed, we do the quote suppression before
 282      * the wrapping, because the wrapping does not preserve the quote
 283      * characters at the beginnings of the lines in that case.
 284      * Otherwise, we defer until after the wrapping.
 285      *
 286      * Also, this is a good place to do quote-replacement on nonflowed
 287      * messages because no other filters depend on the "> ".
 288      * Quote-replacement is easier in the flowed case and occurs
 289      * automatically in the flowed wrapping filter.
 290      */
 291     if(!is_flowed_msg
 292        && ps_global->full_header == 0
 293        && !(att->body->subtype && strucmp(att->body->subtype, "plain"))
 294        && (flags & FM_DISPLAY)){
 295         if(ps_global->quote_suppression_threshold != 0){
 296             memset(&dq, 0, sizeof(dq));
 297             dq.lines = ps_global->quote_suppression_threshold;
 298             dq.is_flowed = is_flowed_msg;
 299             dq.indent_length = 0;               /* indent didn't happen yet */
 300             dq.saved_line = &free_this;
 301             dq.handlesp   = handlesp;
 302             dq.do_color   = (!(flags & FM_NOCOLOR) && pico_usingcolor());
 303
 304             filters[filtcnt].filter = gf_line_test;
 305             filters[filtcnt++].data = gf_line_test_opt(delete_quotes, &dq);
 306         }
 307         if(ps_global->VAR_QUOTE_REPLACE_STRING
 308             && F_ON(F_QUOTE_REPLACE_NOFLOW, ps_global)){
 309             filters[filtcnt].filter = gf_line_test;
 310             filters[filtcnt++].data = gf_line_test_opt(replace_quotes, NULL);
 311         }
 312     }
 313
 314     if(wrapit && !(flags & FM_NOWRAP)){
 315         int wrapflags = (flags & FM_DISPLAY) ? (GFW_HANDLES|GFW_SOFTHYPHEN)
 316                                              : GFW_NONE;
 317
 318         if(flags & FM_DISPLAY
 319            && !(flags & FM_NOCOLOR)
 320            && pico_usingcolor())
 321           wrapflags |= GFW_USECOLOR;
 322
 323         /* text/flowed (RFC 2646 + draft)? */
 324         if(ps_global->full_header != 2 && is_flowed_msg){
 325             wrapflags |= GFW_FLOWED;
 326
 327             if(is_delsp_yes)
 328               wrapflags |= GFW_DELSP;
 329         }
 330
 331         filters[filtcnt].filter = gf_wrap;
 332         filters[filtcnt++].data = gf_wrap_filter_opt(wrapit, column,
 333                                                      (flags & FM_NOINDENT)
 334                                                        ? 0
 335                                                        : format_view_margin(),
 336                                                      0, wrapflags);
 337     }
 338
 339     /*
 340      * This has to come after wrapping has happened because the user tells
 341      * us how many quoted lines to display, and we want that number to be
 342      * the wrapped lines, not the pre-wrapped lines. We do it before the
 343      * wrapping if the message is not flowed, because the wrapping does not
 344      * preserve the quote characters at the beginnings of the lines in that
 345      * case.
 346      */
 347     if(is_flowed_msg
 348        && ps_global->full_header == 0
 349        && !(att->body->subtype && strucmp(att->body->subtype, "plain"))
 350        && (flags & FM_DISPLAY)
 351        && ps_global->quote_suppression_threshold != 0){
 352         memset(&dq, 0, sizeof(dq));
 353         dq.lines = ps_global->quote_suppression_threshold;
 354         dq.is_flowed = is_flowed_msg;
 355         dq.indent_length = (wrapit && !(flags & FM_NOWRAP)
 356                             && !(flags & FM_NOINDENT))
 357                                ? format_view_margin()[0]
 358                                : 0;
 359         dq.saved_line = &free_this;
 360         dq.handlesp   = handlesp;
 361         dq.do_color   = (!(flags & FM_NOCOLOR) && pico_usingcolor());
 362
 363         filters[filtcnt].filter = gf_line_test;
 364         filters[filtcnt++].data = gf_line_test_opt(delete_quotes, &dq);
 365     }
 366
 367     if(charset){
 368         if(F_OFF(F_QUELL_CHARSET_WARNING, ps_global)
 369            && !(flags & FM_NOEDITORIAL)){
 370             int rv = TRUE;
 371             CONV_TABLE *ct = NULL;
 372
 373             /*
 374              * Need editorial if message charset is not ascii
 375              * and the display charset is not either the same
 376              * as the message charset or UTF-8.
 377              */
 378             if(strucmp(charset, "us-ascii")
 379                && !((ps_global->display_charmap
 380                      && !strucmp(charset, ps_global->display_charmap))
 381                     || !strucmp("UTF-8", ps_global->display_charmap))){
 382
 383                 /*
 384                  * This is probably overkill. We're just using this
 385                  * conversion_table routine to get at the quality.
 386                  */
 387                 if(ps_global->display_charmap)
 388                   ct = conversion_table(charset, ps_global->display_charmap);
 389
 390                 rv = charset_editorial(charset, msgno, handlesp, flags,
 391                                        ct ? ct->quality : CV_NO_TRANSLATE_POSSIBLE,
 392                                        column, pc);
 393             }
 394             if(!rv)
 395               goto write_error;
 396         }
 397
 398         fs_give((void **) &charset);
 399     }
 400
 401     /* Format editorial comment about unknown encoding */
 402     if(att->body->encoding > ENCQUOTEDPRINTABLE){
 403         char buf[2048];
 404
 405         snprintf(buf, sizeof(buf), ENCODING_DISCLAIMER, body_encodings[att->body->encoding]);
 406
 407         if(!(format_editorial(buf, column, flags, handlesp, pc) == NULL
 408              && gf_puts(NEWLINE, pc) && gf_puts(NEWLINE, pc)))
 409           goto write_error;
 410     }
 411
 412     err = detach(ps_global->mail_stream, msgno, att->number, 0L,
 413                  NULL, pc, filtcnt ? filters : NULL, 0);
 414
 415     delete_unused_handles(handlesp);
 416
 417     if(free_this)
 418       fs_give((void **) &free_this);
 419
 420     if(warn_so)
 421       so_give(&warn_so);
 422
 423     if(err) {
 424         error_found++;
 425         if(style == QStatus) {
 426             q_status_message1(SM_ORDER, 3, 4, "%.200s", err);
 427         } else if(style == InLine) {
 428             char buftmp[MAILTMPLEN];
 429
 430             snprintf(buftmp, sizeof(buftmp), "%s",
 431                     att->body->description ? att->body->description : "");
 432             snprintf(tmp_20k_buf, SIZEOF_20KBUF, "%s   [Error: %s]  %c%s%c%s%s",
 433                     NEWLINE, err,
 434                     att->body->description ? '\"' : ' ',
 435                     att->body->description ? buftmp : "",
 436                     att->body->description ? '\"' : ' ',
 437                     NEWLINE, NEWLINE);
 438             if(!gf_puts(tmp_20k_buf, pc))
 439               goto write_error;
 440         }
 441     }
 442
 443     if(att->body->subtype
 444        && (!strucmp(att->body->subtype, "richtext")
 445            || !strucmp(att->body->subtype, "enriched"))
 446        && !(flags & FM_DISPLAY)){
 447         if(!gf_puts(NEWLINE, pc) || !gf_puts(NEWLINE, pc))
 448           goto write_error;
 449     }
 450
 451     return(error_found);
 452
 453   write_error:
 454     if(style == QStatus)
 455       q_status_message1(SM_ORDER, 3, 4, "Error writing message: %.200s",
 456                         error_description(errno));
 457
 458     return(1);
 459 }
 460
 461
 462
 463 /*
 464  * Format editorial comment about charset mismatch
 465  */
 466 int
 467 charset_editorial(char *charset, long int msgno, HANDLE_S **handlesp, int flags,
 468                   int quality, int width, gf_io_t pc)
 469 {
 470     char     *p, color[64], buf[2048];
 471     int       i, n;
 472     HANDLE_S *h = NULL;
 473
 474     snprintf(buf, sizeof(buf), CHARSET_DISCLAIMER_1, charset ? charset : "US-ASCII");
 475     p = &buf[strlen(buf)];
 476
 477     if(!(ps_global->display_charmap
 478          && strucmp(ps_global->display_charmap, "US-ASCII"))
 479        && handlesp
 480        && (flags & FM_DISPLAY) == FM_DISPLAY){
 481         h                     = new_handle(handlesp);
 482         h->type               = URL;
 483         h->h.url.path         = cpystr("x-alpine-help:h_config_char_set");
 484
 485         /*
 486          * Because this filter comes after the delete_quotes filter, we need
 487          * to tell delete_quotes that this handle is used, so it won't
 488          * delete it. This is a dangerous thing.
 489          */
 490         h->is_used = 1;
 491
 492         if(!(flags & FM_NOCOLOR)
 493            && scroll_handle_start_color(color, sizeof(color), &n)){
 494             for(i = 0; i < n; i++)
 495               if(p-buf < sizeof(buf))
 496                 *p++ = color[i];
 497         }
 498         else if(F_OFF(F_SLCTBL_ITEM_NOBOLD, ps_global)){
 499             if(p-buf+1 < sizeof(buf)){
 500               *p++ = TAG_EMBED;
 501               *p++ = TAG_BOLDON;
 502             }
 503         }
 504
 505         if(p-buf+1 < sizeof(buf)){
 506           *p++ = TAG_EMBED;
 507           *p++ = TAG_HANDLE;
 508         }
 509
 510         snprintf(p + 1, sizeof(buf)-(p+1-buf), "%d", h->key);
 511         if(p-buf < sizeof(buf))
 512           *p  = strlen(p + 1);
 513
 514         p  += (*p + 1);
 515     }
 516
 517     sstrncpy(&p, CHARSET_DISCLAIMER_2, sizeof(buf)-(p-buf));
 518
 519     if(h){
 520         /* in case it was the current selection */
 521         if(p-buf+1 < sizeof(buf)){
 522           *p++ = TAG_EMBED;
 523           *p++ = TAG_INVOFF;
 524         }
 525
 526         if(scroll_handle_end_color(color, sizeof(color), &n, 0)){
 527             for(i = 0; i < n; i++)
 528               if(p-buf < sizeof(buf))
 529                 *p++ = color[i];
 530         }
 531         else{
 532             if(p-buf+1 < sizeof(buf)){
 533               *p++ = TAG_EMBED;
 534               *p++ = TAG_BOLDOFF;
 535             }
 536         }
 537
 538         if(p-buf < sizeof(buf))
 539           *p = '\0';
 540     }
 541
 542     snprintf(p, sizeof(buf)-(p-buf), CHARSET_DISCLAIMER_3,
 543             ps_global->display_charmap ?  ps_global->display_charmap : "US-ASCII",
 544             (quality == CV_LOSES_SPECIAL_CHARS) ? "special " : "");
 545
 546     buf[sizeof(buf)-1] = '\0';
 547
 548     return(format_editorial(buf, width, flags, handlesp, pc) == NULL
 549            && gf_puts(NEWLINE, pc) && gf_puts(NEWLINE, pc));
 550 }
 551
 552
 553
 554
 555 /*
 556  * This one is a little more complicated because it comes late in the
 557  * filtering sequence after colors have been added and url's hilited and
 558  * so on. So if it wants to look at the beginning of the line it has to
 559  * wade through some stuff done by the other filters first. This one is
 560  * skipping the leading indent added by the viewer-margin stuff and leading
 561  * tags.
 562  */
 563 int
 564 delete_quotes(long int linenum, char *line, LT_INS_S **ins, void *local)
 565 {
 566     DELQ_S *dq;
 567     char   *lp;
 568     int     i, lines, not_a_quote = 0;
 569     size_t  len = 0;
 570
 571     dq = (DELQ_S *) local;
 572     if(!dq)
 573       return(0);
 574
 575     if(dq->lines > 0 || dq->lines == Q_DEL_ALL){
 576
 577         lines = (dq->lines == Q_DEL_ALL) ? 0 : dq->lines;
 578
 579         /*
 580          * First determine if this line is part of a quote.
 581          */
 582
 583         /* skip over indent added by viewer-margin-left */
 584         lp = line;
 585         for(i = dq->indent_length; i > 0 && !not_a_quote && *lp; i--)
 586           if(*lp++ != SPACE)
 587             not_a_quote++;
 588
 589         /* skip over leading tags */
 590         while(!not_a_quote
 591               && ((unsigned char) (*lp) == (unsigned char) TAG_EMBED)){
 592             switch(*++lp){
 593               case '\0':
 594                 not_a_quote++;
 595                 break;
 596
 597               default:
 598                 ++lp;
 599                 break;
 600
 601               case TAG_FGCOLOR:
 602               case TAG_BGCOLOR:
 603               case TAG_HANDLE:
 604                 switch(*lp){
 605                   case TAG_FGCOLOR:
 606                   case TAG_BGCOLOR:
 607                     len = RGBLEN + 1;
 608                     break;
 609
 610                   case TAG_HANDLE:
 611                     len = *++lp + 1;
 612                     break;
 613                 }
 614
 615                 if(strlen(lp) < len)
 616                   not_a_quote++;
 617                 else
 618                   lp += len;
 619
 620                 break;
 621
 622               case TAG_EMBED:
 623                 break;
 624             }
 625         }
 626
 627         /* skip over whitespace */
 628         if(!dq->is_flowed)
 629           while(isspace((unsigned char) *lp))
 630             lp++;
 631
 632         /* check first character to see if it is a quote */
 633         if(!not_a_quote && *lp != '>')
 634           not_a_quote++;
 635
 636         if(not_a_quote){
 637             if(dq->in_quote > lines+1 && !dq->delete_all){
 638               char tmp[500];
 639               COLOR_PAIR *col = NULL;
 640               char cestart[2 * RGBLEN + 5];
 641               char ceend[2 * RGBLEN + 5];
 642
 643               /*
 644                * Display how many lines were hidden.
 645                */
 646
 647               cestart[0] = ceend[0] = '\0';
 648               if(dq->do_color
 649                  && ps_global->VAR_METAMSG_FORE_COLOR
 650                  && ps_global->VAR_METAMSG_BACK_COLOR
 651                  && (col = new_color_pair(ps_global->VAR_METAMSG_FORE_COLOR,
 652                                           ps_global->VAR_METAMSG_BACK_COLOR)))
 653                 if(!pico_is_good_colorpair(col))
 654                   free_color_pair(&col);
 655
 656               if(col){
 657                   strncpy(cestart, color_embed(col->fg, col->bg), sizeof(cestart));
 658                   cestart[sizeof(cestart)-1] = '\0';
 659                   strncpy(ceend, color_embed(ps_global->VAR_NORM_FORE_COLOR,
 660                                              ps_global->VAR_NORM_BACK_COLOR), sizeof(ceend));
 661                   ceend[sizeof(ceend)-1] = '\0';
 662                   free_color_pair(&col);
 663               }
 664
 665               snprintf(tmp, sizeof(tmp),
 666                       "%s[ %s%d lines of quoted text hidden from view%s ]\r\n",
 667                       repeat_char(dq->indent_length, SPACE),
 668                       cestart, dq->in_quote - lines, ceend);
 669               if(strlen(tmp)-strlen(cestart)-strlen(ceend)-2 > ps_global->ttyo->screen_cols){
 670
 671                 snprintf(tmp, sizeof(tmp), "%s[ %s%d lines of quoted text hidden%s ]\r\n",
 672                         repeat_char(dq->indent_length, SPACE),
 673                         cestart, dq->in_quote - lines, ceend);
 674
 675                 if(strlen(tmp)-strlen(cestart)-strlen(ceend)-2 > ps_global->ttyo->screen_cols){
 676                   snprintf(tmp, sizeof(tmp), "%s[ %s%d lines hidden%s ]\r\n",
 677                           repeat_char(dq->indent_length, SPACE),
 678                           cestart, dq->in_quote - lines, ceend);
 679
 680                   if(strlen(tmp)-strlen(cestart)-strlen(ceend)-2 > ps_global->ttyo->screen_cols){
 681                     snprintf(tmp, sizeof(tmp), "%s[ %s%d hidden%s ]\r\n",
 682                             repeat_char(dq->indent_length, SPACE),
 683                             cestart, dq->in_quote - lines, ceend);
 684
 685                     if(strlen(tmp)-strlen(cestart)-strlen(ceend)-2 > ps_global->ttyo->screen_cols){
 686                       snprintf(tmp, sizeof(tmp), "%s[...]\r\n",
 687                               repeat_char(dq->indent_length, SPACE));
 688
 689                       if(strlen(tmp)-2 > ps_global->ttyo->screen_cols){
 690                         snprintf(tmp, sizeof(tmp), "%s...\r\n",
 691                                 repeat_char(dq->indent_length, SPACE));
 692
 693                         if(strlen(tmp)-2 > ps_global->ttyo->screen_cols){
 694                           snprintf(tmp, sizeof(tmp), "%s\r\n",
 695                               repeat_char(MIN(ps_global->ttyo->screen_cols,3),
 696                                           '.'));
 697                         }
 698                       }
 699                     }
 700                   }
 701                 }
 702               }
 703
 704               ins = gf_line_test_new_ins(ins, line, tmp, strlen(tmp));
 705               mark_handles_in_line(line, dq->handlesp, 1);
 706               ps_global->some_quoting_was_suppressed = 1;
 707             }
 708             else if(dq->in_quote == lines+1
 709                     && dq->saved_line && *dq->saved_line){
 710                 /*
 711                  * We would have only had to delete a single line. Instead
 712                  * of deleting it, just show it.
 713                  */
 714                 ins = gf_line_test_new_ins(ins, line,
 715                                            *dq->saved_line,
 716                                            strlen(*dq->saved_line));
 717                 mark_handles_in_line(*dq->saved_line, dq->handlesp, 1);
 718             }
 719             else
 720               mark_handles_in_line(line, dq->handlesp, 1);
 721
 722             if(dq->saved_line && *dq->saved_line)
 723               fs_give((void **) dq->saved_line);
 724
 725             dq->in_quote = 0;
 726         }
 727         else{
 728             dq->in_quote++;                             /* count it */
 729             if(dq->in_quote > lines){
 730                 /*
 731                  * If the hidden part is a single line we'll show it instead.
 732                  */
 733                 if(dq->in_quote == lines+1 && !dq->delete_all){
 734                     if(dq->saved_line && *dq->saved_line)
 735                       fs_give((void **) dq->saved_line);
 736
 737                     *dq->saved_line = fs_get(strlen(line) + 3);
 738                     snprintf(*dq->saved_line, strlen(line)+3, "%s\r\n", line);
 739                 }
 740
 741                 mark_handles_in_line(line, dq->handlesp, 0);
 742                 /* skip it, at least for now */
 743                 return(2);
 744             }
 745
 746             mark_handles_in_line(line, dq->handlesp, 1);
 747         }
 748     }
 749
 750     return(0);
 751 }
 752
 753
 754 /*
 755  * This is a line-at-a-time filter that replaces incoming UTF-8 text with
 756  * ISO-2022-JP text.
 757  */
 758 int
 759 translate_utf8_to_2022_jp(long int linenum, char *line, LT_INS_S **ins, void *local)
 760 {
 761     LOC_2022_JP *lpj;
 762     int ret = 0;
 763
 764     lpj = (LOC_2022_JP *) local;
 765
 766     if(line && line[0]){
 767         char *converted;
 768         size_t len;
 769
 770         converted = utf8_to_charset(line, "ISO-2022-JP", lpj ? lpj->report_err : 0);
 771
 772         if(!converted)
 773           ret = -1;
 774
 775         if(converted && converted != line){
 776             /* delete the old line and replace it with the translation */
 777             len = strlen(line);
 778             ins = gf_line_test_new_ins(ins, line, "", -((int) len));
 779             ins = gf_line_test_new_ins(ins, line+len, converted, strlen(converted));
 780             fs_give((void **) &converted);
 781         }
 782     }
 783
 784     return ret;
 785 }
 786
 787
 788 /*
 789  * Replace quotes of nonflowed messages.  This needs to happen
 790  * towards the end of filtering because a lot of prior filters
 791  * depend on "> ", such as quote coloring and suppression.
 792  * Quotes are already colored here, so we have to account for
 793  * that formatting.
 794  */
 795 int
 796 replace_quotes(long int linenum, char *line, LT_INS_S **ins, void *local)
 797 {
 798     char *lp = line, *prefix = NULL, *last_prefix = NULL;
 799     int no_more_quotes = 0, len = 0, saw_quote = 0;
 800
 801     if(ps_global->VAR_QUOTE_REPLACE_STRING){
 802         get_pair(ps_global->VAR_QUOTE_REPLACE_STRING, &prefix, &last_prefix, 0, 0);
 803         if(!prefix && last_prefix){
 804             prefix = last_prefix;
 805             last_prefix = NULL;
 806         }
 807     }
 808     else
 809       return(0);
 810
 811     while(isspace((unsigned char)(*lp)))
 812       lp++;
 813     while(*lp && !no_more_quotes){
 814         switch(*lp++){
 815           case '>':
 816             if(*lp == ' '){
 817                 ins = gf_line_test_new_ins(ins, lp - 1, "", -2);
 818                 lp++;
 819             }
 820             else
 821               ins = gf_line_test_new_ins(ins, lp - 1, "", -1);
 822             if(strlen(prefix))
 823               ins = gf_line_test_new_ins(ins, lp, prefix, strlen(prefix));
 824             saw_quote = 1;
 825             break;
 826           case TAG_EMBED:
 827             switch(*lp++){
 828               case TAG_FGCOLOR:
 829               case TAG_BGCOLOR:
 830                 len = RGBLEN;
 831                 break;
 832               case TAG_HANDLE:
 833                 len = *lp++ + 1;
 834                 break;
 835             }
 836             if(strlen(lp) < len)
 837               no_more_quotes = 1;
 838             else
 839               lp += len;
 840             break;
 841           case ' ':
 842           case '\t':
 843             break;
 844           default:
 845             if(saw_quote && last_prefix)
 846               ins = gf_line_test_new_ins(ins, lp - 1, last_prefix, strlen(last_prefix));
 847             no_more_quotes = 1;
 848             break;
 849         }
 850     }
 851     if(prefix)
 852       fs_give((void **)&prefix);
 853     if(last_prefix)
 854       fs_give((void **)&last_prefix);
 855     return(0);
 856 }
 857
 858
 859 /*
 860  * We need to delete handles that we are removing from the displayed
 861  * text. But there is a complication. Some handles appear at more than
 862  * one location in the text. So we keep track of which handles we're
 863  * actually using as we go, then at the end we delete the ones we
 864  * didn't use.
 865  *
 866  * Args    line    -- the text line, which includes tags
 867  *       handlesp  -- handles pointer
 868  *         used    -- If 1, set handles in this line to used
 869  *                    if 0, leave handles in this line set as they already are
 870  */
 871 void
 872 mark_handles_in_line(char *line, HANDLE_S **handlesp, int used)
 873 {
 874     unsigned char c;
 875     size_t        length;
 876     int           key, n;
 877     HANDLE_S     *h;
 878
 879     if(!(line && handlesp && *handlesp))
 880       return;
 881
 882     length = strlen(line);
 883
 884     /* mimic code in PutLine0n8b */
 885     while(length--){
 886
 887         c = (unsigned char) *line++;
 888
 889         if(c == (unsigned char) TAG_EMBED && length){
 890
 891             length--;
 892
 893             switch(*line++){
 894               case TAG_HANDLE :
 895                 length -= *line + 1;    /* key length plus length tag */
 896
 897                 for(key = 0, n = *line++; n; n--)
 898                   key = (key * 10) + (*line++ - '0');
 899
 900                 h = get_handle(*handlesp, key);
 901                 if(h){
 902                     h->using_is_used = 1;
 903                     /*
 904                      * If it's already marked is_used, leave it marked
 905                      * that way. We only call this function with used = 0
 906                      * in order to set using_is_used.
 907                      */
 908                     h->is_used = (h->is_used || used) ? 1 : 0;
 909                 }
 910
 911                 break;
 912
 913               case TAG_FGCOLOR :
 914                 if(length < RGBLEN){
 915                     length = 0;
 916                     break;
 917                 }
 918
 919                 length -= RGBLEN;
 920                 line += RGBLEN;
 921                 break;
 922
 923               case TAG_BGCOLOR :
 924                 if(length < RGBLEN){
 925                     length = 0;
 926                     break;
 927                 }
 928
 929                 length -= RGBLEN;
 930                 line += RGBLEN;
 931                 break;
 932
 933               default:
 934                 break;
 935             }
 936         }
 937     }
 938 }
 939
 940
 941 /*
 942  * parsed html risk state functions
 943  */
 944 void
 945 clear_html_risk(void)
 946 {
 947     _html_risk = 0;
 948 }
 949
 950 void
 951 set_html_risk(void)
 952 {
 953     _html_risk = 1;
 954 }
 955
 956 int
 957 get_html_risk(void)
 958 {
 959     return(_html_risk);
 960 }