src/viewer/dump/dump.c

   1 /* Support for dumping to the file on startup (w/o bfu) */
   2
   3 #ifdef HAVE_CONFIG_H
   4 #include "config.h"
   5 #endif
   6
   7 #include <errno.h>
   8 #include <stdio.h>
   9 #include <string.h>
  10 #include <sys/types.h> /* NetBSD flavour */
  11 #ifdef HAVE_SYS_SIGNAL_H
  12 #include <sys/signal.h>
  13 #endif
  14 #ifdef HAVE_FCNTL_H
  15 #include <fcntl.h> /* OS/2 needs this after sys/types.h */
  16 #endif
  17 #ifdef HAVE_UNISTD_H
  18 #include <unistd.h>
  19 #endif
  20
  21 #include "elinks.h"
  22
  23 #include "cache/cache.h"
  24 #include "config/options.h"
  25 #include "document/document.h"
  26 #include "document/options.h"
  27 #include "document/renderer.h"
  28 #include "document/view.h"
  29 #include "intl/charsets.h"
  30 #include "intl/gettext/libintl.h"
  31 #include "main/select.h"
  32 #include "main/main.h"
  33 #include "network/connection.h"
  34 #include "network/state.h"
  35 #include "osdep/ascii.h"
  36 #include "osdep/osdep.h"
  37 #include "protocol/protocol.h"
  38 #include "protocol/uri.h"
  39 #include "session/download.h"
  40 #include "terminal/color.h"
  41 #include "terminal/hardio.h"
  42 #include "terminal/terminal.h"
  43 #include "util/memory.h"
  44 #include "util/string.h"
  45 #include "viewer/dump/dump.h"
  46 #include "viewer/text/view.h"
  47 #include "viewer/text/vs.h"
  48
  49
  50 static int dump_pos;
  51 static struct download dump_download;
  52 static int dump_redir_count = 0;
  53
  54 #define D_BUF   65536
  55
  56 /** A place where dumping functions write their output.  The data
  57  * first goes to the buffer in this structure.  When the buffer is
  58  * full enough, it is flushed to a file descriptor or to a string.  */
  59 struct dump_output {
  60         /** How many bytes are in #buf already.  */
  61         size_t bufpos;
  62
  63         /** A string to which the buffer should eventually be flushed,
  64          * or NULL.  */
  65         struct string *string;
  66
  67         /** A file descriptor to which the buffer should eventually be
  68          * flushed, or -1.  */
  69         int fd;
  70
  71         /** Bytes waiting to be flushed.  */
  72         unsigned char buf[D_BUF];
  73 };
  74
  75 /** Allocate and initialize a struct dump_output.
  76  * The caller should eventually free the structure with mem_free().
  77  *
  78  * @param fd
  79  *   The file descriptor to which the output will be written.
  80  *   Use -1 if the output should go to a string instead.
  81  *
  82  * @param string
  83  *   The string to which the output will be appended.
  84  *   Use NULL if the output should go to a file descriptor instead.
  85  *
  86  * @return The new structure, or NULL on error.
  87  *
  88  * @relates dump_output */
  89 static struct dump_output *
  90 dump_output_alloc(int fd, struct string *string)
  91 {
  92         struct dump_output *out;
  93
  94         assert((fd == -1) ^ (string == NULL));
  95         if_assert_failed return NULL;
  96
  97         out = mem_alloc(sizeof(*out));
  98         if (out) {
  99                 out->fd = fd;
 100                 out->string = string;
 101                 out->bufpos = 0;
 102         }
 103         return out;
 104 }
 105
 106 /** Flush buffered output to the file or string.
 107  *
 108  * @return 0 on success, or -1 on error.
 109  *
 110  * @post If this succeeds, then out->bufpos == 0, so that the buffer
 111  * has room for more data.
 112  *
 113  * @relates dump_output */
 114 static int
 115 dump_output_flush(struct dump_output *out)
 116 {
 117         if (out->string) {
 118                 if (!add_bytes_to_string(out->string, out->buf, out->bufpos))
 119                         return -1;
 120         }
 121         else {
 122                 if (hard_write(out->fd, out->buf, out->bufpos) != out->bufpos)
 123                         return -1;
 124         }
 125
 126         out->bufpos = 0;
 127         return 0;
 128 }
 129
 130 static int
 131 write_char(unsigned char c, struct dump_output *out)
 132 {
 133         if (out->bufpos >= D_BUF) {
 134                 if (dump_output_flush(out))
 135                         return -1;
 136         }
 137
 138         out->buf[out->bufpos++] = c;
 139         return 0;
 140 }
 141
 142 static int
 143 write_color_16(unsigned char color, struct dump_output *out)
 144 {
 145         unsigned char bufor[] = "\033[0;30;40m";
 146         unsigned char *data = bufor;
 147         int background = (color >> 4) & 7;
 148         int foreground = color & 7;
 149
 150         bufor[5] += foreground;
 151         if (background) bufor[8] += background;
 152         else {
 153                 bufor[6] = 'm';
 154                 bufor[7] = '\0';
 155         }
 156         while(*data) {
 157                 if (write_char(*data++, out)) return -1;
 158         }
 159         return 0;
 160 }
 161
 162 #define DUMP_COLOR_MODE_16
 163 #define DUMP_FUNCTION_COLOR   dump_16color
 164 #define DUMP_FUNCTION_UTF8    dump_16color_utf8
 165 #define DUMP_FUNCTION_UNIBYTE dump_16color_unibyte
 166 #include "dump-color-mode.h"
 167 #undef DUMP_COLOR_MODE_16
 168 #undef DUMP_FUNCTION_COLOR
 169 #undef DUMP_FUNCTION_UTF8
 170 #undef DUMP_FUNCTION_UNIBYTE
 171
 172 /* configure --enable-debug uses gcc -Wall -Werror, and -Wall includes
 173  * -Wunused-function, so declaring or defining any unused function
 174  * would break the build. */
 175 #if defined(CONFIG_88_COLORS) || defined(CONFIG_256_COLORS)
 176
 177 static int
 178 write_color_256(const unsigned char *str, unsigned char color,
 179                 struct dump_output *out)
 180 {
 181         unsigned char bufor[16];
 182         unsigned char *data = bufor;
 183
 184         snprintf(bufor, 16, "\033[%s;5;%dm", str, color);
 185         while(*data) {
 186                 if (write_char(*data++, out)) return -1;
 187         }
 188         return 0;
 189 }
 190
 191 #define DUMP_COLOR_MODE_256
 192 #define DUMP_FUNCTION_COLOR   dump_256color
 193 #define DUMP_FUNCTION_UTF8    dump_256color_utf8
 194 #define DUMP_FUNCTION_UNIBYTE dump_256color_unibyte
 195 #include "dump-color-mode.h"
 196 #undef DUMP_COLOR_MODE_256
 197 #undef DUMP_FUNCTION_COLOR
 198 #undef DUMP_FUNCTION_UTF8
 199 #undef DUMP_FUNCTION_UNIBYTE
 200
 201 #endif /* defined(CONFIG_88_COLORS) || defined(CONFIG_256_COLORS) */
 202
 203 #ifdef CONFIG_TRUE_COLOR
 204
 205 static int
 206 write_true_color(const unsigned char *str, const unsigned char *color,
 207                  struct dump_output *out)
 208 {
 209         unsigned char bufor[24];
 210         unsigned char *data = bufor;
 211
 212         snprintf(bufor, 24, "\033[%s;2;%d;%d;%dm", str, color[0], color[1], color[2]);
 213         while(*data) {
 214                 if (write_char(*data++, out)) return -1;
 215         }
 216         return 0;
 217 }
 218
 219 #define DUMP_COLOR_MODE_TRUE
 220 #define DUMP_FUNCTION_COLOR   dump_truecolor
 221 #define DUMP_FUNCTION_UTF8    dump_truecolor_utf8
 222 #define DUMP_FUNCTION_UNIBYTE dump_truecolor_unibyte
 223 #include "dump-color-mode.h"
 224 #undef DUMP_COLOR_MODE_TRUE
 225 #undef DUMP_FUNCTION_COLOR
 226 #undef DUMP_FUNCTION_UTF8
 227 #undef DUMP_FUNCTION_UNIBYTE
 228
 229 #endif /* CONFIG_TRUE_COLOR */
 230
 231 #define DUMP_FUNCTION_COLOR   dump_nocolor
 232 #define DUMP_FUNCTION_UTF8    dump_nocolor_utf8
 233 #define DUMP_FUNCTION_UNIBYTE dump_nocolor_unibyte
 234 #include "dump-color-mode.h"
 235 #undef DUMP_FUNCTION_COLOR
 236 #undef DUMP_FUNCTION_UTF8
 237 #undef DUMP_FUNCTION_UNIBYTE
 238
 239 /*! @return 0 on success, -1 on error */
 240 static int
 241 dump_references(struct document *document, int fd, unsigned char buf[D_BUF])
 242 {
 243         if (document->nlinks && get_opt_bool("document.dump.references")) {
 244                 int x;
 245                 unsigned char *header = "\nReferences\n\n   Visible links\n";
 246                 int headlen = strlen(header);
 247
 248                 if (hard_write(fd, header, headlen) != headlen)
 249                         return -1;
 250
 251                 for (x = 0; x < document->nlinks; x++) {
 252                         struct link *link = &document->links[x];
 253                         unsigned char *where = link->where;
 254                         size_t reflen;
 255
 256                         if (!where) continue;
 257
 258                         if (document->options.links_numbering) {
 259                                 if (link->title && *link->title)
 260                                         snprintf(buf, D_BUF, "%4d. %s\n\t%s\n",
 261                                                  x + 1, link->title, where);
 262                                 else
 263                                         snprintf(buf, D_BUF, "%4d. %s\n",
 264                                                  x + 1, where);
 265                         } else {
 266                                 if (link->title && *link->title)
 267                                         snprintf(buf, D_BUF, "   . %s\n\t%s\n",
 268                                                  link->title, where);
 269                                 else
 270                                         snprintf(buf, D_BUF, "   . %s\n", where);
 271                         }
 272
 273                         reflen = strlen(buf);
 274                         if (hard_write(fd, buf, reflen) != reflen)
 275                                 return -1;
 276                 }
 277         }
 278
 279         return 0;
 280 }
 281
 282 int
 283 dump_to_file(struct document *document, int fd)
 284 {
 285         struct dump_output *out = dump_output_alloc(fd, NULL);
 286         int error;
 287
 288         if (!out) return -1;
 289
 290         error = dump_nocolor(document, out);
 291         if (!error)
 292                 error = dump_references(document, fd, out->buf);
 293
 294         mem_free(out);
 295         return error;
 296 }
 297
 298 /* This dumps the given @cached's formatted output onto @fd. */
 299 static void
 300 dump_formatted(int fd, struct download *download, struct cache_entry *cached)
 301 {
 302         struct document_options o;
 303         struct document_view formatted;
 304         struct view_state vs;
 305         int width;
 306         struct dump_output *out;
 307
 308         if (!cached) return;
 309
 310         memset(&formatted, 0, sizeof(formatted));
 311
 312         init_document_options(&o);
 313         width = get_opt_int("document.dump.width");
 314         set_box(&o.box, 0, 1, width, DEFAULT_TERMINAL_HEIGHT);
 315
 316         o.cp = get_opt_codepage("document.dump.codepage");
 317         o.color_mode = get_opt_int("document.dump.color_mode");
 318         o.plain = 0;
 319         o.frames = 0;
 320         o.links_numbering = get_opt_bool("document.dump.numbering");
 321
 322         init_vs(&vs, cached->uri, -1);
 323
 324         render_document(&vs, &formatted, &o);
 325
 326         out = dump_output_alloc(fd, NULL);
 327         if (out) {
 328                 int error;
 329
 330                 switch (o.color_mode) {
 331                 case COLOR_MODE_DUMP:
 332                 case COLOR_MODE_MONO: /* FIXME: inversion */
 333                         error = dump_nocolor(formatted.document, out);
 334                         break;
 335
 336                 default:
 337                         /* If the desired color mode was not compiled in,
 338                          * use 16 colors.  */
 339                 case COLOR_MODE_16:
 340                         error = dump_16color(formatted.document, out);
 341                         break;
 342
 343 #ifdef CONFIG_88_COLORS
 344                 case COLOR_MODE_88:
 345                         error = dump_256color(formatted.document, out);
 346                         break;
 347 #endif
 348
 349 #ifdef CONFIG_256_COLORS
 350                 case COLOR_MODE_256:
 351                         error = dump_256color(formatted.document, out);
 352                         break;
 353 #endif
 354
 355 #ifdef CONFIG_TRUE_COLOR
 356                 case COLOR_MODE_TRUE_COLOR:
 357                         error = dump_truecolor(formatted.document, out);
 358                         break;
 359 #endif
 360                 }
 361
 362                 if (!error)
 363                         dump_references(formatted.document, fd, out->buf);
 364
 365                 mem_free(out);
 366         } /* if out */
 367
 368         detach_formatted(&formatted);
 369         destroy_vs(&vs, 1);
 370 }
 371
 372 #undef D_BUF
 373
 374 /* This dumps the given @cached's source onto @fd nothing more. It returns 0 if it
 375  * all went fine and 1 if something isn't quite right and we should terminate
 376  * ourselves ASAP. */
 377 static int
 378 dump_source(int fd, struct download *download, struct cache_entry *cached)
 379 {
 380         struct fragment *frag;
 381
 382         if (!cached) return 0;
 383
 384 nextfrag:
 385         foreach (frag, cached->frag) {
 386                 int d = dump_pos - frag->offset;
 387                 int l, w;
 388
 389                 if (d < 0 || frag->length <= d)
 390                         continue;
 391
 392                 l = frag->length - d;
 393                 w = hard_write(fd, frag->data + d, l);
 394
 395                 if (w != l) {
 396                         detach_connection(download, dump_pos);
 397
 398                         if (w < 0)
 399                                 ERROR(gettext("Can't write to stdout: %s"),
 400                                       (unsigned char *) strerror(errno));
 401                         else
 402                                 ERROR(gettext("Can't write to stdout."));
 403
 404                         program.retval = RET_ERROR;
 405                         return 1;
 406                 }
 407
 408                 dump_pos += w;
 409                 detach_connection(download, dump_pos);
 410                 goto nextfrag;
 411         }
 412
 413         return 0;
 414 }
 415
 416 static unsigned char *
 417 subst_url(unsigned char *str, struct string *url)
 418 {
 419         struct string string;
 420
 421         if (!init_string(&string)) return NULL;
 422
 423         while (*str) {
 424                 int p;
 425
 426                 for (p = 0; str[p] && str[p] != '%' && str[p] != '\\'; p++);
 427
 428                 add_bytes_to_string(&string, str, p);
 429                 str += p;
 430
 431                 if (*str == '\\') {
 432                         unsigned char ch;
 433
 434                         str++;
 435                         switch (*str) {
 436                                 case 'f':
 437                                         ch = '\f';
 438                                         break;
 439                                 case 'n':
 440                                         ch = '\n';
 441                                         break;
 442                                 case 't':
 443                                         ch = '\t';
 444                                         break;
 445                                 default:
 446                                         ch = *str;
 447                         }
 448                         if (*str) {
 449                                 add_char_to_string(&string, ch);
 450                                 str++;
 451                         }
 452                         continue;
 453
 454                 } else if (*str != '%') {
 455                         break;
 456                 }
 457
 458                 str++;
 459                 switch (*str) {
 460                         case 'u':
 461                                 if (url) add_string_to_string(&string, url);
 462                                 break;
 463                 }
 464
 465                 if (*str) str++;
 466         }
 467
 468         return string.source;
 469 }
 470
 471 static void
 472 dump_print(unsigned char *option, struct string *url)
 473 {
 474         unsigned char *str = get_opt_str(option);
 475
 476         if (str) {
 477                 unsigned char *realstr = subst_url(str, url);
 478
 479                 if (realstr) {
 480                         printf("%s", realstr);
 481                         fflush(stdout);
 482                         mem_free(realstr);
 483                 }
 484         }
 485 }
 486
 487 static void
 488 dump_loading_callback(struct download *download, void *p)
 489 {
 490         struct cache_entry *cached = download->cached;
 491         int fd = get_output_handle();
 492
 493         if (fd == -1) return;
 494         if (cached && cached->redirect && dump_redir_count++ < MAX_REDIRECTS) {
 495                 struct uri *uri = cached->redirect;
 496
 497                 cancel_download(download, 0);
 498
 499                 load_uri(uri, cached->uri, download, PRI_MAIN, 0, -1);
 500                 return;
 501         }
 502
 503         if (is_in_queued_state(download->state)) return;
 504
 505         if (get_cmd_opt_bool("dump")) {
 506                 if (is_in_transfering_state(download->state))
 507                         return;
 508
 509                 dump_formatted(fd, download, cached);
 510
 511         } else {
 512                 if (dump_source(fd, download, cached) > 0)
 513                         goto terminate;
 514
 515                 if (is_in_progress_state(download->state))
 516                         return;
 517
 518         }
 519
 520         if (!is_in_state(download->state, S_OK)) {
 521                 usrerror(get_state_message(download->state, NULL));
 522                 program.retval = RET_ERROR;
 523                 goto terminate;
 524         }
 525
 526 terminate:
 527         program.terminate = 1;
 528         dump_next(NULL);
 529 }
 530
 531 static void
 532 dump_start(unsigned char *url)
 533 {
 534         unsigned char *wd = get_cwd();
 535         struct uri *uri = get_translated_uri(url, wd);
 536
 537         mem_free_if(wd);
 538
 539         if (!uri || get_protocol_external_handler(NULL, uri)) {
 540                 usrerror(gettext("URL protocol not supported (%s)."), url);
 541                 goto terminate;
 542         }
 543
 544         dump_download.callback = (download_callback_T *) dump_loading_callback;
 545         dump_pos = 0;
 546
 547         if (load_uri(uri, NULL, &dump_download, PRI_MAIN, 0, -1)) {
 548 terminate:
 549                 dump_next(NULL);
 550                 program.terminate = 1;
 551                 program.retval = RET_SYNTAX;
 552         }
 553
 554         if (uri) done_uri(uri);
 555 }
 556
 557 void
 558 dump_next(LIST_OF(struct string_list_item) *url_list)
 559 {
 560         static INIT_LIST_OF(struct string_list_item, todo_list);
 561         static INIT_LIST_OF(struct string_list_item, done_list);
 562         struct string_list_item *item;
 563
 564         if (url_list) {
 565                 /* Steal all them nice list items but keep the same order */
 566                 while (!list_empty(*url_list)) {
 567                         item = url_list->next;
 568                         del_from_list(item);
 569                         add_to_list_end(todo_list, item);
 570                 }
 571         }
 572
 573         /* Dump each url list item one at a time */
 574         if (!list_empty(todo_list)) {
 575                 static int first = 1;
 576
 577                 program.terminate = 0;
 578
 579                 item = todo_list.next;
 580                 del_from_list(item);
 581                 add_to_list(done_list, item);
 582
 583                 if (!first) {
 584                         dump_print("document.dump.separator", NULL);
 585                 } else {
 586                         first = 0;
 587                 }
 588
 589                 dump_print("document.dump.header", &item->string);
 590                 dump_start(item->string.source);
 591                 /* XXX: I think it ought to print footer at the end of
 592                  * the whole dump (not only this file). Testing required.
 593                  * --pasky */
 594                 dump_print("document.dump.footer", &item->string);
 595
 596         } else {
 597                 free_string_list(&done_list);
 598                 program.terminate = 1;
 599         }
 600 }
 601
 602 struct string *
 603 add_document_to_string(struct string *string, struct document *document)
 604 {
 605         struct dump_output *out;
 606         int error;
 607
 608         assert(string && document);
 609         if_assert_failed return NULL;
 610
 611         out = dump_output_alloc(-1, string);
 612         if (!out) return NULL;
 613
 614         error = dump_nocolor(document, out);
 615
 616         mem_free(out);
 617         return error ? NULL : string;
 618 }