release/src/router/busybox/networking/wget.c

   1 /* vi: set sw=4 ts=4: */
   2 /*
   3  * wget - retrieve a file using HTTP or FTP
   4  *
   5  * Chip Rosenthal Covad Communications <chip@laserlink.net>
   6  *
   7  * Licensed under GPLv2, see file LICENSE in this tarball for details.
   8  */
   9
  10 #include "libbb.h"
  11
  12 struct host_info {
  13         // May be used if we ever will want to free() all xstrdup()s...
  14         /* char *allocated; */
  15         const char *path;
  16         const char *user;
  17         char       *host;
  18         int         port;
  19         smallint    is_ftp;
  20 };
  21
  22
  23 /* Globals (can be accessed from signal handlers) */
  24 struct globals {
  25         off_t content_len;        /* Content-length of the file */
  26         off_t beg_range;          /* Range at which continue begins */
  27 #if ENABLE_FEATURE_WGET_STATUSBAR
  28         off_t lastsize;
  29         off_t totalsize;
  30         off_t transferred;        /* Number of bytes transferred so far */
  31         const char *curfile;      /* Name of current file being transferred */
  32         unsigned lastupdate_sec;
  33         unsigned start_sec;
  34 #endif
  35         smallint chunked;             /* chunked transfer encoding */
  36 };
  37 #define G (*(struct globals*)&bb_common_bufsiz1)
  38 struct BUG_G_too_big {
  39         char BUG_G_too_big[sizeof(G) <= COMMON_BUFSIZE ? 1 : -1];
  40 };
  41 #define content_len     (G.content_len    )
  42 #define beg_range       (G.beg_range      )
  43 #define lastsize        (G.lastsize       )
  44 #define totalsize       (G.totalsize      )
  45 #define transferred     (G.transferred    )
  46 #define curfile         (G.curfile        )
  47 #define lastupdate_sec  (G.lastupdate_sec )
  48 #define start_sec       (G.start_sec      )
  49 #define chunked         (G.chunked        )
  50 #define INIT_G() do { } while (0)
  51
  52
  53 #if ENABLE_FEATURE_WGET_STATUSBAR
  54 enum {
  55         STALLTIME = 5                   /* Seconds when xfer considered "stalled" */
  56 };
  57
  58 static unsigned int get_tty2_width(void)
  59 {
  60         unsigned width;
  61         get_terminal_width_height(2, &width, NULL);
  62         return width;
  63 }
  64
  65 static void progress_meter(int flag)
  66 {
  67         /* We can be called from signal handler */
  68         int save_errno = errno;
  69         off_t abbrevsize;
  70         unsigned since_last_update, elapsed;
  71         unsigned ratio;
  72         int barlength, i;
  73
  74         if (flag == -1) { /* first call to progress_meter */
  75                 start_sec = monotonic_sec();
  76                 lastupdate_sec = start_sec;
  77                 lastsize = 0;
  78                 totalsize = content_len + beg_range; /* as content_len changes.. */
  79         }
  80
  81         ratio = 100;
  82         if (totalsize != 0 && !chunked) {
  83                 /* long long helps to have it working even if !LFS */
  84                 ratio = (unsigned) (100ULL * (transferred+beg_range) / totalsize);
  85                 if (ratio > 100) ratio = 100;
  86         }
  87
  88         fprintf(stderr, "\r%-20.20s%4d%% ", curfile, ratio);
  89
  90         barlength = get_tty2_width() - 49;
  91         if (barlength > 0) {
  92                 /* god bless gcc for variable arrays :) */
  93                 i = barlength * ratio / 100;
  94                 {
  95                         char buf[i+1];
  96                         memset(buf, '*', i);
  97                         buf[i] = '\0';
  98                         fprintf(stderr, "|%s%*s|", buf, barlength - i, "");
  99                 }
 100         }
 101         i = 0;
 102         abbrevsize = transferred + beg_range;
 103         while (abbrevsize >= 100000) {
 104                 i++;
 105                 abbrevsize >>= 10;
 106         }
 107         /* see http://en.wikipedia.org/wiki/Tera */
 108         fprintf(stderr, "%6d%c ", (int)abbrevsize, " kMGTPEZY"[i]);
 109
 110 // Nuts! Ain't it easier to update progress meter ONLY when we transferred++?
 111
 112         elapsed = monotonic_sec();
 113         since_last_update = elapsed - lastupdate_sec;
 114         if (transferred > lastsize) {
 115                 lastupdate_sec = elapsed;
 116                 lastsize = transferred;
 117                 if (since_last_update >= STALLTIME) {
 118                         /* We "cut off" these seconds from elapsed time
 119                          * by adjusting start time */
 120                         start_sec += since_last_update;
 121                 }
 122                 since_last_update = 0; /* we are un-stalled now */
 123         }
 124         elapsed -= start_sec; /* now it's "elapsed since start" */
 125
 126         if (since_last_update >= STALLTIME) {
 127                 fprintf(stderr, " - stalled -");
 128         } else {
 129                 off_t to_download = totalsize - beg_range;
 130                 if (transferred <= 0 || (int)elapsed <= 0 || transferred > to_download || chunked) {
 131                         fprintf(stderr, "--:--:-- ETA");
 132                 } else {
 133                         /* to_download / (transferred/elapsed) - elapsed: */
 134                         int eta = (int) ((unsigned long long)to_download*elapsed/transferred - elapsed);
 135                         /* (long long helps to have working ETA even if !LFS) */
 136                         i = eta % 3600;
 137                         fprintf(stderr, "%02d:%02d:%02d ETA", eta / 3600, i / 60, i % 60);
 138                 }
 139         }
 140
 141         if (flag == 0) {
 142                 /* last call to progress_meter */
 143                 alarm(0);
 144                 transferred = 0;
 145                 fputc('\n', stderr);
 146         } else {
 147                 if (flag == -1) { /* first call to progress_meter */
 148                         signal_SA_RESTART_empty_mask(SIGALRM, progress_meter);
 149                 }
 150                 alarm(1);
 151         }
 152
 153         errno = save_errno;
 154 }
 155 /* Original copyright notice which applies to the CONFIG_FEATURE_WGET_STATUSBAR stuff,
 156  * much of which was blatantly stolen from openssh.  */
 157 /*-
 158  * Copyright (c) 1992, 1993
 159  *      The Regents of the University of California.  All rights reserved.
 160  *
 161  * Redistribution and use in source and binary forms, with or without
 162  * modification, are permitted provided that the following conditions
 163  * are met:
 164  * 1. Redistributions of source code must retain the above copyright
 165  *    notice, this list of conditions and the following disclaimer.
 166  * 2. Redistributions in binary form must reproduce the above copyright
 167  *    notice, this list of conditions and the following disclaimer in the
 168  *    documentation and/or other materials provided with the distribution.
 169  *
 170  * 3. <BSD Advertising Clause omitted per the July 22, 1999 licensing change
 171  *              ftp://ftp.cs.berkeley.edu/pub/4bsd/README.Impt.License.Change>
 172  *
 173  * 4. Neither the name of the University nor the names of its contributors
 174  *    may be used to endorse or promote products derived from this software
 175  *    without specific prior written permission.
 176  *
 177  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 178  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 179  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 180  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 181  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 182  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 183  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 184  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 185  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 186  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 187  * SUCH DAMAGE.
 188  *
 189  */
 190 #else /* FEATURE_WGET_STATUSBAR */
 191
 192 static ALWAYS_INLINE void progress_meter(int flag UNUSED_PARAM) { }
 193
 194 #endif
 195
 196
 197 /* Read NMEMB bytes into PTR from STREAM.  Returns the number of bytes read,
 198  * and a short count if an eof or non-interrupt error is encountered.  */
 199 static size_t safe_fread(void *ptr, size_t nmemb, FILE *stream)
 200 {
 201         size_t ret;
 202         char *p = (char*)ptr;
 203
 204         do {
 205                 clearerr(stream);
 206                 errno = 0;
 207                 ret = fread(p, 1, nmemb, stream);
 208                 p += ret;
 209                 nmemb -= ret;
 210         } while (nmemb && ferror(stream) && errno == EINTR);
 211
 212         return p - (char*)ptr;
 213 }
 214
 215 /* Read a line or SIZE-1 bytes into S, whichever is less, from STREAM.
 216  * Returns S, or NULL if an eof or non-interrupt error is encountered.  */
 217 static char *safe_fgets(char *s, int size, FILE *stream)
 218 {
 219         char *ret;
 220
 221         do {
 222                 clearerr(stream);
 223                 errno = 0;
 224                 ret = fgets(s, size, stream);
 225         } while (ret == NULL && ferror(stream) && errno == EINTR);
 226
 227         return ret;
 228 }
 229
 230 #if ENABLE_FEATURE_WGET_AUTHENTICATION
 231 /* Base64-encode character string. buf is assumed to be char buf[512]. */
 232 static char *base64enc_512(char buf[512], const char *str)
 233 {
 234         unsigned len = strlen(str);
 235         if (len > 512/4*3 - 10) /* paranoia */
 236                 len = 512/4*3 - 10;
 237         bb_uuencode(buf, str, len, bb_uuenc_tbl_base64);
 238         return buf;
 239 }
 240 #endif
 241
 242
 243 static FILE *open_socket(len_and_sockaddr *lsa)
 244 {
 245         FILE *fp;
 246
 247         /* glibc 2.4 seems to try seeking on it - ??! */
 248         /* hopefully it understands what ESPIPE means... */
 249         fp = fdopen(xconnect_stream(lsa), "r+");
 250         if (fp == NULL)
 251                 bb_perror_msg_and_die("fdopen");
 252
 253         return fp;
 254 }
 255
 256
 257 static int ftpcmd(const char *s1, const char *s2, FILE *fp, char *buf)
 258 {
 259         int result;
 260         if (s1) {
 261                 if (!s2) s2 = "";
 262                 fprintf(fp, "%s%s\r\n", s1, s2);
 263                 fflush(fp);
 264         }
 265
 266         do {
 267                 char *buf_ptr;
 268
 269                 if (fgets(buf, 510, fp) == NULL) {
 270                         bb_perror_msg_and_die("error getting response");
 271                 }
 272                 buf_ptr = strstr(buf, "\r\n");
 273                 if (buf_ptr) {
 274                         *buf_ptr = '\0';
 275                 }
 276         } while (!isdigit(buf[0]) || buf[3] != ' ');
 277
 278         buf[3] = '\0';
 279         result = xatoi_u(buf);
 280         buf[3] = ' ';
 281         return result;
 282 }
 283
 284
 285 static void parse_url(char *src_url, struct host_info *h)
 286 {
 287         char *url, *p, *sp;
 288
 289         /* h->allocated = */ url = xstrdup(src_url);
 290
 291         if (strncmp(url, "http://", 7) == 0) {
 292                 h->port = bb_lookup_port("http", "tcp", 80);
 293                 h->host = url + 7;
 294                 h->is_ftp = 0;
 295         } else if (strncmp(url, "ftp://", 6) == 0) {
 296                 h->port = bb_lookup_port("ftp", "tcp", 21);
 297                 h->host = url + 6;
 298                 h->is_ftp = 1;
 299         } else
 300                 bb_error_msg_and_die("not an http or ftp url: %s", url);
 301
 302         // FYI:
 303         // "Real" wget 'http://busybox.net?var=a/b' sends this request:
 304         //   'GET /?var=a/b HTTP 1.0'
 305         //   and saves 'index.html?var=a%2Fb' (we save 'b')
 306         // wget 'http://busybox.net?login=john@doe':
 307         //   request: 'GET /?login=john@doe HTTP/1.0'
 308         //   saves: 'index.html?login=john@doe' (we save '?login=john@doe')
 309         // wget 'http://busybox.net#test/test':
 310         //   request: 'GET / HTTP/1.0'
 311         //   saves: 'index.html' (we save 'test')
 312         //
 313         // We also don't add unique .N suffix if file exists...
 314         sp = strchr(h->host, '/');
 315         p = strchr(h->host, '?'); if (!sp || (p && sp > p)) sp = p;
 316         p = strchr(h->host, '#'); if (!sp || (p && sp > p)) sp = p;
 317         if (!sp) {
 318                 h->path = "";
 319         } else if (*sp == '/') {
 320                 *sp = '\0';
 321                 h->path = sp + 1;
 322         } else { // '#' or '?'
 323                 // http://busybox.net?login=john@doe is a valid URL
 324                 // memmove converts to:
 325                 // http:/busybox.nett?login=john@doe...
 326                 memmove(h->host - 1, h->host, sp - h->host);
 327                 h->host--;
 328                 sp[-1] = '\0';
 329                 h->path = sp;
 330         }
 331
 332         // We used to set h->user to NULL here, but this interferes
 333         // with handling of code 302 ("object was moved")
 334
 335         sp = strrchr(h->host, '@');
 336         if (sp != NULL) {
 337                 h->user = h->host;
 338                 *sp = '\0';
 339                 h->host = sp + 1;
 340         }
 341
 342         sp = h->host;
 343 }
 344
 345
 346 static char *gethdr(char *buf, size_t bufsiz, FILE *fp /*, int *istrunc*/)
 347 {
 348         char *s, *hdrval;
 349         int c;
 350
 351         /* *istrunc = 0; */
 352
 353         /* retrieve header line */
 354         if (fgets(buf, bufsiz, fp) == NULL)
 355                 return NULL;
 356
 357         /* see if we are at the end of the headers */
 358         for (s = buf; *s == '\r'; ++s)
 359                 continue;
 360         if (*s == '\n')
 361                 return NULL;
 362
 363         /* convert the header name to lower case */
 364         for (s = buf; isalnum(*s) || *s == '-' || *s == '.'; ++s)
 365                 *s = tolower(*s);
 366
 367         /* verify we are at the end of the header name */
 368         if (*s != ':')
 369                 bb_error_msg_and_die("bad header line: %s", buf);
 370
 371         /* locate the start of the header value */
 372         *s++ = '\0';
 373         hdrval = skip_whitespace(s);
 374
 375         /* locate the end of header */
 376         while (*s && *s != '\r' && *s != '\n')
 377                 ++s;
 378
 379         /* end of header found */
 380         if (*s) {
 381                 *s = '\0';
 382                 return hdrval;
 383         }
 384
 385         /* Rats! The buffer isn't big enough to hold the entire header value. */
 386         while (c = getc(fp), c != EOF && c != '\n')
 387                 continue;
 388         /* *istrunc = 1; */
 389         return hdrval;
 390 }
 391
 392 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
 393 static char *URL_escape(const char *str)
 394 {
 395         /* URL encode, see RFC 2396 */
 396         char *dst;
 397         char *res = dst = xmalloc(strlen(str) * 3 + 1);
 398         unsigned char c;
 399
 400         while (1) {
 401                 c = *str++;
 402                 if (c == '\0'
 403                 /* || strchr("!&'()*-.=_~", c) - more code */
 404                  || c == '!'
 405                  || c == '&'
 406                  || c == '\''
 407                  || c == '('
 408                  || c == ')'
 409                  || c == '*'
 410                  || c == '-'
 411                  || c == '.'
 412                  || c == '='
 413                  || c == '_'
 414                  || c == '~'
 415                  || (c >= '0' && c <= '9')
 416                  || ((c|0x20) >= 'a' && (c|0x20) <= 'z')
 417                 ) {
 418                         *dst++ = c;
 419                         if (c == '\0')
 420                                 return res;
 421                 } else {
 422                         *dst++ = '%';
 423                         *dst++ = bb_hexdigits_upcase[c >> 4];
 424                         *dst++ = bb_hexdigits_upcase[c & 0xf];
 425                 }
 426         }
 427 }
 428 #endif
 429
 430 int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
 431 int wget_main(int argc UNUSED_PARAM, char **argv)
 432 {
 433         char buf[512];
 434         struct host_info server, target;
 435         len_and_sockaddr *lsa;
 436         int status;
 437         int port;
 438         int try = 5;
 439         unsigned opt;
 440         char *str;
 441         char *proxy = 0;
 442         char *dir_prefix = NULL;
 443 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
 444         char *post_data;
 445         char *extra_headers = NULL;
 446         llist_t *headers_llist = NULL;
 447 #endif
 448         FILE *sfp = NULL;               /* socket to web/ftp server         */
 449         FILE *dfp;                      /* socket to ftp server (data)      */
 450         char *fname_out;                /* where to direct output (-O)      */
 451         bool got_clen = 0;              /* got content-length: from server  */
 452         int output_fd = -1;
 453         bool use_proxy = 1;             /* Use proxies if env vars are set  */
 454         const char *proxy_flag = "on";  /* Use proxies if env vars are set  */
 455         const char *user_agent = "Wget";/* "User-Agent" header field        */
 456
 457         static const char keywords[] ALIGN1 =
 458                 "content-length\0""transfer-encoding\0""chunked\0""location\0";
 459         enum {
 460                 KEY_content_length = 1, KEY_transfer_encoding, KEY_chunked, KEY_location
 461         };
 462         enum {
 463                 WGET_OPT_CONTINUE   = (1 << 0),
 464                 WGET_OPT_SPIDER     = (1 << 1),
 465                 WGET_OPT_QUIET      = (1 << 2),
 466                 WGET_OPT_OUTNAME    = (1 << 3),
 467                 WGET_OPT_PREFIX     = (1 << 4),
 468                 WGET_OPT_PROXY      = (1 << 5),
 469                 WGET_OPT_USER_AGENT = (1 << 6),
 470                 WGET_OPT_RETRIES    = (1 << 7),
 471                 WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 8),
 472                 WGET_OPT_PASSIVE    = (1 << 9),
 473                 WGET_OPT_HEADER     = (1 << 10) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
 474                 WGET_OPT_POST_DATA  = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
 475         };
 476 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
 477         static const char wget_longopts[] ALIGN1 =
 478                 /* name, has_arg, val */
 479                 "continue\0"         No_argument       "c"
 480                 "spider\0"           No_argument       "s"
 481                 "quiet\0"            No_argument       "q"
 482                 "output-document\0"  Required_argument "O"
 483                 "directory-prefix\0" Required_argument "P"
 484                 "proxy\0"            Required_argument "Y"
 485                 "user-agent\0"       Required_argument "U"
 486                 /* Ignored: */
 487                 // "tries\0"            Required_argument "t"
 488                 // "timeout\0"          Required_argument "T"
 489                 /* Ignored (we always use PASV): */
 490                 "passive-ftp\0"      No_argument       "\xff"
 491                 "header\0"           Required_argument "\xfe"
 492                 "post-data\0"        Required_argument "\xfd"
 493                 ;
 494 #endif
 495
 496         INIT_G();
 497
 498 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
 499         applet_long_options = wget_longopts;
 500 #endif
 501         /* server.allocated = target.allocated = NULL; */
 502         opt_complementary = "-1" USE_FEATURE_WGET_LONG_OPTIONS(":\xfe::");
 503         opt = getopt32(argv, "csqO:P:Y:U:" /*ignored:*/ "t:T:",
 504                                 &fname_out, &dir_prefix,
 505                                 &proxy_flag, &user_agent,
 506                                 NULL, /* -t RETRIES */
 507                                 NULL /* -T NETWORK_READ_TIMEOUT */
 508                                 USE_FEATURE_WGET_LONG_OPTIONS(, &headers_llist)
 509                                 USE_FEATURE_WGET_LONG_OPTIONS(, &post_data)
 510                                 );
 511         if (strcmp(proxy_flag, "off") == 0) {
 512                 /* Use the proxy if necessary */
 513                 use_proxy = 0;
 514         }
 515 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
 516         if (headers_llist) {
 517                 int size = 1;
 518                 char *cp;
 519                 llist_t *ll = headers_llist;
 520                 while (ll) {
 521                         size += strlen(ll->data) + 2;
 522                         ll = ll->link;
 523                 }
 524                 extra_headers = cp = xmalloc(size);
 525                 while (headers_llist) {
 526                         cp += sprintf(cp, "%s\r\n", (char*)llist_pop(&headers_llist));
 527                 }
 528         }
 529 #endif
 530
 531         target.user = NULL;
 532         parse_url(argv[optind], &target);
 533         server.host = target.host;
 534         server.port = target.port;
 535
 536         /* Use the proxy if necessary */
 537         if (use_proxy) {
 538                 proxy = getenv(target.is_ftp ? "ftp_proxy" : "http_proxy");
 539                 if (proxy && *proxy) {
 540                         server.user = NULL;
 541                         parse_url(proxy, &server);
 542                 } else {
 543                         use_proxy = 0;
 544                 }
 545         }
 546
 547         /* Guess an output filename, if there was no -O FILE */
 548         if (!(opt & WGET_OPT_OUTNAME)) {
 549                 fname_out = bb_get_last_path_component_nostrip(target.path);
 550                 /* handle "wget http://kernel.org//" */
 551                 if (fname_out[0] == '/' || !fname_out[0])
 552                         fname_out = (char*)"index.html";
 553                 /* -P DIR is considered only if there was no -O FILE */
 554                 if (dir_prefix)
 555                         fname_out = concat_path_file(dir_prefix, fname_out);
 556         } else {
 557                 if (LONE_DASH(fname_out)) {
 558                         /* -O - */
 559                         output_fd = 1;
 560                         opt &= ~WGET_OPT_CONTINUE;
 561                 }
 562         }
 563 #if ENABLE_FEATURE_WGET_STATUSBAR
 564         curfile = bb_get_last_path_component_nostrip(fname_out);
 565 #endif
 566
 567         /* Impossible?
 568         if ((opt & WGET_OPT_CONTINUE) && !fname_out)
 569                 bb_error_msg_and_die("cannot specify continue (-c) without a filename (-O)"); */
 570
 571         /* Determine where to start transfer */
 572         if (opt & WGET_OPT_CONTINUE) {
 573                 output_fd = open(fname_out, O_WRONLY);
 574                 if (output_fd >= 0) {
 575                         beg_range = xlseek(output_fd, 0, SEEK_END);
 576                 }
 577                 /* File doesn't exist. We do not create file here yet.
 578                    We are not sure it exists on remove side */
 579         }
 580
 581         /* We want to do exactly _one_ DNS lookup, since some
 582          * sites (i.e. ftp.us.debian.org) use round-robin DNS
 583          * and we want to connect to only one IP... */
 584         lsa = xhost2sockaddr(server.host, server.port);
 585         if (!(opt & WGET_OPT_QUIET)) {
 586                 fprintf(stderr, "Connecting to %s (%s)\n", server.host,
 587                                 xmalloc_sockaddr2dotted(&lsa->u.sa));
 588                 /* We leak result of xmalloc_sockaddr2dotted */
 589         }
 590
 591         if (use_proxy || !target.is_ftp) {
 592                 /*
 593                  *  HTTP session
 594                  */
 595                 do {
 596                         got_clen = 0;
 597                         chunked = 0;
 598
 599                         if (!--try)
 600                                 bb_error_msg_and_die("too many redirections");
 601
 602                         /* Open socket to http server */
 603                         if (sfp) fclose(sfp);
 604                         sfp = open_socket(lsa);
 605
 606                         /* Send HTTP request.  */
 607                         if (use_proxy) {
 608                                 fprintf(sfp, "GET %stp://%s/%s HTTP/1.1\r\n",
 609                                         target.is_ftp ? "f" : "ht", target.host,
 610                                         target.path);
 611                         } else {
 612                                 if (opt & WGET_OPT_POST_DATA)
 613                                         fprintf(sfp, "POST /%s HTTP/1.1\r\n", target.path);
 614                                 else
 615                                         fprintf(sfp, "GET /%s HTTP/1.1\r\n", target.path);
 616                         }
 617
 618                         fprintf(sfp, "Host: %s\r\nUser-Agent: %s\r\n",
 619                                 target.host, user_agent);
 620
 621 #if ENABLE_FEATURE_WGET_AUTHENTICATION
 622                         if (target.user) {
 623                                 fprintf(sfp, "Proxy-Authorization: Basic %s\r\n"+6,
 624                                         base64enc_512(buf, target.user));
 625                         }
 626                         if (use_proxy && server.user) {
 627                                 fprintf(sfp, "Proxy-Authorization: Basic %s\r\n",
 628                                         base64enc_512(buf, server.user));
 629                         }
 630 #endif
 631
 632                         if (beg_range)
 633                                 fprintf(sfp, "Range: bytes=%"OFF_FMT"u-\r\n", beg_range);
 634 #if ENABLE_FEATURE_WGET_LONG_OPTIONS
 635                         if (extra_headers)
 636                                 fputs(extra_headers, sfp);
 637
 638                         if (opt & WGET_OPT_POST_DATA) {
 639                                 char *estr = URL_escape(post_data);
 640                                 fprintf(sfp, "Content-Type: application/x-www-form-urlencoded\r\n");
 641                                 fprintf(sfp, "Content-Length: %u\r\n" "\r\n" "%s",
 642                                                 (int) strlen(estr), estr);
 643                                 /*fprintf(sfp, "Connection: Keep-Alive\r\n\r\n");*/
 644                                 /*fprintf(sfp, "%s\r\n", estr);*/
 645                                 free(estr);
 646                         } else
 647 #endif
 648                         { /* If "Connection:" is needed, document why */
 649                                 fprintf(sfp, /* "Connection: close\r\n" */ "\r\n");
 650                         }
 651
 652                         /*
 653                          * Retrieve HTTP response line and check for "200" status code.
 654                          */
 655  read_response:
 656                         if (fgets(buf, sizeof(buf), sfp) == NULL)
 657                                 bb_error_msg_and_die("no response from server");
 658
 659                         str = buf;
 660                         str = skip_non_whitespace(str);
 661                         str = skip_whitespace(str);
 662                         // FIXME: no error check
 663                         // xatou wouldn't work: "200 OK"
 664                         status = atoi(str);
 665                         switch (status) {
 666                         case 0:
 667                         case 100:
 668                                 while (gethdr(buf, sizeof(buf), sfp /*, &n*/) != NULL)
 669                                         /* eat all remaining headers */;
 670                                 goto read_response;
 671                         case 200:
 672 /*
 673 Response 204 doesn't say "null file", it says "metadata
 674 has changed but data didn't":
 675
 676 "10.2.5 204 No Content
 677 The server has fulfilled the request but does not need to return
 678 an entity-body, and might want to return updated metainformation.
 679 The response MAY include new or updated metainformation in the form
 680 of entity-headers, which if present SHOULD be associated with
 681 the requested variant.
 682
 683 If the client is a user agent, it SHOULD NOT change its document
 684 view from that which caused the request to be sent. This response
 685 is primarily intended to allow input for actions to take place
 686 without causing a change to the user agent's active document view,
 687 although any new or updated metainformation SHOULD be applied
 688 to the document currently in the user agent's active view.
 689
 690 The 204 response MUST NOT include a message-body, and thus
 691 is always terminated by the first empty line after the header fields."
 692
 693 However, in real world it was observed that some web servers
 694 (e.g. Boa/0.94.14rc21) simply use code 204 when file size is zero.
 695 */
 696                         case 204:
 697                                 break;
 698                         case 300:       /* redirection */
 699                         case 301:
 700                         case 302:
 701                         case 303:
 702                                 break;
 703                         case 206:
 704                                 if (beg_range)
 705                                         break;
 706                                 /* fall through */
 707                         default:
 708                                 /* Show first line only and kill any ESC tricks */
 709                                 buf[strcspn(buf, "\n\r\x1b")] = '\0';
 710                                 bb_error_msg_and_die("server returned error: %s", buf);
 711                         }
 712
 713                         /*
 714                          * Retrieve HTTP headers.
 715                          */
 716                         while ((str = gethdr(buf, sizeof(buf), sfp /*, &n*/)) != NULL) {
 717                                 /* gethdr did already convert the "FOO:" string to lowercase */
 718                                 smalluint key;
 719                                 /* strip trailing whitespace */
 720                                 char *s = strchrnul(str, '\0') - 1;
 721                                 while (s >= str && (*s == ' ' || *s == '\t')) {
 722                                         *s = '\0';
 723                                         s--;
 724                                 }
 725                                 key = index_in_strings(keywords, buf) + 1;
 726                                 if (key == KEY_content_length) {
 727                                         content_len = BB_STRTOOFF(str, NULL, 10);
 728                                         if (content_len < 0 || errno) {
 729                                                 bb_error_msg_and_die("content-length %s is garbage", str);
 730                                         }
 731                                         got_clen = 1;
 732                                         continue;
 733                                 }
 734                                 if (key == KEY_transfer_encoding) {
 735                                         if (index_in_strings(keywords, str_tolower(str)) + 1 != KEY_chunked)
 736                                                 bb_error_msg_and_die("transfer encoding '%s' is not supported", str);
 737                                         chunked = got_clen = 1;
 738                                 }
 739                                 if (key == KEY_location) {
 740                                         if (str[0] == '/')
 741                                                 /* free(target.allocated); */
 742                                                 target.path = /* target.allocated = */ xstrdup(str+1);
 743                                         else {
 744                                                 parse_url(str, &target);
 745                                                 if (use_proxy == 0) {
 746                                                         server.host = target.host;
 747                                                         server.port = target.port;
 748                                                 }
 749                                                 free(lsa);
 750                                                 lsa = xhost2sockaddr(server.host, server.port);
 751                                                 break;
 752                                         }
 753                                 }
 754                         }
 755                 } while (status >= 300);
 756
 757                 dfp = sfp;
 758
 759         } else {
 760
 761                 /*
 762                  *  FTP session
 763                  */
 764                 if (!target.user)
 765                         target.user = xstrdup("anonymous:busybox@");
 766
 767                 sfp = open_socket(lsa);
 768                 if (ftpcmd(NULL, NULL, sfp, buf) != 220)
 769                         bb_error_msg_and_die("%s", buf+4);
 770
 771                 /*
 772                  * Splitting username:password pair,
 773                  * trying to log in
 774                  */
 775                 str = strchr(target.user, ':');
 776                 if (str)
 777                         *(str++) = '\0';
 778                 switch (ftpcmd("USER ", target.user, sfp, buf)) {
 779                 case 230:
 780                         break;
 781                 case 331:
 782                         if (ftpcmd("PASS ", str, sfp, buf) == 230)
 783                                 break;
 784                         /* fall through (failed login) */
 785                 default:
 786                         bb_error_msg_and_die("ftp login: %s", buf+4);
 787                 }
 788
 789                 ftpcmd("TYPE I", NULL, sfp, buf);
 790
 791                 /*
 792                  * Querying file size
 793                  */
 794                 if (ftpcmd("SIZE ", target.path, sfp, buf) == 213) {
 795                         content_len = BB_STRTOOFF(buf+4, NULL, 10);
 796                         if (content_len < 0 || errno) {
 797                                 bb_error_msg_and_die("SIZE value is garbage");
 798                         }
 799                         got_clen = 1;
 800                 }
 801
 802                 /*
 803                  * Entering passive mode
 804                  */
 805                 if (ftpcmd("PASV", NULL, sfp, buf) != 227) {
 806  pasv_error:
 807                         bb_error_msg_and_die("bad response to %s: %s", "PASV", buf);
 808                 }
 809                 // Response is "227 garbageN1,N2,N3,N4,P1,P2[)garbage]
 810                 // Server's IP is N1.N2.N3.N4 (we ignore it)
 811                 // Server's port for data connection is P1*256+P2
 812                 str = strrchr(buf, ')');
 813                 if (str) str[0] = '\0';
 814                 str = strrchr(buf, ',');
 815                 if (!str) goto pasv_error;
 816                 port = xatou_range(str+1, 0, 255);
 817                 *str = '\0';
 818                 str = strrchr(buf, ',');
 819                 if (!str) goto pasv_error;
 820                 port += xatou_range(str+1, 0, 255) * 256;
 821                 set_nport(lsa, htons(port));
 822                 dfp = open_socket(lsa);
 823
 824                 if (beg_range) {
 825                         sprintf(buf, "REST %"OFF_FMT"u", beg_range);
 826                         if (ftpcmd(buf, NULL, sfp, buf) == 350)
 827                                 content_len -= beg_range;
 828                 }
 829
 830                 if (ftpcmd("RETR ", target.path, sfp, buf) > 150)
 831                         bb_error_msg_and_die("bad response to %s: %s", "RETR", buf);
 832         }
 833
 834         if (opt & WGET_OPT_SPIDER) {
 835                 if (ENABLE_FEATURE_CLEAN_UP)
 836                         fclose(sfp);
 837                 return EXIT_SUCCESS;
 838         }
 839
 840         /*
 841          * Retrieve file
 842          */
 843
 844         /* Do it before progress_meter (want to have nice error message) */
 845         if (output_fd < 0) {
 846                 int o_flags = O_WRONLY | O_CREAT | O_TRUNC | O_EXCL;
 847                 /* compat with wget: -O FILE can overwrite */
 848                 if (opt & WGET_OPT_OUTNAME)
 849                         o_flags = O_WRONLY | O_CREAT | O_TRUNC;
 850                 output_fd = xopen(fname_out, o_flags);
 851         }
 852
 853         if (!(opt & WGET_OPT_QUIET))
 854                 progress_meter(-1);
 855
 856         if (chunked)
 857                 goto get_clen;
 858
 859         /* Loops only if chunked */
 860         while (1) {
 861                 while (1) {
 862                         int n;
 863                         unsigned rdsz;
 864
 865                         rdsz = sizeof(buf);
 866                         if (got_clen) {
 867                                 if (content_len < (off_t)sizeof(buf)) {
 868                                         if ((int)content_len <= 0)
 869                                                 break;
 870                                         rdsz = (unsigned)content_len;
 871                                 }
 872                         }
 873                         n = safe_fread(buf, rdsz, dfp);
 874                         if (n <= 0) {
 875                                 if (ferror(dfp)) {
 876                                         /* perror will not work: ferror doesn't set errno */
 877                                         bb_error_msg_and_die(bb_msg_read_error);
 878                                 }
 879                                 break;
 880                         }
 881                         xwrite(output_fd, buf, n);
 882 #if ENABLE_FEATURE_WGET_STATUSBAR
 883                         transferred += n;
 884 #endif
 885                         if (got_clen)
 886                                 content_len -= n;
 887                 }
 888
 889                 if (!chunked)
 890                         break;
 891
 892                 safe_fgets(buf, sizeof(buf), dfp); /* This is a newline */
 893  get_clen:
 894                 safe_fgets(buf, sizeof(buf), dfp);
 895                 content_len = STRTOOFF(buf, NULL, 16);
 896                 /* FIXME: error check? */
 897                 if (content_len == 0)
 898                         break; /* all done! */
 899                 got_clen = 1;
 900         }
 901
 902         if (!(opt & WGET_OPT_QUIET))
 903                 progress_meter(0);
 904         if (close(output_fd))
 905                 bb_perror_msg_and_die("close failed");
 906
 907         if ((use_proxy == 0) && target.is_ftp) {
 908                 fclose(dfp);
 909                 if (ftpcmd(NULL, NULL, sfp, buf) != 226)
 910                         bb_error_msg_and_die("ftp error: %s", buf+4);
 911                 /* ftpcmd("QUIT", NULL, sfp, buf); - why bother? */
 912         }
 913
 914         return EXIT_SUCCESS;
 915 }