contrib/tnftp/src/fetch.c

   1 /*      $NetBSD: fetch.c,v 1.26 2021/08/27 01:38:49 lukem Exp $ */
   2 /*      from    NetBSD: fetch.c,v 1.234 2021/08/01 15:29:30 andvar Exp  */
   3
   4 /*-
   5  * Copyright (c) 1997-2015 The NetBSD Foundation, Inc.
   6  * All rights reserved.
   7  *
   8  * This code is derived from software contributed to The NetBSD Foundation
   9  * by Luke Mewburn.
  10  *
  11  * This code is derived from software contributed to The NetBSD Foundation
  12  * by Scott Aaron Bamford.
  13  *
  14  * This code is derived from software contributed to The NetBSD Foundation
  15  * by Thomas Klausner.
  16  *
  17  * Redistribution and use in source and binary forms, with or without
  18  * modification, are permitted provided that the following conditions
  19  * are met:
  20  * 1. Redistributions of source code must retain the above copyright
  21  *    notice, this list of conditions and the following disclaimer.
  22  * 2. Redistributions in binary form must reproduce the above copyright
  23  *    notice, this list of conditions and the following disclaimer in the
  24  *    documentation and/or other materials provided with the distribution.
  25  *
  26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
  27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
  30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  36  * POSSIBILITY OF SUCH DAMAGE.
  37  */
  38
  39 #include "tnftp.h"
  40
  41 #if 0   /* tnftp */
  42
  43 #include <sys/cdefs.h>
  44 #ifndef lint
  45 __RCSID(" NetBSD: fetch.c,v 1.234 2021/08/01 15:29:30 andvar Exp  ");
  46 #endif /* not lint */
  47
  48 /*
  49  * FTP User Program -- Command line file retrieval
  50  */
  51
  52 #include <sys/types.h>
  53 #include <sys/param.h>
  54 #include <sys/socket.h>
  55 #include <sys/stat.h>
  56 #include <sys/time.h>
  57
  58 #include <netinet/in.h>
  59
  60 #include <arpa/ftp.h>
  61 #include <arpa/inet.h>
  62
  63 #include <assert.h>
  64 #include <ctype.h>
  65 #include <err.h>
  66 #include <errno.h>
  67 #include <netdb.h>
  68 #include <fcntl.h>
  69 #include <stdio.h>
  70 #include <stdlib.h>
  71 #include <string.h>
  72 #include <unistd.h>
  73 #include <time.h>
  74
  75 #endif  /* tnftp */
  76
  77 #include "ssl.h"
  78 #include "ftp_var.h"
  79 #include "version.h"
  80
  81 typedef enum {
  82         UNKNOWN_URL_T=-1,
  83         HTTP_URL_T,
  84         HTTPS_URL_T,
  85         FTP_URL_T,
  86         FILE_URL_T,
  87         CLASSIC_URL_T
  88 } url_t;
  89
  90 struct authinfo {
  91         char *auth;
  92         char *user;
  93         char *pass;
  94 };
  95
  96 struct urlinfo {
  97         char *host;
  98         char *port;
  99         char *path;
 100         url_t utype;
 101         in_port_t portnum;
 102 };
 103
 104 struct posinfo {
 105         off_t rangestart;
 106         off_t rangeend;
 107         off_t entitylen;
 108 };
 109
 110 __dead static void      aborthttp(int);
 111 __dead static void      timeouthttp(int);
 112 #ifndef NO_AUTH
 113 static int      auth_url(const char *, char **, const struct authinfo *);
 114 static void     base64_encode(const unsigned char *, size_t, unsigned char *);
 115 #endif
 116 static int      go_fetch(const char *);
 117 static int      fetch_ftp(const char *);
 118 static int      fetch_url(const char *, const char *, char *, char *);
 119 static const char *match_token(const char **, const char *);
 120 static int      parse_url(const char *, const char *, struct urlinfo *,
 121     struct authinfo *);
 122 static void     url_decode(char *);
 123 static void     freeauthinfo(struct authinfo *);
 124 static void     freeurlinfo(struct urlinfo *);
 125
 126 static int      redirect_loop;
 127
 128
 129 #define STRNEQUAL(a,b)  (strncasecmp((a), (b), sizeof((b))-1) == 0)
 130 #define ISLWS(x)        ((x)=='\r' || (x)=='\n' || (x)==' ' || (x)=='\t')
 131 #define SKIPLWS(x)      do { while (ISLWS((*x))) x++; } while (0)
 132
 133
 134 #define ABOUT_URL       "about:"        /* propaganda */
 135 #define FILE_URL        "file://"       /* file URL prefix */
 136 #define FTP_URL         "ftp://"        /* ftp URL prefix */
 137 #define HTTP_URL        "http://"       /* http URL prefix */
 138 #ifdef WITH_SSL
 139 #define HTTPS_URL       "https://"      /* https URL prefix */
 140
 141 #define IS_HTTP_TYPE(urltype) \
 142         (((urltype) == HTTP_URL_T) || ((urltype) == HTTPS_URL_T))
 143 #else
 144 #define IS_HTTP_TYPE(urltype) \
 145         ((urltype) == HTTP_URL_T)
 146 #endif
 147
 148 /**
 149  * fwrite(3) replacement that just uses write(2). Many stdio implementations
 150  * don't handle interrupts properly and corrupt the output. We are taking
 151  * alarm interrupts because of the progress bar.
 152  *
 153  * Assumes `fp' is pristine with no prior I/O calls on it.
 154  */
 155 static size_t
 156 maxwrite(const void *buf, size_t size, size_t nmemb, FILE *fp)
 157 {
 158         const char *p = buf;
 159         ssize_t nwr = 0;
 160         ssize_t n;
 161         int fd = fileno(fp);
 162
 163         size *= nmemb;  /* assume no overflow */
 164
 165         while (size > 0) {
 166                 if ((n = write(fd, p, size)) == -1) {
 167                         switch (errno) {
 168                         case EINTR:
 169                         case EAGAIN:
 170 #if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN
 171                         case EWOULDBLOCK:
 172 #endif
 173                                 continue;
 174                         default:
 175                                 return nwr;
 176                         }
 177                 }
 178                 p += n;
 179                 nwr += n;
 180                 size -= n;
 181         }
 182         return nwr;
 183 }
 184
 185 /*
 186  * Determine if token is the next word in buf (case insensitive).
 187  * If so, advance buf past the token and any trailing LWS, and
 188  * return a pointer to the token (in buf).  Otherwise, return NULL.
 189  * token may be preceded by LWS.
 190  * token must be followed by LWS or NUL.  (I.e, don't partial match).
 191  */
 192 static const char *
 193 match_token(const char **buf, const char *token)
 194 {
 195         const char      *p, *orig;
 196         size_t          tlen;
 197
 198         tlen = strlen(token);
 199         p = *buf;
 200         SKIPLWS(p);
 201         orig = p;
 202         if (strncasecmp(p, token, tlen) != 0)
 203                 return NULL;
 204         p += tlen;
 205         if (*p != '\0' && !ISLWS(*p))
 206                 return NULL;
 207         SKIPLWS(p);
 208         orig = *buf;
 209         *buf = p;
 210         return orig;
 211 }
 212
 213 static void
 214 initposinfo(struct posinfo *pi)
 215 {
 216         pi->rangestart = pi->rangeend = pi->entitylen = -1;
 217 }
 218
 219 static void
 220 initauthinfo(struct authinfo *ai, char *auth)
 221 {
 222         ai->auth = auth;
 223         ai->user = ai->pass = 0;
 224 }
 225
 226 static void
 227 freeauthinfo(struct authinfo *a)
 228 {
 229         FREEPTR(a->user);
 230         if (a->pass != NULL)
 231                 memset(a->pass, 0, strlen(a->pass));
 232         FREEPTR(a->pass);
 233 }
 234
 235 static void
 236 initurlinfo(struct urlinfo *ui)
 237 {
 238         ui->host = ui->port = ui->path = 0;
 239         ui->utype = UNKNOWN_URL_T;
 240         ui->portnum = 0;
 241 }
 242
 243 static void
 244 copyurlinfo(struct urlinfo *dui, struct urlinfo *sui)
 245 {
 246         dui->host = ftp_strdup(sui->host);
 247         dui->port = ftp_strdup(sui->port);
 248         dui->path = ftp_strdup(sui->path);
 249         dui->utype = sui->utype;
 250         dui->portnum = sui->portnum;
 251 }
 252
 253 static void
 254 freeurlinfo(struct urlinfo *ui)
 255 {
 256         FREEPTR(ui->host);
 257         FREEPTR(ui->port);
 258         FREEPTR(ui->path);
 259 }
 260
 261 #ifndef NO_AUTH
 262 /*
 263  * Generate authorization response based on given authentication challenge.
 264  * Returns -1 if an error occurred, otherwise 0.
 265  * Sets response to a malloc(3)ed string; caller should free.
 266  */
 267 static int
 268 auth_url(const char *challenge, char **response, const struct authinfo *auth)
 269 {
 270         const char      *cp, *scheme, *errormsg;
 271         char            *ep, *clear, *realm;
 272         char             uuser[BUFSIZ], *gotpass;
 273         const char      *upass;
 274         int              rval;
 275         size_t           len, clen, rlen;
 276
 277         *response = NULL;
 278         clear = realm = NULL;
 279         rval = -1;
 280         cp = challenge;
 281         scheme = "Basic";       /* only support Basic authentication */
 282         gotpass = NULL;
 283
 284         DPRINTF("auth_url: challenge `%s'\n", challenge);
 285
 286         if (! match_token(&cp, scheme)) {
 287                 warnx("Unsupported authentication challenge `%s'",
 288                     challenge);
 289                 goto cleanup_auth_url;
 290         }
 291
 292 #define REALM "realm=\""
 293         if (STRNEQUAL(cp, REALM))
 294                 cp += sizeof(REALM) - 1;
 295         else {
 296                 warnx("Unsupported authentication challenge `%s'",
 297                     challenge);
 298                 goto cleanup_auth_url;
 299         }
 300 /* XXX: need to improve quoted-string parsing to support \ quoting, etc. */
 301         if ((ep = strchr(cp, '\"')) != NULL) {
 302                 len = ep - cp;
 303                 realm = (char *)ftp_malloc(len + 1);
 304                 (void)strlcpy(realm, cp, len + 1);
 305         } else {
 306                 warnx("Unsupported authentication challenge `%s'",
 307                     challenge);
 308                 goto cleanup_auth_url;
 309         }
 310
 311         fprintf(ttyout, "Username for `%s': ", realm);
 312         if (auth->user != NULL) {
 313                 (void)strlcpy(uuser, auth->user, sizeof(uuser));
 314                 fprintf(ttyout, "%s\n", uuser);
 315         } else {
 316                 (void)fflush(ttyout);
 317                 if (get_line(stdin, uuser, sizeof(uuser), &errormsg) < 0) {
 318                         warnx("%s; can't authenticate", errormsg);
 319                         goto cleanup_auth_url;
 320                 }
 321         }
 322         if (auth->pass != NULL)
 323                 upass = auth->pass;
 324         else {
 325                 gotpass = getpass("Password: ");
 326                 if (gotpass == NULL) {
 327                         warnx("Can't read password");
 328                         goto cleanup_auth_url;
 329                 }
 330                 upass = gotpass;
 331         }
 332
 333         clen = strlen(uuser) + strlen(upass) + 2;       /* user + ":" + pass + "\0" */
 334         clear = (char *)ftp_malloc(clen);
 335         (void)strlcpy(clear, uuser, clen);
 336         (void)strlcat(clear, ":", clen);
 337         (void)strlcat(clear, upass, clen);
 338         if (gotpass)
 339                 memset(gotpass, 0, strlen(gotpass));
 340
 341                                                 /* scheme + " " + enc + "\0" */
 342         rlen = strlen(scheme) + 1 + (clen + 2) * 4 / 3 + 1;
 343         *response = ftp_malloc(rlen);
 344         (void)strlcpy(*response, scheme, rlen);
 345         len = strlcat(*response, " ", rlen);
 346                         /* use  `clen - 1'  to not encode the trailing NUL */
 347         base64_encode((unsigned char *)clear, clen - 1,
 348             (unsigned char *)*response + len);
 349         memset(clear, 0, clen);
 350         rval = 0;
 351
 352  cleanup_auth_url:
 353         FREEPTR(clear);
 354         FREEPTR(realm);
 355         return (rval);
 356 }
 357
 358 /*
 359  * Encode len bytes starting at clear using base64 encoding into encoded,
 360  * which should be at least ((len + 2) * 4 / 3 + 1) in size.
 361  */
 362 static void
 363 base64_encode(const unsigned char *clear, size_t len, unsigned char *encoded)
 364 {
 365         static const unsigned char enc[] =
 366             "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 367         unsigned char   *cp;
 368         size_t   i;
 369
 370         cp = encoded;
 371         for (i = 0; i < len; i += 3) {
 372                 *(cp++) = enc[((clear[i + 0] >> 2))];
 373                 *(cp++) = enc[((clear[i + 0] << 4) & 0x30)
 374                             | ((clear[i + 1] >> 4) & 0x0f)];
 375                 *(cp++) = enc[((clear[i + 1] << 2) & 0x3c)
 376                             | ((clear[i + 2] >> 6) & 0x03)];
 377                 *(cp++) = enc[((clear[i + 2]     ) & 0x3f)];
 378         }
 379         *cp = '\0';
 380         while (i-- > len)
 381                 *(--cp) = '=';
 382 }
 383 #endif
 384
 385 /*
 386  * Decode %xx escapes in given string, `in-place'.
 387  */
 388 static void
 389 url_decode(char *url)
 390 {
 391         unsigned char *p, *q;
 392
 393         if (EMPTYSTRING(url))
 394                 return;
 395         p = q = (unsigned char *)url;
 396
 397 #define HEXTOINT(x) (x - (isdigit(x) ? '0' : (islower(x) ? 'a' : 'A') - 10))
 398         while (*p) {
 399                 if (p[0] == '%'
 400                     && p[1] && isxdigit((unsigned char)p[1])
 401                     && p[2] && isxdigit((unsigned char)p[2])) {
 402                         *q++ = HEXTOINT(p[1]) * 16 + HEXTOINT(p[2]);
 403                         p+=3;
 404                 } else
 405                         *q++ = *p++;
 406         }
 407         *q = '\0';
 408 }
 409
 410
 411 /*
 412  * Parse URL of form (per RFC 3986):
 413  *      <type>://[<user>[:<password>]@]<host>[:<port>][/<path>]
 414  * Returns -1 if a parse error occurred, otherwise 0.
 415  * It's the caller's responsibility to url_decode() the returned
 416  * user, pass and path.
 417  *
 418  * Sets type to url_t, each of the given char ** pointers to a
 419  * malloc(3)ed strings of the relevant section, and port to
 420  * the number given, or ftpport if ftp://, or httpport if http://.
 421  *
 422  * XXX: this is not totally RFC 3986 compliant; <path> will have the
 423  * leading `/' unless it's an ftp:// URL, as this makes things easier
 424  * for file:// and http:// URLs.  ftp:// URLs have the `/' between the
 425  * host and the URL-path removed, but any additional leading slashes
 426  * in the URL-path are retained (because they imply that we should
 427  * later do "CWD" with a null argument).
 428  *
 429  * Examples:
 430  *       input URL                       output path
 431  *       ---------                       -----------
 432  *      "http://host"                   "/"
 433  *      "http://host/"                  "/"
 434  *      "http://host/path"              "/path"
 435  *      "file://host/dir/file"          "dir/file"
 436  *      "ftp://host"                    ""
 437  *      "ftp://host/"                   ""
 438  *      "ftp://host//"                  "/"
 439  *      "ftp://host/dir/file"           "dir/file"
 440  *      "ftp://host//dir/file"          "/dir/file"
 441  */
 442
 443 static int
 444 parse_url(const char *url, const char *desc, struct urlinfo *ui,
 445     struct authinfo *auth)
 446 {
 447         const char      *origurl, *tport;
 448         char            *cp, *ep, *thost;
 449         size_t           len;
 450
 451         if (url == NULL || desc == NULL || ui == NULL || auth == NULL)
 452                 errx(1, "parse_url: invoked with NULL argument!");
 453         DPRINTF("parse_url: %s `%s'\n", desc, url);
 454
 455         origurl = url;
 456         tport = NULL;
 457
 458         if (STRNEQUAL(url, HTTP_URL)) {
 459                 url += sizeof(HTTP_URL) - 1;
 460                 ui->utype = HTTP_URL_T;
 461                 ui->portnum = HTTP_PORT;
 462                 tport = httpport;
 463         } else if (STRNEQUAL(url, FTP_URL)) {
 464                 url += sizeof(FTP_URL) - 1;
 465                 ui->utype = FTP_URL_T;
 466                 ui->portnum = FTP_PORT;
 467                 tport = ftpport;
 468         } else if (STRNEQUAL(url, FILE_URL)) {
 469                 url += sizeof(FILE_URL) - 1;
 470                 ui->utype = FILE_URL_T;
 471                 tport = "";
 472 #ifdef WITH_SSL
 473         } else if (STRNEQUAL(url, HTTPS_URL)) {
 474                 url += sizeof(HTTPS_URL) - 1;
 475                 ui->utype = HTTPS_URL_T;
 476                 ui->portnum = HTTPS_PORT;
 477                 tport = httpsport;
 478 #endif
 479         } else {
 480                 warnx("Invalid %s `%s'", desc, url);
 481  cleanup_parse_url:
 482                 freeauthinfo(auth);
 483                 freeurlinfo(ui);
 484                 return (-1);
 485         }
 486
 487         if (*url == '\0')
 488                 return (0);
 489
 490                         /* find [user[:pass]@]host[:port] */
 491         ep = strchr(url, '/');
 492         if (ep == NULL)
 493                 thost = ftp_strdup(url);
 494         else {
 495                 len = ep - url;
 496                 thost = (char *)ftp_malloc(len + 1);
 497                 (void)strlcpy(thost, url, len + 1);
 498                 if (ui->utype == FTP_URL_T)     /* skip first / for ftp URLs */
 499                         ep++;
 500                 ui->path = ftp_strdup(ep);
 501         }
 502
 503         cp = strchr(thost, '@');        /* look for user[:pass]@ in URLs */
 504         if (cp != NULL) {
 505                 if (ui->utype == FTP_URL_T)
 506                         anonftp = 0;    /* disable anonftp */
 507                 auth->user = thost;
 508                 *cp = '\0';
 509                 thost = ftp_strdup(cp + 1);
 510                 cp = strchr(auth->user, ':');
 511                 if (cp != NULL) {
 512                         *cp = '\0';
 513                         auth->pass = ftp_strdup(cp + 1);
 514                 }
 515                 url_decode(auth->user);
 516                 if (auth->pass)
 517                         url_decode(auth->pass);
 518         }
 519
 520 #ifdef INET6
 521                         /*
 522                          * Check if thost is an encoded IPv6 address, as per
 523                          * RFC 3986:
 524                          *      `[' ipv6-address ']'
 525                          */
 526         if (*thost == '[') {
 527                 cp = thost + 1;
 528                 if ((ep = strchr(cp, ']')) == NULL ||
 529                     (ep[1] != '\0' && ep[1] != ':')) {
 530                         warnx("Invalid address `%s' in %s `%s'",
 531                             thost, desc, origurl);
 532                         goto cleanup_parse_url;
 533                 }
 534                 len = ep - cp;          /* change `[xyz]' -> `xyz' */
 535                 memmove(thost, thost + 1, len);
 536                 thost[len] = '\0';
 537                 if (! isipv6addr(thost)) {
 538                         warnx("Invalid IPv6 address `%s' in %s `%s'",
 539                             thost, desc, origurl);
 540                         goto cleanup_parse_url;
 541                 }
 542                 cp = ep + 1;
 543                 if (*cp == ':')
 544                         cp++;
 545                 else
 546                         cp = NULL;
 547         } else
 548 #endif /* INET6 */
 549                 if ((cp = strchr(thost, ':')) != NULL)
 550                         *cp++ = '\0';
 551         ui->host = thost;
 552
 553                         /* look for [:port] */
 554         if (cp != NULL) {
 555                 unsigned long   nport;
 556
 557                 nport = strtoul(cp, &ep, 10);
 558                 if (*cp == '\0' || *ep != '\0' ||
 559                     nport < 1 || nport > MAX_IN_PORT_T) {
 560                         warnx("Unknown port `%s' in %s `%s'",
 561                             cp, desc, origurl);
 562                         goto cleanup_parse_url;
 563                 }
 564                 ui->portnum = nport;
 565                 tport = cp;
 566         }
 567
 568         if (tport != NULL)
 569                 ui->port = ftp_strdup(tport);
 570         if (ui->path == NULL) {
 571                 const char *emptypath = "/";
 572                 if (ui->utype == FTP_URL_T)     /* skip first / for ftp URLs */
 573                         emptypath++;
 574                 ui->path = ftp_strdup(emptypath);
 575         }
 576
 577         DPRINTF("parse_url: user `%s' pass `%s' host %s port %s(%d) "
 578             "path `%s'\n",
 579             STRorNULL(auth->user), STRorNULL(auth->pass),
 580             STRorNULL(ui->host), STRorNULL(ui->port),
 581             ui->portnum ? ui->portnum : -1, STRorNULL(ui->path));
 582
 583         return (0);
 584 }
 585
 586 sigjmp_buf      httpabort;
 587
 588 static int
 589 ftp_socket(const struct urlinfo *ui, void **ssl)
 590 {
 591         struct addrinfo hints, *res, *res0 = NULL;
 592         int error;
 593         int s;
 594         const char *host = ui->host;
 595         const char *port = ui->port;
 596
 597         if (ui->utype != HTTPS_URL_T)
 598                 ssl = NULL;
 599
 600         memset(&hints, 0, sizeof(hints));
 601         hints.ai_flags = 0;
 602         hints.ai_family = family;
 603         hints.ai_socktype = SOCK_STREAM;
 604         hints.ai_protocol = 0;
 605
 606         error = getaddrinfo(host, port, &hints, &res0);
 607         if (error) {
 608                 warnx("Can't LOOKUP `%s:%s': %s", host, port,
 609                     (error == EAI_SYSTEM) ? strerror(errno)
 610                                           : gai_strerror(error));
 611                 return -1;
 612         }
 613
 614         if (res0->ai_canonname)
 615                 host = res0->ai_canonname;
 616
 617         s = -1;
 618         if (ssl)
 619                 *ssl = NULL;
 620         for (res = res0; res; res = res->ai_next) {
 621                 char    hname[NI_MAXHOST], sname[NI_MAXSERV];
 622
 623                 ai_unmapped(res);
 624                 if (getnameinfo(res->ai_addr, res->ai_addrlen,
 625                     hname, sizeof(hname), sname, sizeof(sname),
 626                     NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
 627                         strlcpy(hname, "?", sizeof(hname));
 628                         strlcpy(sname, "?", sizeof(sname));
 629                 }
 630
 631                 if (verbose && res0->ai_next) {
 632 #ifdef INET6
 633                         if(res->ai_family == AF_INET6) {
 634                                 fprintf(ttyout, "Trying [%s]:%s ...\n",
 635                                     hname, sname);
 636                         } else {
 637 #endif
 638                                 fprintf(ttyout, "Trying %s:%s ...\n",
 639                                     hname, sname);
 640 #ifdef INET6
 641                         }
 642 #endif
 643                 }
 644
 645                 s = socket(res->ai_family, SOCK_STREAM, res->ai_protocol);
 646                 if (s < 0) {
 647                         warn(
 648                             "Can't create socket for connection to "
 649                             "`%s:%s'", hname, sname);
 650                         continue;
 651                 }
 652
 653                 if (ftp_connect(s, res->ai_addr, res->ai_addrlen,
 654                     verbose || !res->ai_next) < 0) {
 655                         close(s);
 656                         s = -1;
 657                         continue;
 658                 }
 659
 660 #ifdef WITH_SSL
 661                 if (ssl) {
 662                         if ((*ssl = fetch_start_ssl(s, host)) == NULL) {
 663                                 close(s);
 664                                 s = -1;
 665                                 continue;
 666                         }
 667                 }
 668 #endif
 669                 break;
 670         }
 671         if (res0)
 672                 freeaddrinfo(res0);
 673         return s;
 674 }
 675
 676 static int
 677 handle_noproxy(const char *host, in_port_t portnum)
 678 {
 679
 680         char *cp, *ep, *np, *np_copy, *np_iter, *no_proxy;
 681         unsigned long np_port;
 682         size_t hlen, plen;
 683         int isproxy = 1;
 684
 685         /* check URL against list of no_proxied sites */
 686         no_proxy = getoptionvalue("no_proxy");
 687         if (EMPTYSTRING(no_proxy))
 688                 return isproxy;
 689
 690         np_iter = np_copy = ftp_strdup(no_proxy);
 691         hlen = strlen(host);
 692         while ((cp = strsep(&np_iter, " ,")) != NULL) {
 693                 if (*cp == '\0')
 694                         continue;
 695                 if ((np = strrchr(cp, ':')) != NULL) {
 696                         *np++ =  '\0';
 697                         np_port = strtoul(np, &ep, 10);
 698                         if (*np == '\0' || *ep != '\0')
 699                                 continue;
 700                         if (np_port != portnum)
 701                                 continue;
 702                 }
 703                 plen = strlen(cp);
 704                 if (hlen < plen)
 705                         continue;
 706                 if (strncasecmp(host + hlen - plen, cp, plen) == 0) {
 707                         isproxy = 0;
 708                         break;
 709                 }
 710         }
 711         FREEPTR(np_copy);
 712         return isproxy;
 713 }
 714
 715 static int
 716 handle_proxy(const char *url, const char *penv, struct urlinfo *ui,
 717     struct authinfo *pauth)
 718 {
 719         struct urlinfo pui;
 720
 721         if (isipv6addr(ui->host) && strchr(ui->host, '%') != NULL) {
 722                 warnx("Scoped address notation `%s' disallowed via web proxy",
 723                     ui->host);
 724                 return -1;
 725         }
 726
 727         initurlinfo(&pui);
 728         if (parse_url(penv, "proxy URL", &pui, pauth) == -1)
 729                 return -1;
 730
 731         if ((!IS_HTTP_TYPE(pui.utype) && pui.utype != FTP_URL_T) ||
 732             EMPTYSTRING(pui.host) ||
 733             (! EMPTYSTRING(pui.path) && strcmp(pui.path, "/") != 0)) {
 734                 warnx("Malformed proxy URL `%s'", penv);
 735                 freeurlinfo(&pui);
 736                 return -1;
 737         }
 738
 739         FREEPTR(pui.path);
 740         pui.path = ftp_strdup(url);
 741
 742         freeurlinfo(ui);
 743         *ui = pui;
 744
 745         return 0;
 746 }
 747
 748 static void
 749 print_host(FETCH *fin, const struct urlinfo *ui)
 750 {
 751         char *h, *p;
 752
 753         if (strchr(ui->host, ':') == NULL) {
 754                 fetch_printf(fin, "Host: %s", ui->host);
 755         } else {
 756                 /*
 757                  * strip off IPv6 scope identifier, since it is
 758                  * local to the node
 759                  */
 760                 h = ftp_strdup(ui->host);
 761                 if (isipv6addr(h) && (p = strchr(h, '%')) != NULL)
 762                         *p = '\0';
 763
 764                 fetch_printf(fin, "Host: [%s]", h);
 765                 free(h);
 766         }
 767
 768         if ((ui->utype == HTTP_URL_T && ui->portnum != HTTP_PORT) ||
 769             (ui->utype == HTTPS_URL_T && ui->portnum != HTTPS_PORT))
 770                 fetch_printf(fin, ":%u", ui->portnum);
 771         fetch_printf(fin, "\r\n");
 772 }
 773
 774 static void
 775 print_agent(FETCH *fin)
 776 {
 777         const char *useragent;
 778         if ((useragent = getenv("FTPUSERAGENT")) != NULL) {
 779                 fetch_printf(fin, "User-Agent: %s\r\n", useragent);
 780         } else {
 781                 fetch_printf(fin, "User-Agent: %s/%s\r\n",
 782                     FTP_PRODUCT, FTP_VERSION);
 783         }
 784 }
 785
 786 static void
 787 print_cache(FETCH *fin, int isproxy)
 788 {
 789         fetch_printf(fin, isproxy ?
 790             "Pragma: no-cache\r\n" :
 791             "Cache-Control: no-cache\r\n");
 792 }
 793
 794 static int
 795 print_get(FETCH *fin, int hasleading, int isproxy, const struct urlinfo *oui,
 796     const struct urlinfo *ui)
 797 {
 798         const char *leading = hasleading ? ", " : "  (";
 799
 800         if (isproxy) {
 801                 if (verbose) {
 802                         fprintf(ttyout, "%svia %s:%u", leading,
 803                             ui->host, ui->portnum);
 804                         leading = ", ";
 805                         hasleading++;
 806                 }
 807                 fetch_printf(fin, "GET %s HTTP/1.0\r\n", ui->path);
 808                 print_host(fin, oui);
 809                 return hasleading;
 810         }
 811
 812         fetch_printf(fin, "GET %s HTTP/1.1\r\n", ui->path);
 813         print_host(fin, ui);
 814         fetch_printf(fin, "Accept: */*\r\n");
 815         fetch_printf(fin, "Connection: close\r\n");
 816         if (restart_point) {
 817                 fputs(leading, ttyout);
 818                 fetch_printf(fin, "Range: bytes=" LLF "-\r\n",
 819                     (LLT)restart_point);
 820                 fprintf(ttyout, "restarting at " LLF, (LLT)restart_point);
 821                 hasleading++;
 822         }
 823         return hasleading;
 824 }
 825
 826 static void
 827 getmtime(const char *cp, time_t *mtime)
 828 {
 829         struct tm parsed;
 830         const char *t;
 831
 832         memset(&parsed, 0, sizeof(parsed));
 833         t = parse_rfc2616time(&parsed, cp);
 834
 835         if (t == NULL)
 836                 return;
 837
 838         parsed.tm_isdst = -1;
 839         if (*t == '\0')
 840                 *mtime = timegm(&parsed);
 841
 842 #ifndef NO_DEBUG
 843         if (ftp_debug && *mtime != -1) {
 844                 fprintf(ttyout, "parsed time as: %s",
 845                     rfc2822time(localtime(mtime)));
 846         }
 847 #endif
 848 }
 849
 850 static int
 851 print_proxy(FETCH *fin, int hasleading, const char *wwwauth,
 852     const char *proxyauth)
 853 {
 854         const char *leading = hasleading ? ", " : "  (";
 855
 856         if (wwwauth) {
 857                 if (verbose) {
 858                         fprintf(ttyout, "%swith authorization", leading);
 859                         hasleading++;
 860                 }
 861                 fetch_printf(fin, "Authorization: %s\r\n", wwwauth);
 862         }
 863         if (proxyauth) {
 864                 if (verbose) {
 865                         fprintf(ttyout, "%swith proxy authorization", leading);
 866                         hasleading++;
 867                 }
 868                 fetch_printf(fin, "Proxy-Authorization: %s\r\n", proxyauth);
 869         }
 870         return hasleading;
 871 }
 872
 873 #ifdef WITH_SSL
 874 static void
 875 print_connect(FETCH *fin, const struct urlinfo *ui)
 876 {
 877         char hname[NI_MAXHOST], *p;
 878         const char *h;
 879
 880         if (isipv6addr(ui->host)) {
 881                 /*
 882                  * strip off IPv6 scope identifier,
 883                  * since it is local to the node
 884                  */
 885                 if ((p = strchr(ui->host, '%')) == NULL)
 886                         snprintf(hname, sizeof(hname), "[%s]", ui->host);
 887                 else
 888                         snprintf(hname, sizeof(hname), "[%.*s]",
 889                             (int)(p - ui->host), ui->host);
 890                 h = hname;
 891         } else
 892                 h = ui->host;
 893
 894         fetch_printf(fin, "CONNECT %s:%d HTTP/1.1\r\n", h, ui->portnum);
 895         fetch_printf(fin, "Host: %s:%d\r\n", h, ui->portnum);
 896 }
 897 #endif
 898
 899 #define C_OK 0
 900 #define C_CLEANUP 1
 901 #define C_IMPROPER 2
 902
 903 static int
 904 getresponseline(FETCH *fin, char *buf, size_t buflen, int *len)
 905 {
 906         const char *errormsg;
 907
 908         alarmtimer(quit_time ? quit_time : 60);
 909         *len = fetch_getline(fin, buf, buflen, &errormsg);
 910         alarmtimer(0);
 911         if (*len < 0) {
 912                 if (*errormsg == '\n')
 913                         errormsg++;
 914                 warnx("Receiving HTTP reply: %s", errormsg);
 915                 return C_CLEANUP;
 916         }
 917         while (*len > 0 && (ISLWS(buf[*len-1])))
 918                 buf[--*len] = '\0';
 919
 920         if (*len)
 921                 DPRINTF("%s: received `%s'\n", __func__, buf);
 922         return C_OK;
 923 }
 924
 925 static int
 926 getresponse(FETCH *fin, char **cp, size_t buflen, int *hcode)
 927 {
 928         int len, rv;
 929         char *ep, *buf = *cp;
 930
 931         *hcode = 0;
 932         if ((rv = getresponseline(fin, buf, buflen, &len)) != C_OK)
 933                 return rv;
 934
 935         /* Determine HTTP response code */
 936         *cp = strchr(buf, ' ');
 937         if (*cp == NULL)
 938                 return C_IMPROPER;
 939
 940         (*cp)++;
 941
 942         *hcode = strtol(*cp, &ep, 10);
 943         if (*ep != '\0' && !isspace((unsigned char)*ep))
 944                 return C_IMPROPER;
 945
 946         return C_OK;
 947 }
 948
 949 static int
 950 parse_posinfo(const char **cp, struct posinfo *pi)
 951 {
 952         char *ep;
 953         if (!match_token(cp, "bytes"))
 954                 return -1;
 955
 956         if (**cp == '*')
 957                 (*cp)++;
 958         else {
 959                 pi->rangestart = STRTOLL(*cp, &ep, 10);
 960                 if (pi->rangestart < 0 || *ep != '-')
 961                         return -1;
 962                 *cp = ep + 1;
 963                 pi->rangeend = STRTOLL(*cp, &ep, 10);
 964                 if (pi->rangeend < 0 || pi->rangeend < pi->rangestart)
 965                         return -1;
 966                 *cp = ep;
 967         }
 968         if (**cp != '/')
 969                 return -1;
 970         (*cp)++;
 971         if (**cp == '*')
 972                 (*cp)++;
 973         else {
 974                 pi->entitylen = STRTOLL(*cp, &ep, 10);
 975                 if (pi->entitylen < 0)
 976                         return -1;
 977                 *cp = ep;
 978         }
 979         if (**cp != '\0')
 980                 return -1;
 981
 982 #ifndef NO_DEBUG
 983         if (ftp_debug) {
 984                 fprintf(ttyout, "parsed range as: ");
 985                 if (pi->rangestart == -1)
 986                         fprintf(ttyout, "*");
 987                 else
 988                         fprintf(ttyout, LLF "-" LLF, (LLT)pi->rangestart,
 989                             (LLT)pi->rangeend);
 990                 fprintf(ttyout, "/" LLF "\n", (LLT)pi->entitylen);
 991         }
 992 #endif
 993         return 0;
 994 }
 995
 996 #ifndef NO_AUTH
 997 static void
 998 do_auth(int hcode, const char *url, const char *penv, struct authinfo *wauth,
 999     struct authinfo *pauth, char **auth, const char *message,
1000     volatile int *rval)
1001 {
1002         struct authinfo aauth;
1003         char *response;
1004
1005         if (hcode == 401)
1006                 aauth = *wauth;
1007         else
1008                 aauth = *pauth;
1009
1010         if (verbose || aauth.auth == NULL ||
1011             aauth.user == NULL || aauth.pass == NULL)
1012                 fprintf(ttyout, "%s\n", message);
1013         if (EMPTYSTRING(*auth)) {
1014                 warnx("No authentication challenge provided by server");
1015                 return;
1016         }
1017
1018         if (aauth.auth != NULL) {
1019                 char reply[10];
1020
1021                 fprintf(ttyout, "Authorization failed. Retry (y/n)? ");
1022                 if (get_line(stdin, reply, sizeof(reply), NULL) < 0) {
1023                         return;
1024                 }
1025                 if (tolower((unsigned char)reply[0]) != 'y')
1026                         return;
1027
1028                 aauth.user = NULL;
1029                 aauth.pass = NULL;
1030         }
1031
1032         if (auth_url(*auth, &response, &aauth) == 0) {
1033                 *rval = fetch_url(url, penv,
1034                     hcode == 401 ? pauth->auth : response,
1035                     hcode == 401 ? response: wauth->auth);
1036                 memset(response, 0, strlen(response));
1037                 FREEPTR(response);
1038         }
1039 }
1040 #endif
1041
1042 static int
1043 negotiate_connection(FETCH *fin, const char *url, const char *penv,
1044     struct posinfo *pi, time_t *mtime, struct authinfo *wauth,
1045     struct authinfo *pauth, volatile int *rval, volatile int *ischunked,
1046     char **auth)
1047 {
1048         int                     len, hcode, rv;
1049         char                    buf[FTPBUFLEN], *ep;
1050         const char              *cp, *token;
1051         char                    *location, *message;
1052
1053         *auth = message = location = NULL;
1054
1055         /* Read the response */
1056         ep = buf;
1057         switch (getresponse(fin, &ep, sizeof(buf), &hcode)) {
1058         case C_CLEANUP:
1059                 goto cleanup_fetch_url;
1060         case C_IMPROPER:
1061                 goto improper;
1062         case C_OK:
1063                 message = ftp_strdup(ep);
1064                 break;
1065         }
1066
1067         /* Read the rest of the header. */
1068
1069         for (;;) {
1070                 if ((rv = getresponseline(fin, buf, sizeof(buf), &len)) != C_OK)
1071                         goto cleanup_fetch_url;
1072                 if (len == 0)
1073                         break;
1074
1075         /*
1076          * Look for some headers
1077          */
1078
1079                 cp = buf;
1080
1081                 if (match_token(&cp, "Content-Length:")) {
1082                         filesize = STRTOLL(cp, &ep, 10);
1083                         if (filesize < 0 || *ep != '\0')
1084                                 goto improper;
1085                         DPRINTF("%s: parsed len as: " LLF "\n",
1086                             __func__, (LLT)filesize);
1087
1088                 } else if (match_token(&cp, "Content-Range:")) {
1089                         if (parse_posinfo(&cp, pi) == -1)
1090                                 goto improper;
1091                         if (! restart_point) {
1092                                 warnx(
1093                             "Received unexpected Content-Range header");
1094                                 goto cleanup_fetch_url;
1095                         }
1096
1097                 } else if (match_token(&cp, "Last-Modified:")) {
1098                         getmtime(cp, mtime);
1099
1100                 } else if (match_token(&cp, "Location:")) {
1101                         location = ftp_strdup(cp);
1102                         DPRINTF("%s: parsed location as `%s'\n",
1103                             __func__, cp);
1104
1105                 } else if (match_token(&cp, "Transfer-Encoding:")) {
1106                         if (match_token(&cp, "binary")) {
1107                                 warnx(
1108                 "Bogus transfer encoding `binary' (fetching anyway)");
1109                                 continue;
1110                         }
1111                         if (! (token = match_token(&cp, "chunked"))) {
1112                                 warnx(
1113                             "Unsupported transfer encoding `%s'",
1114                                     token);
1115                                 goto cleanup_fetch_url;
1116                         }
1117                         (*ischunked)++;
1118                         DPRINTF("%s: using chunked encoding\n",
1119                             __func__);
1120
1121                 } else if (match_token(&cp, "Proxy-Authenticate:")
1122                         || match_token(&cp, "WWW-Authenticate:")) {
1123                         if (! (token = match_token(&cp, "Basic"))) {
1124                                 DPRINTF("%s: skipping unknown auth "
1125                                     "scheme `%s'\n", __func__, token);
1126                                 continue;
1127                         }
1128                         FREEPTR(*auth);
1129                         *auth = ftp_strdup(token);
1130                         DPRINTF("%s: parsed auth as `%s'\n",
1131                             __func__, cp);
1132                 }
1133
1134         }
1135                         /* finished parsing header */
1136
1137         switch (hcode) {
1138         case 200:
1139                 break;
1140         case 206:
1141                 if (! restart_point) {
1142                         warnx("Not expecting partial content header");
1143                         goto cleanup_fetch_url;
1144                 }
1145                 break;
1146         case 300:
1147         case 301:
1148         case 302:
1149         case 303:
1150         case 305:
1151         case 307:
1152                 if (EMPTYSTRING(location)) {
1153                         warnx(
1154                         "No redirection Location provided by server");
1155                         goto cleanup_fetch_url;
1156                 }
1157                 if (redirect_loop++ > 5) {
1158                         warnx("Too many redirections requested");
1159                         goto cleanup_fetch_url;
1160                 }
1161                 if (hcode == 305) {
1162                         if (verbose)
1163                                 fprintf(ttyout, "Redirected via %s\n",
1164                                     location);
1165                         *rval = fetch_url(url, location,
1166                             pauth->auth, wauth->auth);
1167                 } else {
1168                         if (verbose)
1169                                 fprintf(ttyout, "Redirected to %s\n",
1170                                     location);
1171                         *rval = go_fetch(location);
1172                 }
1173                 goto cleanup_fetch_url;
1174 #ifndef NO_AUTH
1175         case 401:
1176         case 407:
1177                 do_auth(hcode, url, penv, wauth, pauth, auth, message, rval);
1178                 goto cleanup_fetch_url;
1179 #endif
1180         default:
1181                 if (message)
1182                         warnx("Error retrieving file `%s'", message);
1183                 else
1184                         warnx("Unknown error retrieving file");
1185                 goto cleanup_fetch_url;
1186         }
1187         rv = C_OK;
1188         goto out;
1189
1190 cleanup_fetch_url:
1191         rv = C_CLEANUP;
1192         goto out;
1193 improper:
1194         rv = C_IMPROPER;
1195         goto out;
1196 out:
1197         FREEPTR(message);
1198         FREEPTR(location);
1199         return rv;
1200 }               /* end of ftp:// or http:// specific setup */
1201
1202 #ifdef WITH_SSL
1203 static int
1204 connectmethod(FETCH *fin, const char *url, const char *penv,
1205     struct urlinfo *oui, struct urlinfo *ui, struct authinfo *wauth,
1206     struct authinfo *pauth, char **auth, int *hasleading, volatile int *rval)
1207 {
1208         void *ssl;
1209         int hcode, rv;
1210         const char *cp;
1211         char buf[FTPBUFLEN], *ep;
1212         char *message = NULL;
1213
1214         print_connect(fin, oui);
1215
1216         print_agent(fin);
1217         *hasleading = print_proxy(fin, *hasleading, NULL, pauth->auth);
1218
1219         if (verbose && *hasleading)
1220                 fputs(")\n", ttyout);
1221         *hasleading = 0;
1222
1223         fetch_printf(fin, "\r\n");
1224         if (fetch_flush(fin) == EOF) {
1225                 warn("Writing HTTP request");
1226                 alarmtimer(0);
1227                 goto cleanup_fetch_url;
1228         }
1229         alarmtimer(0);
1230
1231         /* Read the response */
1232         ep = buf;
1233         switch (getresponse(fin, &ep, sizeof(buf), &hcode)) {
1234         case C_CLEANUP:
1235                 goto cleanup_fetch_url;
1236         case C_IMPROPER:
1237                 goto improper;
1238         case C_OK:
1239                 message = ftp_strdup(ep);
1240                 break;
1241         }
1242
1243         for (;;) {
1244                 int len;
1245                 if (getresponseline(fin, buf, sizeof(buf), &len) != C_OK)
1246                         goto cleanup_fetch_url;
1247                 if (len == 0)
1248                         break;
1249
1250                 cp = buf;
1251                 if (match_token(&cp, "Proxy-Authenticate:")) {
1252                         const char *token;
1253                         if (!(token = match_token(&cp, "Basic"))) {
1254                                 DPRINTF(
1255                                     "%s: skipping unknown auth scheme `%s'\n",
1256                                     __func__, token);
1257                                 continue;
1258                         }
1259                         FREEPTR(*auth);
1260                         *auth = ftp_strdup(token);
1261                         DPRINTF("%s: parsed auth as " "`%s'\n", __func__, cp);
1262                 }
1263         }
1264
1265         /* finished parsing header */
1266         switch (hcode) {
1267         case 200:
1268                 break;
1269 #ifndef NO_AUTH
1270         case 407:
1271                 do_auth(hcode, url, penv, wauth, pauth, auth, message, rval);
1272                 goto cleanup_fetch_url;
1273 #endif
1274         default:
1275                 if (message)
1276                         warnx("Error proxy connect " "`%s'", message);
1277                 else
1278                         warnx("Unknown error proxy " "connect");
1279                 goto cleanup_fetch_url;
1280         }
1281
1282         if ((ssl = fetch_start_ssl(fetch_fileno(fin), oui->host)) == NULL)
1283                 goto cleanup_fetch_url;
1284         fetch_set_ssl(fin, ssl);
1285
1286         rv = C_OK;
1287         goto out;
1288 improper:
1289         rv = C_IMPROPER;
1290         goto out;
1291 cleanup_fetch_url:
1292         rv = C_CLEANUP;
1293         goto out;
1294 out:
1295         FREEPTR(message);
1296         return rv;
1297 }
1298 #endif
1299
1300 /*
1301  * Retrieve URL, via a proxy if necessary, using HTTP.
1302  * If proxyenv is set, use that for the proxy, otherwise try ftp_proxy or
1303  * http_proxy/https_proxy as appropriate.
1304  * Supports HTTP redirects.
1305  * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection
1306  * is still open (e.g, ftp xfer with trailing /)
1307  */
1308 static int
1309 fetch_url(const char *url, const char *proxyenv, char *proxyauth, char *wwwauth)
1310 {
1311         sigfunc volatile        oldint;
1312         sigfunc volatile        oldpipe;
1313         sigfunc volatile        oldalrm;
1314         sigfunc volatile        oldquit;
1315         int volatile            s;
1316         struct stat             sb;
1317         int volatile            isproxy;
1318         int volatile            rval, ischunked;
1319         size_t                  flen;
1320         static size_t           bufsize;
1321         static char             *xferbuf;
1322         const char              *cp;
1323         char                    *ep;
1324         char                    *volatile auth;
1325         char                    *volatile savefile;
1326         char                    *volatile location;
1327         char                    *volatile message;
1328         char                    *volatile decodedpath;
1329         struct authinfo         wauth, pauth;
1330         struct posinfo          pi;
1331         off_t                   hashbytes;
1332         int                     (*volatile closefunc)(FILE *);
1333         FETCH                   *volatile fin;
1334         FILE                    *volatile fout;
1335         const char              *volatile penv = proxyenv;
1336         struct urlinfo          ui, oui;
1337         time_t                  mtime;
1338         void                    *ssl = NULL;
1339
1340         DPRINTF("%s: `%s' proxyenv `%s'\n", __func__, url, STRorNULL(penv));
1341
1342         oldquit = oldalrm = oldint = oldpipe = SIG_ERR;
1343         closefunc = NULL;
1344         fin = NULL;
1345         fout = NULL;
1346         s = -1;
1347         savefile = NULL;
1348         auth = location = message = NULL;
1349         ischunked = isproxy = 0;
1350         rval = 1;
1351
1352         initurlinfo(&ui);
1353         initurlinfo(&oui);
1354         initauthinfo(&wauth, wwwauth);
1355         initauthinfo(&pauth, proxyauth);
1356
1357         decodedpath = NULL;
1358
1359         if (sigsetjmp(httpabort, 1))
1360                 goto cleanup_fetch_url;
1361
1362         if (parse_url(url, "URL", &ui, &wauth) == -1)
1363                 goto cleanup_fetch_url;
1364
1365         copyurlinfo(&oui, &ui);
1366
1367         if (ui.utype == FILE_URL_T && ! EMPTYSTRING(ui.host)
1368             && strcasecmp(ui.host, "localhost") != 0) {
1369                 warnx("No support for non local file URL `%s'", url);
1370                 goto cleanup_fetch_url;
1371         }
1372
1373         if (EMPTYSTRING(ui.path)) {
1374                 if (ui.utype == FTP_URL_T) {
1375                         rval = fetch_ftp(url);
1376                         goto cleanup_fetch_url;
1377                 }
1378                 if (!IS_HTTP_TYPE(ui.utype) || outfile == NULL)  {
1379                         warnx("Invalid URL (no file after host) `%s'", url);
1380                         goto cleanup_fetch_url;
1381                 }
1382         }
1383
1384         decodedpath = ftp_strdup(ui.path);
1385         url_decode(decodedpath);
1386
1387         if (outfile)
1388                 savefile = outfile;
1389         else {
1390                 cp = strrchr(decodedpath, '/');         /* find savefile */
1391                 if (cp != NULL)
1392                         savefile = ftp_strdup(cp + 1);
1393                 else
1394                         savefile = ftp_strdup(decodedpath);
1395                 /*
1396                  * Use the first URL we requested not the name after a
1397                  * possible redirect, but careful to save it because our
1398                  * "safety" check is the match to outfile.
1399                  */
1400                 outfile = ftp_strdup(savefile);
1401         }
1402         DPRINTF("%s: savefile `%s'\n", __func__, savefile);
1403         if (EMPTYSTRING(savefile)) {
1404                 if (ui.utype == FTP_URL_T) {
1405                         rval = fetch_ftp(url);
1406                         goto cleanup_fetch_url;
1407                 }
1408                 warnx("No file after directory (you must specify an "
1409                     "output file) `%s'", url);
1410                 goto cleanup_fetch_url;
1411         }
1412
1413         restart_point = 0;
1414         filesize = -1;
1415         initposinfo(&pi);
1416         mtime = -1;
1417         if (restartautofetch) {
1418                 if (stat(savefile, &sb) == 0)
1419                         restart_point = sb.st_size;
1420         }
1421         if (ui.utype == FILE_URL_T) {           /* file:// URLs */
1422                 direction = "copied";
1423                 fin = fetch_open(decodedpath, "r");
1424                 if (fin == NULL) {
1425                         warn("Can't open `%s'", decodedpath);
1426                         goto cleanup_fetch_url;
1427                 }
1428                 if (fstat(fetch_fileno(fin), &sb) == 0) {
1429                         mtime = sb.st_mtime;
1430                         filesize = sb.st_size;
1431                 }
1432                 if (restart_point) {
1433                         if (lseek(fetch_fileno(fin), restart_point, SEEK_SET) < 0) {
1434                                 warn("Can't seek to restart `%s'",
1435                                     decodedpath);
1436                                 goto cleanup_fetch_url;
1437                         }
1438                 }
1439                 if (verbose) {
1440                         fprintf(ttyout, "Copying %s", decodedpath);
1441                         if (restart_point)
1442                                 fprintf(ttyout, " (restarting at " LLF ")",
1443                                     (LLT)restart_point);
1444                         fputs("\n", ttyout);
1445                 }
1446                 if (0 == rcvbuf_size) {
1447                         rcvbuf_size = 8 * 1024; /* XXX */
1448                 }
1449         } else {                                /* ftp:// or http:// URLs */
1450                 int hasleading;
1451
1452                 if (penv == NULL) {
1453 #ifdef WITH_SSL
1454                         if (ui.utype == HTTPS_URL_T)
1455                                 penv = getoptionvalue("https_proxy");
1456 #endif
1457                         if (penv == NULL && IS_HTTP_TYPE(ui.utype))
1458                                 penv = getoptionvalue("http_proxy");
1459                         else if (ui.utype == FTP_URL_T)
1460                                 penv = getoptionvalue("ftp_proxy");
1461                 }
1462                 direction = "retrieved";
1463                 if (! EMPTYSTRING(penv)) {                      /* use proxy */
1464
1465                         isproxy = handle_noproxy(ui.host, ui.portnum);
1466
1467                         if (isproxy == 0 && ui.utype == FTP_URL_T) {
1468                                 rval = fetch_ftp(url);
1469                                 goto cleanup_fetch_url;
1470                         }
1471
1472                         if (isproxy) {
1473                                 if (restart_point) {
1474                                         warnx(
1475                                             "Can't restart via proxy URL `%s'",
1476                                             penv);
1477                                         goto cleanup_fetch_url;
1478                                 }
1479                                 if (handle_proxy(url, penv, &ui, &pauth) < 0)
1480                                         goto cleanup_fetch_url;
1481                         }
1482                 } /* ! EMPTYSTRING(penv) */
1483
1484                 s = ftp_socket(&ui, &ssl);
1485                 if (s < 0) {
1486                         warnx("Can't connect to `%s:%s'", ui.host, ui.port);
1487                         goto cleanup_fetch_url;
1488                 }
1489
1490                 oldalrm = xsignal(SIGALRM, timeouthttp);
1491                 alarmtimer(quit_time ? quit_time : 60);
1492                 fin = fetch_fdopen(s, "r+");
1493                 fetch_set_ssl(fin, ssl);
1494                 alarmtimer(0);
1495
1496                 alarmtimer(quit_time ? quit_time : 60);
1497                 /*
1498                  * Construct and send the request.
1499                  */
1500                 if (verbose)
1501                         fprintf(ttyout, "Requesting %s\n", url);
1502
1503                 hasleading = 0;
1504 #ifdef WITH_SSL
1505                 if (isproxy && oui.utype == HTTPS_URL_T) {
1506                         switch (connectmethod(fin, url, penv, &oui, &ui,
1507                             &wauth, &pauth, __UNVOLATILE(&auth), &hasleading,
1508                             &rval)) {
1509                         case C_CLEANUP:
1510                                 goto cleanup_fetch_url;
1511                         case C_IMPROPER:
1512                                 goto improper;
1513                         case C_OK:
1514                                 break;
1515                         default:
1516                                 abort();
1517                         }
1518                 }
1519 #endif
1520
1521                 hasleading = print_get(fin, hasleading, isproxy, &oui, &ui);
1522
1523                 if (flushcache)
1524                         print_cache(fin, isproxy);
1525
1526                 print_agent(fin);
1527                 hasleading = print_proxy(fin, hasleading, wauth.auth,
1528                      auth ? NULL : pauth.auth);
1529                 if (hasleading) {
1530                         hasleading = 0;
1531                         if (verbose)
1532                                 fputs(")\n", ttyout);
1533                 }
1534
1535                 fetch_printf(fin, "\r\n");
1536                 if (fetch_flush(fin) == EOF) {
1537                         warn("Writing HTTP request");
1538                         alarmtimer(0);
1539                         goto cleanup_fetch_url;
1540                 }
1541                 alarmtimer(0);
1542
1543                 switch (negotiate_connection(fin, url, penv, &pi,
1544                     &mtime, &wauth, &pauth, &rval, &ischunked,
1545                     __UNVOLATILE(&auth))) {
1546                 case C_OK:
1547                         break;
1548                 case C_CLEANUP:
1549                         goto cleanup_fetch_url;
1550                 case C_IMPROPER:
1551                         goto improper;
1552                 default:
1553                         abort();
1554                 }
1555         }
1556
1557         /* Open the output file. */
1558
1559         /*
1560          * Only trust filenames with special meaning if they came from
1561          * the command line
1562          */
1563         if (outfile == savefile) {
1564                 if (strcmp(savefile, "-") == 0) {
1565                         fout = stdout;
1566                 } else if (*savefile == '|') {
1567 #if 0
1568                         oldpipe = xsignal(SIGPIPE, SIG_IGN);
1569                         fout = popen(savefile + 1, "w");
1570                         if (fout == NULL) {
1571                                 warn("Can't execute `%s'", savefile + 1);
1572                                 goto cleanup_fetch_url;
1573                         }
1574                         closefunc = pclose;
1575 #endif
1576                 }
1577         }
1578         if (fout == NULL) {
1579                 if ((pi.rangeend != -1 && pi.rangeend <= restart_point) ||
1580                     (pi.rangestart == -1 &&
1581                     filesize != -1 && filesize <= restart_point)) {
1582                         /* already done */
1583                         if (verbose)
1584                                 fprintf(ttyout, "already done\n");
1585                         rval = 0;
1586                         goto cleanup_fetch_url;
1587                 }
1588                 if (restart_point && pi.rangestart != -1) {
1589                         if (pi.entitylen != -1)
1590                                 filesize = pi.entitylen;
1591                         if (pi.rangestart != restart_point) {
1592                                 warnx(
1593                                     "Size of `%s' differs from save file `%s'",
1594                                     url, savefile);
1595                                 goto cleanup_fetch_url;
1596                         }
1597                         fout = fopen(savefile, "a");
1598                 } else
1599                         fout = fopen(savefile, "w");
1600                 if (fout == NULL) {
1601                         warn("Can't open `%s'", savefile);
1602                         goto cleanup_fetch_url;
1603                 }
1604                 closefunc = fclose;
1605         }
1606
1607                         /* Trap signals */
1608         oldquit = xsignal(SIGQUIT, psummary);
1609         oldint = xsignal(SIGINT, aborthttp);
1610
1611         assert(rcvbuf_size > 0);
1612         if ((size_t)rcvbuf_size > bufsize) {
1613                 if (xferbuf)
1614                         (void)free(xferbuf);
1615                 bufsize = rcvbuf_size;
1616                 xferbuf = ftp_malloc(bufsize);
1617         }
1618
1619         bytes = 0;
1620         hashbytes = mark;
1621         if (oldalrm != SIG_ERR) {
1622                 (void)xsignal(SIGALRM, oldalrm);
1623                 oldalrm = SIG_ERR;
1624         }
1625         progressmeter(-1);
1626
1627                         /* Finally, suck down the file. */
1628         do {
1629                 long chunksize;
1630                 short lastchunk;
1631
1632                 chunksize = 0;
1633                 lastchunk = 0;
1634                                         /* read chunk-size */
1635                 if (ischunked) {
1636                         if (fetch_getln(xferbuf, bufsize, fin) == NULL) {
1637                                 warnx("Unexpected EOF reading chunk-size");
1638                                 goto cleanup_fetch_url;
1639                         }
1640                         errno = 0;
1641                         chunksize = strtol(xferbuf, &ep, 16);
1642                         if (ep == xferbuf) {
1643                                 warnx("Invalid chunk-size");
1644                                 goto cleanup_fetch_url;
1645                         }
1646                         if (errno == ERANGE || chunksize < 0) {
1647                                 errno = ERANGE;
1648                                 warn("Chunk-size `%.*s'",
1649                                     (int)(ep-xferbuf), xferbuf);
1650                                 goto cleanup_fetch_url;
1651                         }
1652
1653                                 /*
1654                                  * XXX: Work around bug in Apache 1.3.9 and
1655                                  *      1.3.11, which incorrectly put trailing
1656                                  *      space after the chunk-size.
1657                                  */
1658                         while (*ep == ' ')
1659                                 ep++;
1660
1661                                         /* skip [ chunk-ext ] */
1662                         if (*ep == ';') {
1663                                 while (*ep && *ep != '\r')
1664                                         ep++;
1665                         }
1666
1667                         if (strcmp(ep, "\r\n") != 0) {
1668                                 warnx("Unexpected data following chunk-size");
1669                                 goto cleanup_fetch_url;
1670                         }
1671                         DPRINTF("%s: got chunk-size of " LLF "\n", __func__,
1672                             (LLT)chunksize);
1673                         if (chunksize == 0) {
1674                                 lastchunk = 1;
1675                                 goto chunkdone;
1676                         }
1677                 }
1678                                         /* transfer file or chunk */
1679                 while (1) {
1680                         struct timeval then, now, td;
1681                         volatile off_t bufrem;
1682
1683                         if (rate_get)
1684                                 (void)gettimeofday(&then, NULL);
1685                         bufrem = rate_get ? rate_get : (off_t)bufsize;
1686                         if (ischunked)
1687                                 bufrem = MIN(chunksize, bufrem);
1688                         while (bufrem > 0) {
1689                                 size_t nr = MIN((off_t)bufsize, bufrem);
1690                                 flen = fetch_read(xferbuf, sizeof(char),
1691                                     nr, fin);
1692                                 if (flen == 0) {
1693                                         if (fetch_error(fin))
1694                                                 goto chunkerror;
1695                                         goto chunkdone;
1696                                 }
1697                                 bytes += flen;
1698                                 bufrem -= flen;
1699                                 if (maxwrite(xferbuf, sizeof(char), flen, fout)
1700                                     != flen) {
1701                                         warn("Writing `%s'", savefile);
1702                                         goto cleanup_fetch_url;
1703                                 }
1704                                 if (hash && !progress) {
1705                                         while (bytes >= hashbytes) {
1706                                                 (void)putc('#', ttyout);
1707                                                 hashbytes += mark;
1708                                         }
1709                                         (void)fflush(ttyout);
1710                                 }
1711                                 if (ischunked) {
1712                                         chunksize -= flen;
1713                                         if (chunksize <= 0)
1714                                                 break;
1715                                 }
1716                         }
1717                         if (rate_get) {
1718                                 while (1) {
1719                                         (void)gettimeofday(&now, NULL);
1720                                         timersub(&now, &then, &td);
1721                                         if (td.tv_sec > 0)
1722                                                 break;
1723                                         usleep(1000000 - td.tv_usec);
1724                                 }
1725                         }
1726                         if (ischunked && chunksize <= 0)
1727                                 break;
1728                 }
1729                                         /* read CRLF after chunk*/
1730  chunkdone:
1731                 if (ischunked) {
1732                         if (fetch_getln(xferbuf, bufsize, fin) == NULL) {
1733                                 alarmtimer(0);
1734                                 warnx("Unexpected EOF reading chunk CRLF");
1735                                 goto cleanup_fetch_url;
1736                         }
1737                         if (strcmp(xferbuf, "\r\n") != 0) {
1738                                 warnx("Unexpected data following chunk");
1739                                 goto cleanup_fetch_url;
1740                         }
1741                         if (lastchunk)
1742                                 break;
1743                 }
1744         } while (ischunked);
1745
1746 /* XXX: deal with optional trailer & CRLF here? */
1747 chunkerror:
1748         if (hash && !progress && bytes > 0) {
1749                 if (bytes < mark)
1750                         (void)putc('#', ttyout);
1751                 (void)putc('\n', ttyout);
1752         }
1753         if (fetch_error(fin)) {
1754                 warn("Reading file");
1755                 goto cleanup_fetch_url;
1756         }
1757         progressmeter(1);
1758         (void)fflush(fout);
1759         if (closefunc == fclose && mtime != -1) {
1760                 struct timeval tval[2];
1761
1762                 (void)gettimeofday(&tval[0], NULL);
1763                 tval[1].tv_sec = mtime;
1764                 tval[1].tv_usec = 0;
1765                 (*closefunc)(fout);
1766                 fout = NULL;
1767
1768                 if (utimes(savefile, tval) == -1) {
1769                         fprintf(ttyout,
1770                             "Can't change modification time to %s",
1771                             rfc2822time(localtime(&mtime)));
1772                 }
1773         }
1774         if (bytes > 0)
1775                 ptransfer(0);
1776         bytes = 0;
1777
1778         rval = 0;
1779         goto cleanup_fetch_url;
1780
1781  improper:
1782         warnx("Improper response from `%s:%s'", ui.host, ui.port);
1783
1784  cleanup_fetch_url:
1785         if (oldint != SIG_ERR)
1786                 (void)xsignal(SIGINT, oldint);
1787         if (oldpipe != SIG_ERR)
1788                 (void)xsignal(SIGPIPE, oldpipe);
1789         if (oldalrm != SIG_ERR)
1790                 (void)xsignal(SIGALRM, oldalrm);
1791         if (oldquit != SIG_ERR)
1792                 (void)xsignal(SIGQUIT, oldquit);
1793         if (fin != NULL)
1794                 fetch_close(fin);
1795         else if (s != -1)
1796                 close(s);
1797         if (closefunc != NULL && fout != NULL)
1798                 (*closefunc)(fout);
1799         if (savefile != outfile)
1800                 FREEPTR(savefile);
1801         freeurlinfo(&ui);
1802         freeurlinfo(&oui);
1803         freeauthinfo(&wauth);
1804         freeauthinfo(&pauth);
1805         FREEPTR(decodedpath);
1806         FREEPTR(auth);
1807         FREEPTR(location);
1808         FREEPTR(message);
1809         return (rval);
1810 }
1811
1812 /*
1813  * Abort a HTTP retrieval
1814  */
1815 static void
1816 aborthttp(int notused)
1817 {
1818         char msgbuf[100];
1819         int len;
1820
1821         sigint_raised = 1;
1822         alarmtimer(0);
1823         if (fromatty) {
1824                 len = snprintf(msgbuf, sizeof(msgbuf),
1825                     "\n%s: HTTP fetch aborted.\n", getprogname());
1826                 if (len > 0)
1827                         write(fileno(ttyout), msgbuf, len);
1828         }
1829         siglongjmp(httpabort, 1);
1830 }
1831
1832 static void
1833 timeouthttp(int notused)
1834 {
1835         char msgbuf[100];
1836         int len;
1837
1838         alarmtimer(0);
1839         if (fromatty) {
1840                 len = snprintf(msgbuf, sizeof(msgbuf),
1841                     "\n%s: HTTP fetch timeout.\n", getprogname());
1842                 if (len > 0)
1843                         write(fileno(ttyout), msgbuf, len);
1844         }
1845         siglongjmp(httpabort, 1);
1846 }
1847
1848 /*
1849  * Retrieve ftp URL or classic ftp argument using FTP.
1850  * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection
1851  * is still open (e.g, ftp xfer with trailing /)
1852  */
1853 static int
1854 fetch_ftp(const char *url)
1855 {
1856         char            *cp, *xargv[5], rempath[MAXPATHLEN];
1857         char            *dir, *file;
1858         char             cmdbuf[MAXPATHLEN];
1859         char             dirbuf[4];
1860         int              dirhasglob, filehasglob, rval, transtype, xargc;
1861         int              oanonftp, oautologin;
1862         struct authinfo  auth;
1863         struct urlinfo   ui;
1864
1865         DPRINTF("fetch_ftp: `%s'\n", url);
1866         dir = file = NULL;
1867         rval = 1;
1868         transtype = TYPE_I;
1869
1870         initurlinfo(&ui);
1871         initauthinfo(&auth, NULL);
1872
1873         if (STRNEQUAL(url, FTP_URL)) {
1874                 if ((parse_url(url, "URL", &ui, &auth) == -1) ||
1875                     (auth.user != NULL && *auth.user == '\0') ||
1876                     EMPTYSTRING(ui.host)) {
1877                         warnx("Invalid URL `%s'", url);
1878                         goto cleanup_fetch_ftp;
1879                 }
1880                 /*
1881                  * Note: Don't url_decode(path) here.  We need to keep the
1882                  * distinction between "/" and "%2F" until later.
1883                  */
1884
1885                                         /* check for trailing ';type=[aid]' */
1886                 if (! EMPTYSTRING(ui.path) && (cp = strrchr(ui.path, ';')) != NULL) {
1887                         if (strcasecmp(cp, ";type=a") == 0)
1888                                 transtype = TYPE_A;
1889                         else if (strcasecmp(cp, ";type=i") == 0)
1890                                 transtype = TYPE_I;
1891                         else if (strcasecmp(cp, ";type=d") == 0) {
1892                                 warnx(
1893                             "Directory listing via a URL is not supported");
1894                                 goto cleanup_fetch_ftp;
1895                         } else {
1896                                 warnx("Invalid suffix `%s' in URL `%s'", cp,
1897                                     url);
1898                                 goto cleanup_fetch_ftp;
1899                         }
1900                         *cp = 0;
1901                 }
1902         } else {                        /* classic style `[user@]host:[file]' */
1903                 ui.utype = CLASSIC_URL_T;
1904                 ui.host = ftp_strdup(url);
1905                 cp = strchr(ui.host, '@');
1906                 if (cp != NULL) {
1907                         *cp = '\0';
1908                         auth.user = ui.host;
1909                         anonftp = 0;    /* disable anonftp */
1910                         ui.host = ftp_strdup(cp + 1);
1911                 }
1912                 cp = strchr(ui.host, ':');
1913                 if (cp != NULL) {
1914                         *cp = '\0';
1915                         ui.path = ftp_strdup(cp + 1);
1916                 }
1917         }
1918         if (EMPTYSTRING(ui.host))
1919                 goto cleanup_fetch_ftp;
1920
1921                         /* Extract the file and (if present) directory name. */
1922         dir = ui.path;
1923         if (! EMPTYSTRING(dir)) {
1924                 /*
1925                  * If we are dealing with classic `[user@]host:[path]' syntax,
1926                  * then a path of the form `/file' (resulting from input of the
1927                  * form `host:/file') means that we should do "CWD /" before
1928                  * retrieving the file.  So we set dir="/" and file="file".
1929                  *
1930                  * But if we are dealing with URLs like `ftp://host/path' then
1931                  * a path of the form `/file' (resulting from a URL of the form
1932                  * `ftp://host//file') means that we should do `CWD ' (with an
1933                  * empty argument) before retrieving the file.  So we set
1934                  * dir="" and file="file".
1935                  *
1936                  * If the path does not contain / at all, we set dir=NULL.
1937                  * (We get a path without any slashes if we are dealing with
1938                  * classic `[user@]host:[file]' or URL `ftp://host/file'.)
1939                  *
1940                  * In all other cases, we set dir to a string that does not
1941                  * include the final '/' that separates the dir part from the
1942                  * file part of the path.  (This will be the empty string if
1943                  * and only if we are dealing with a path of the form `/file'
1944                  * resulting from an URL of the form `ftp://host//file'.)
1945                  */
1946                 cp = strrchr(dir, '/');
1947                 if (cp == dir && ui.utype == CLASSIC_URL_T) {
1948                         file = cp + 1;
1949                         (void)strlcpy(dirbuf, "/", sizeof(dirbuf));
1950                         dir = dirbuf;
1951                 } else if (cp != NULL) {
1952                         *cp++ = '\0';
1953                         file = cp;
1954                 } else {
1955                         file = dir;
1956                         dir = NULL;
1957                 }
1958         } else
1959                 dir = NULL;
1960         if (ui.utype == FTP_URL_T && file != NULL) {
1961                 url_decode(file);
1962                 /* but still don't url_decode(dir) */
1963         }
1964         DPRINTF("fetch_ftp: user `%s' pass `%s' host %s port %s "
1965             "path `%s' dir `%s' file `%s'\n",
1966             STRorNULL(auth.user), STRorNULL(auth.pass),
1967             STRorNULL(ui.host), STRorNULL(ui.port),
1968             STRorNULL(ui.path), STRorNULL(dir), STRorNULL(file));
1969
1970         dirhasglob = filehasglob = 0;
1971         if (doglob &&
1972             (ui.utype == CLASSIC_URL_T || ui.utype == FTP_URL_T)) {
1973                 if (! EMPTYSTRING(dir) && strpbrk(dir, "*?[]{}") != NULL)
1974                         dirhasglob = 1;
1975                 if (! EMPTYSTRING(file) && strpbrk(file, "*?[]{}") != NULL)
1976                         filehasglob = 1;
1977         }
1978
1979                         /* Set up the connection */
1980         oanonftp = anonftp;
1981         if (connected)
1982                 disconnect(0, NULL);
1983         anonftp = oanonftp;
1984         (void)strlcpy(cmdbuf, getprogname(), sizeof(cmdbuf));
1985         xargv[0] = cmdbuf;
1986         xargv[1] = ui.host;
1987         xargv[2] = NULL;
1988         xargc = 2;
1989         if (ui.port) {
1990                 xargv[2] = ui.port;
1991                 xargv[3] = NULL;
1992                 xargc = 3;
1993         }
1994         oautologin = autologin;
1995                 /* don't autologin in setpeer(), use ftp_login() below */
1996         autologin = 0;
1997         setpeer(xargc, xargv);
1998         autologin = oautologin;
1999         if ((connected == 0) ||
2000             (connected == 1 && !ftp_login(ui.host, auth.user, auth.pass))) {
2001                 warnx("Can't connect or login to host `%s:%s'",
2002                         ui.host, ui.port ? ui.port : "?");
2003                 goto cleanup_fetch_ftp;
2004         }
2005
2006         switch (transtype) {
2007         case TYPE_A:
2008                 setascii(1, xargv);
2009                 break;
2010         case TYPE_I:
2011                 setbinary(1, xargv);
2012                 break;
2013         default:
2014                 errx(1, "fetch_ftp: unknown transfer type %d", transtype);
2015         }
2016
2017                 /*
2018                  * Change directories, if necessary.
2019                  *
2020                  * Note: don't use EMPTYSTRING(dir) below, because
2021                  * dir=="" means something different from dir==NULL.
2022                  */
2023         if (dir != NULL && !dirhasglob) {
2024                 char *nextpart;
2025
2026                 /*
2027                  * If we are dealing with a classic `[user@]host:[path]'
2028                  * (urltype is CLASSIC_URL_T) then we have a raw directory
2029                  * name (not encoded in any way) and we can change
2030                  * directories in one step.
2031                  *
2032                  * If we are dealing with an `ftp://host/path' URL
2033                  * (urltype is FTP_URL_T), then RFC 3986 says we need to
2034                  * send a separate CWD command for each unescaped "/"
2035                  * in the path, and we have to interpret %hex escaping
2036                  * *after* we find the slashes.  It's possible to get
2037                  * empty components here, (from multiple adjacent
2038                  * slashes in the path) and RFC 3986 says that we should
2039                  * still do `CWD ' (with a null argument) in such cases.
2040                  *
2041                  * Many ftp servers don't support `CWD ', so if there's an
2042                  * error performing that command, bail out with a descriptive
2043                  * message.
2044                  *
2045                  * Examples:
2046                  *
2047                  * host:                        dir="", urltype=CLASSIC_URL_T
2048                  *              logged in (to default directory)
2049                  * host:file                    dir=NULL, urltype=CLASSIC_URL_T
2050                  *              "RETR file"
2051                  * host:dir/                    dir="dir", urltype=CLASSIC_URL_T
2052                  *              "CWD dir", logged in
2053                  * ftp://host/                  dir="", urltype=FTP_URL_T
2054                  *              logged in (to default directory)
2055                  * ftp://host/dir/              dir="dir", urltype=FTP_URL_T
2056                  *              "CWD dir", logged in
2057                  * ftp://host/file              dir=NULL, urltype=FTP_URL_T
2058                  *              "RETR file"
2059                  * ftp://host//file             dir="", urltype=FTP_URL_T
2060                  *              "CWD ", "RETR file"
2061                  * host:/file                   dir="/", urltype=CLASSIC_URL_T
2062                  *              "CWD /", "RETR file"
2063                  * ftp://host///file            dir="/", urltype=FTP_URL_T
2064                  *              "CWD ", "CWD ", "RETR file"
2065                  * ftp://host/%2F/file          dir="%2F", urltype=FTP_URL_T
2066                  *              "CWD /", "RETR file"
2067                  * ftp://host/foo/file          dir="foo", urltype=FTP_URL_T
2068                  *              "CWD foo", "RETR file"
2069                  * ftp://host/foo/bar/file      dir="foo/bar"
2070                  *              "CWD foo", "CWD bar", "RETR file"
2071                  * ftp://host//foo/bar/file     dir="/foo/bar"
2072                  *              "CWD ", "CWD foo", "CWD bar", "RETR file"
2073                  * ftp://host/foo//bar/file     dir="foo//bar"
2074                  *              "CWD foo", "CWD ", "CWD bar", "RETR file"
2075                  * ftp://host/%2F/foo/bar/file  dir="%2F/foo/bar"
2076                  *              "CWD /", "CWD foo", "CWD bar", "RETR file"
2077                  * ftp://host/%2Ffoo/bar/file   dir="%2Ffoo/bar"
2078                  *              "CWD /foo", "CWD bar", "RETR file"
2079                  * ftp://host/%2Ffoo%2Fbar/file dir="%2Ffoo%2Fbar"
2080                  *              "CWD /foo/bar", "RETR file"
2081                  * ftp://host/%2Ffoo%2Fbar%2Ffile       dir=NULL
2082                  *              "RETR /foo/bar/file"
2083                  *
2084                  * Note that we don't need `dir' after this point.
2085                  */
2086                 do {
2087                         if (ui.utype == FTP_URL_T) {
2088                                 nextpart = strchr(dir, '/');
2089                                 if (nextpart) {
2090                                         *nextpart = '\0';
2091                                         nextpart++;
2092                                 }
2093                                 url_decode(dir);
2094                         } else
2095                                 nextpart = NULL;
2096                         DPRINTF("fetch_ftp: dir `%s', nextpart `%s'\n",
2097                             STRorNULL(dir), STRorNULL(nextpart));
2098                         if (ui.utype == FTP_URL_T || *dir != '\0') {
2099                                 (void)strlcpy(cmdbuf, "cd", sizeof(cmdbuf));
2100                                 xargv[0] = cmdbuf;
2101                                 xargv[1] = dir;
2102                                 xargv[2] = NULL;
2103                                 dirchange = 0;
2104                                 cd(2, xargv);
2105                                 if (! dirchange) {
2106                                         if (*dir == '\0' && code == 500)
2107                                                 fprintf(stderr,
2108 "\n"
2109 "ftp: The `CWD ' command (without a directory), which is required by\n"
2110 "     RFC 3986 to support the empty directory in the URL pathname (`//'),\n"
2111 "     conflicts with the server's conformance to RFC 959.\n"
2112 "     Try the same URL without the `//' in the URL pathname.\n"
2113 "\n");
2114                                         goto cleanup_fetch_ftp;
2115                                 }
2116                         }
2117                         dir = nextpart;
2118                 } while (dir != NULL);
2119         }
2120
2121         if (EMPTYSTRING(file)) {
2122                 rval = -1;
2123                 goto cleanup_fetch_ftp;
2124         }
2125
2126         if (dirhasglob) {
2127                 (void)strlcpy(rempath, dir,     sizeof(rempath));
2128                 (void)strlcat(rempath, "/",     sizeof(rempath));
2129                 (void)strlcat(rempath, file,    sizeof(rempath));
2130                 file = rempath;
2131         }
2132
2133                         /* Fetch the file(s). */
2134         xargc = 2;
2135         (void)strlcpy(cmdbuf, "get", sizeof(cmdbuf));
2136         xargv[0] = cmdbuf;
2137         xargv[1] = file;
2138         xargv[2] = NULL;
2139         if (dirhasglob || filehasglob) {
2140                 int ointeractive;
2141
2142                 ointeractive = interactive;
2143                 interactive = 0;
2144                 if (restartautofetch)
2145                         (void)strlcpy(cmdbuf, "mreget", sizeof(cmdbuf));
2146                 else
2147                         (void)strlcpy(cmdbuf, "mget", sizeof(cmdbuf));
2148                 xargv[0] = cmdbuf;
2149                 mget(xargc, xargv);
2150                 interactive = ointeractive;
2151         } else {
2152                 char *destfile = outfile;
2153                 if (destfile == NULL) {
2154                         cp = strrchr(file, '/');        /* find savefile */
2155                         if (cp != NULL)
2156                                 destfile = cp + 1;
2157                         else
2158                                 destfile = file;
2159                 }
2160                 xargv[2] = (char *)destfile;
2161                 xargv[3] = NULL;
2162                 xargc++;
2163                 if (restartautofetch)
2164                         reget(xargc, xargv);
2165                 else
2166                         get(xargc, xargv);
2167         }
2168
2169         if ((code / 100) == COMPLETE)
2170                 rval = 0;
2171
2172  cleanup_fetch_ftp:
2173         freeurlinfo(&ui);
2174         freeauthinfo(&auth);
2175         return (rval);
2176 }
2177
2178 /*
2179  * Retrieve the given file to outfile.
2180  * Supports arguments of the form:
2181  *      "host:path", "ftp://host/path"  if $ftpproxy, call fetch_url() else
2182  *                                      call fetch_ftp()
2183  *      "http://host/path"              call fetch_url() to use HTTP
2184  *      "file:///path"                  call fetch_url() to copy
2185  *      "about:..."                     print a message
2186  *
2187  * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection
2188  * is still open (e.g, ftp xfer with trailing /)
2189  */
2190 static int
2191 go_fetch(const char *url)
2192 {
2193         char *proxyenv;
2194         char *p;
2195
2196 #ifndef NO_ABOUT
2197         /*
2198          * Check for about:*
2199          */
2200         if (STRNEQUAL(url, ABOUT_URL)) {
2201                 url += sizeof(ABOUT_URL) -1;
2202                 if (strcasecmp(url, "ftp") == 0 ||
2203                     strcasecmp(url, "tnftp") == 0) {
2204                         fputs(
2205 "This version of ftp has been enhanced by Luke Mewburn <lukem@NetBSD.org>\n"
2206 "for the NetBSD project.  Execute `man ftp' for more details.\n", ttyout);
2207                 } else if (strcasecmp(url, "lukem") == 0) {
2208                         fputs(
2209 "Luke Mewburn is the author of most of the enhancements in this ftp client.\n"
2210 "Please email feedback to <lukem@NetBSD.org>.\n", ttyout);
2211                 } else if (strcasecmp(url, "netbsd") == 0) {
2212                         fputs(
2213 "NetBSD is a freely available and redistributable UNIX-like operating system.\n"
2214 "For more information, see http://www.NetBSD.org/\n", ttyout);
2215                 } else if (strcasecmp(url, "version") == 0) {
2216                         fprintf(ttyout, "Version: %s %s%s\n",
2217                             FTP_PRODUCT, FTP_VERSION,
2218 #ifdef INET6
2219                             ""
2220 #else
2221                             " (-IPv6)"
2222 #endif
2223                         );
2224                 } else {
2225                         fprintf(ttyout, "`%s' is an interesting topic.\n", url);
2226                 }
2227                 fputs("\n", ttyout);
2228                 return (0);
2229         }
2230 #endif
2231
2232         /*
2233          * Check for file:// and http:// URLs.
2234          */
2235         if (STRNEQUAL(url, HTTP_URL)
2236 #ifdef WITH_SSL
2237             || STRNEQUAL(url, HTTPS_URL)
2238 #endif
2239             || STRNEQUAL(url, FILE_URL))
2240                 return (fetch_url(url, NULL, NULL, NULL));
2241
2242         /*
2243          * If it contains "://" but does not begin with ftp://
2244          * or something that was already handled, then it's
2245          * unsupported.
2246          *
2247          * If it contains ":" but not "://" then we assume the
2248          * part before the colon is a host name, not an URL scheme,
2249          * so we don't try to match that here.
2250          */
2251         if ((p = strstr(url, "://")) != NULL && ! STRNEQUAL(url, FTP_URL))
2252                 errx(1, "Unsupported URL scheme `%.*s'", (int)(p - url), url);
2253
2254         /*
2255          * Try FTP URL-style and host:file arguments next.
2256          * If ftpproxy is set with an FTP URL, use fetch_url()
2257          * Otherwise, use fetch_ftp().
2258          */
2259         proxyenv = getoptionvalue("ftp_proxy");
2260         if (!EMPTYSTRING(proxyenv) && STRNEQUAL(url, FTP_URL))
2261                 return (fetch_url(url, NULL, NULL, NULL));
2262
2263         return (fetch_ftp(url));
2264 }
2265
2266 /*
2267  * Retrieve multiple files from the command line,
2268  * calling go_fetch() for each file.
2269  *
2270  * If an ftp path has a trailing "/", the path will be cd-ed into and
2271  * the connection remains open, and the function will return -1
2272  * (to indicate the connection is alive).
2273  * If an error occurs the return value will be the offset+1 in
2274  * argv[] of the file that caused a problem (i.e, argv[x]
2275  * returns x+1)
2276  * Otherwise, 0 is returned if all files retrieved successfully.
2277  */
2278 int
2279 auto_fetch(int argc, char *argv[])
2280 {
2281         volatile int    argpos, rval;
2282
2283         argpos = rval = 0;
2284
2285         if (sigsetjmp(toplevel, 1)) {
2286                 if (connected)
2287                         disconnect(0, NULL);
2288                 if (rval > 0)
2289                         rval = argpos + 1;
2290                 return (rval);
2291         }
2292         (void)xsignal(SIGINT, intr);
2293         (void)xsignal(SIGPIPE, lostpeer);
2294
2295         /*
2296          * Loop through as long as there's files to fetch.
2297          */
2298         for (; (rval == 0) && (argpos < argc); argpos++) {
2299                 if (strchr(argv[argpos], ':') == NULL)
2300                         break;
2301                 redirect_loop = 0;
2302                 if (!anonftp)
2303                         anonftp = 2;    /* Handle "automatic" transfers. */
2304                 rval = go_fetch(argv[argpos]);
2305                 if (outfile != NULL && strcmp(outfile, "-") != 0
2306                     && outfile[0] != '|') {
2307                         FREEPTR(outfile);
2308                 }
2309                 if (rval > 0)
2310                         rval = argpos + 1;
2311         }
2312
2313         if (connected && rval != -1)
2314                 disconnect(0, NULL);
2315         return (rval);
2316 }
2317
2318
2319 /*
2320  * Upload multiple files from the command line.
2321  *
2322  * If an error occurs the return value will be the offset+1 in
2323  * argv[] of the file that caused a problem (i.e, argv[x]
2324  * returns x+1)
2325  * Otherwise, 0 is returned if all files uploaded successfully.
2326  */
2327 int
2328 auto_put(int argc, char **argv, const char *uploadserver)
2329 {
2330         char    *uargv[4], *path, *pathsep;
2331         int      uargc, rval, argpos;
2332         size_t   len;
2333         char     cmdbuf[MAX_C_NAME];
2334
2335         (void)strlcpy(cmdbuf, "mput", sizeof(cmdbuf));
2336         uargv[0] = cmdbuf;
2337         uargv[1] = argv[0];
2338         uargc = 2;
2339         uargv[2] = uargv[3] = NULL;
2340         pathsep = NULL;
2341         rval = 1;
2342
2343         DPRINTF("auto_put: target `%s'\n", uploadserver);
2344
2345         path = ftp_strdup(uploadserver);
2346         len = strlen(path);
2347         if (path[len - 1] != '/' && path[len - 1] != ':') {
2348                         /*
2349                          * make sure we always pass a directory to auto_fetch
2350                          */
2351                 if (argc > 1) {         /* more than one file to upload */
2352                         len = strlen(uploadserver) + 2; /* path + "/" + "\0" */
2353                         free(path);
2354                         path = (char *)ftp_malloc(len);
2355                         (void)strlcpy(path, uploadserver, len);
2356                         (void)strlcat(path, "/", len);
2357                 } else {                /* single file to upload */
2358                         (void)strlcpy(cmdbuf, "put", sizeof(cmdbuf));
2359                         uargv[0] = cmdbuf;
2360                         pathsep = strrchr(path, '/');
2361                         if (pathsep == NULL) {
2362                                 pathsep = strrchr(path, ':');
2363                                 if (pathsep == NULL) {
2364                                         warnx("Invalid URL `%s'", path);
2365                                         goto cleanup_auto_put;
2366                                 }
2367                                 pathsep++;
2368                                 uargv[2] = ftp_strdup(pathsep);
2369                                 pathsep[0] = '/';
2370                         } else
2371                                 uargv[2] = ftp_strdup(pathsep + 1);
2372                         pathsep[1] = '\0';
2373                         uargc++;
2374                 }
2375         }
2376         DPRINTF("auto_put: URL `%s' argv[2] `%s'\n",
2377             path, STRorNULL(uargv[2]));
2378
2379                         /* connect and cwd */
2380         rval = auto_fetch(1, &path);
2381         if(rval >= 0)
2382                 goto cleanup_auto_put;
2383
2384         rval = 0;
2385
2386                         /* target filename provided; upload 1 file */
2387                         /* XXX : is this the best way? */
2388         if (uargc == 3) {
2389                 uargv[1] = argv[0];
2390                 put(uargc, uargv);
2391                 if ((code / 100) != COMPLETE)
2392                         rval = 1;
2393         } else {        /* otherwise a target dir: upload all files to it */
2394                 for(argpos = 0; argv[argpos] != NULL; argpos++) {
2395                         uargv[1] = argv[argpos];
2396                         mput(uargc, uargv);
2397                         if ((code / 100) != COMPLETE) {
2398                                 rval = argpos + 1;
2399                                 break;
2400                         }
2401                 }
2402         }
2403
2404  cleanup_auto_put:
2405         free(path);
2406         FREEPTR(uargv[2]);
2407         return (rval);
2408 }