1 /* $NetBSD: fetch.c,v 1.26 2021/08/27 01:38:49 lukem Exp $ */
2 /* from NetBSD: fetch.c,v 1.234 2021/08/01 15:29:30 andvar Exp */
5 * Copyright (c) 1997-2015 The NetBSD Foundation, Inc.
8 * This code is derived from software contributed to The NetBSD Foundation
11 * This code is derived from software contributed to The NetBSD Foundation
12 * by Scott Aaron Bamford.
14 * This code is derived from software contributed to The NetBSD Foundation
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
20 * 1. Redistributions of source code must retain the above copyright
21 * notice, this list of conditions and the following disclaimer.
22 * 2. Redistributions in binary form must reproduce the above copyright
23 * notice, this list of conditions and the following disclaimer in the
24 * documentation and/or other materials provided with the distribution.
26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36 * POSSIBILITY OF SUCH DAMAGE.
43 #include <sys/cdefs.h>
45 __RCSID(" NetBSD: fetch.c,v 1.234 2021/08/01 15:29:30 andvar Exp ");
49 * FTP User Program -- Command line file retrieval
52 #include <sys/types.h>
53 #include <sys/param.h>
54 #include <sys/socket.h>
58 #include <netinet/in.h>
61 #include <arpa/inet.h>
110 __dead
static void aborthttp(int);
111 __dead
static void timeouthttp(int);
113 static int auth_url(const char *, char **, const struct authinfo
*);
114 static void base64_encode(const unsigned char *, size_t, unsigned char *);
116 static int go_fetch(const char *);
117 static int fetch_ftp(const char *);
118 static int fetch_url(const char *, const char *, char *, char *);
119 static const char *match_token(const char **, const char *);
120 static int parse_url(const char *, const char *, struct urlinfo
*,
122 static void url_decode(char *);
123 static void freeauthinfo(struct authinfo
*);
124 static void freeurlinfo(struct urlinfo
*);
126 static int redirect_loop
;
129 #define STRNEQUAL(a,b) (strncasecmp((a), (b), sizeof((b))-1) == 0)
130 #define ISLWS(x) ((x)=='\r' || (x)=='\n' || (x)==' ' || (x)=='\t')
131 #define SKIPLWS(x) do { while (ISLWS((*x))) x++; } while (0)
134 #define ABOUT_URL "about:" /* propaganda */
135 #define FILE_URL "file://" /* file URL prefix */
136 #define FTP_URL "ftp://" /* ftp URL prefix */
137 #define HTTP_URL "http://" /* http URL prefix */
139 #define HTTPS_URL "https://" /* https URL prefix */
141 #define IS_HTTP_TYPE(urltype) \
142 (((urltype) == HTTP_URL_T) || ((urltype) == HTTPS_URL_T))
144 #define IS_HTTP_TYPE(urltype) \
145 ((urltype) == HTTP_URL_T)
149 * fwrite(3) replacement that just uses write(2). Many stdio implementations
150 * don't handle interrupts properly and corrupt the output. We are taking
151 * alarm interrupts because of the progress bar.
153 * Assumes `fp' is pristine with no prior I/O calls on it.
156 maxwrite(const void *buf
, size_t size
, size_t nmemb
, FILE *fp
)
163 size
*= nmemb
; /* assume no overflow */
166 if ((n
= write(fd
, p
, size
)) == -1) {
170 #if defined(EWOULDBLOCK) && EWOULDBLOCK != EAGAIN
186 * Determine if token is the next word in buf (case insensitive).
187 * If so, advance buf past the token and any trailing LWS, and
188 * return a pointer to the token (in buf). Otherwise, return NULL.
189 * token may be preceded by LWS.
190 * token must be followed by LWS or NUL. (I.e, don't partial match).
193 match_token(const char **buf
, const char *token
)
195 const char *p
, *orig
;
198 tlen
= strlen(token
);
202 if (strncasecmp(p
, token
, tlen
) != 0)
205 if (*p
!= '\0' && !ISLWS(*p
))
214 initposinfo(struct posinfo
*pi
)
216 pi
->rangestart
= pi
->rangeend
= pi
->entitylen
= -1;
220 initauthinfo(struct authinfo
*ai
, char *auth
)
223 ai
->user
= ai
->pass
= 0;
227 freeauthinfo(struct authinfo
*a
)
231 memset(a
->pass
, 0, strlen(a
->pass
));
236 initurlinfo(struct urlinfo
*ui
)
238 ui
->host
= ui
->port
= ui
->path
= 0;
239 ui
->utype
= UNKNOWN_URL_T
;
244 copyurlinfo(struct urlinfo
*dui
, struct urlinfo
*sui
)
246 dui
->host
= ftp_strdup(sui
->host
);
247 dui
->port
= ftp_strdup(sui
->port
);
248 dui
->path
= ftp_strdup(sui
->path
);
249 dui
->utype
= sui
->utype
;
250 dui
->portnum
= sui
->portnum
;
254 freeurlinfo(struct urlinfo
*ui
)
263 * Generate authorization response based on given authentication challenge.
264 * Returns -1 if an error occurred, otherwise 0.
265 * Sets response to a malloc(3)ed string; caller should free.
268 auth_url(const char *challenge
, char **response
, const struct authinfo
*auth
)
270 const char *cp
, *scheme
, *errormsg
;
271 char *ep
, *clear
, *realm
;
272 char uuser
[BUFSIZ
], *gotpass
;
275 size_t len
, clen
, rlen
;
278 clear
= realm
= NULL
;
281 scheme
= "Basic"; /* only support Basic authentication */
284 DPRINTF("auth_url: challenge `%s'\n", challenge
);
286 if (! match_token(&cp
, scheme
)) {
287 warnx("Unsupported authentication challenge `%s'",
289 goto cleanup_auth_url
;
292 #define REALM "realm=\""
293 if (STRNEQUAL(cp
, REALM
))
294 cp
+= sizeof(REALM
) - 1;
296 warnx("Unsupported authentication challenge `%s'",
298 goto cleanup_auth_url
;
300 /* XXX: need to improve quoted-string parsing to support \ quoting, etc. */
301 if ((ep
= strchr(cp
, '\"')) != NULL
) {
303 realm
= (char *)ftp_malloc(len
+ 1);
304 (void)strlcpy(realm
, cp
, len
+ 1);
306 warnx("Unsupported authentication challenge `%s'",
308 goto cleanup_auth_url
;
311 fprintf(ttyout
, "Username for `%s': ", realm
);
312 if (auth
->user
!= NULL
) {
313 (void)strlcpy(uuser
, auth
->user
, sizeof(uuser
));
314 fprintf(ttyout
, "%s\n", uuser
);
316 (void)fflush(ttyout
);
317 if (get_line(stdin
, uuser
, sizeof(uuser
), &errormsg
) < 0) {
318 warnx("%s; can't authenticate", errormsg
);
319 goto cleanup_auth_url
;
322 if (auth
->pass
!= NULL
)
325 gotpass
= getpass("Password: ");
326 if (gotpass
== NULL
) {
327 warnx("Can't read password");
328 goto cleanup_auth_url
;
333 clen
= strlen(uuser
) + strlen(upass
) + 2; /* user + ":" + pass + "\0" */
334 clear
= (char *)ftp_malloc(clen
);
335 (void)strlcpy(clear
, uuser
, clen
);
336 (void)strlcat(clear
, ":", clen
);
337 (void)strlcat(clear
, upass
, clen
);
339 memset(gotpass
, 0, strlen(gotpass
));
341 /* scheme + " " + enc + "\0" */
342 rlen
= strlen(scheme
) + 1 + (clen
+ 2) * 4 / 3 + 1;
343 *response
= ftp_malloc(rlen
);
344 (void)strlcpy(*response
, scheme
, rlen
);
345 len
= strlcat(*response
, " ", rlen
);
346 /* use `clen - 1' to not encode the trailing NUL */
347 base64_encode((unsigned char *)clear
, clen
- 1,
348 (unsigned char *)*response
+ len
);
349 memset(clear
, 0, clen
);
359 * Encode len bytes starting at clear using base64 encoding into encoded,
360 * which should be at least ((len + 2) * 4 / 3 + 1) in size.
363 base64_encode(const unsigned char *clear
, size_t len
, unsigned char *encoded
)
365 static const unsigned char enc
[] =
366 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
371 for (i
= 0; i
< len
; i
+= 3) {
372 *(cp
++) = enc
[((clear
[i
+ 0] >> 2))];
373 *(cp
++) = enc
[((clear
[i
+ 0] << 4) & 0x30)
374 | ((clear
[i
+ 1] >> 4) & 0x0f)];
375 *(cp
++) = enc
[((clear
[i
+ 1] << 2) & 0x3c)
376 | ((clear
[i
+ 2] >> 6) & 0x03)];
377 *(cp
++) = enc
[((clear
[i
+ 2] ) & 0x3f)];
386 * Decode %xx escapes in given string, `in-place'.
389 url_decode(char *url
)
391 unsigned char *p
, *q
;
393 if (EMPTYSTRING(url
))
395 p
= q
= (unsigned char *)url
;
397 #define HEXTOINT(x) (x - (isdigit(x) ? '0' : (islower(x) ? 'a' : 'A') - 10))
400 && p
[1] && isxdigit((unsigned char)p
[1])
401 && p
[2] && isxdigit((unsigned char)p
[2])) {
402 *q
++ = HEXTOINT(p
[1]) * 16 + HEXTOINT(p
[2]);
412 * Parse URL of form (per RFC 3986):
413 * <type>://[<user>[:<password>]@]<host>[:<port>][/<path>]
414 * Returns -1 if a parse error occurred, otherwise 0.
415 * It's the caller's responsibility to url_decode() the returned
416 * user, pass and path.
418 * Sets type to url_t, each of the given char ** pointers to a
419 * malloc(3)ed strings of the relevant section, and port to
420 * the number given, or ftpport if ftp://, or httpport if http://.
422 * XXX: this is not totally RFC 3986 compliant; <path> will have the
423 * leading `/' unless it's an ftp:// URL, as this makes things easier
424 * for file:// and http:// URLs. ftp:// URLs have the `/' between the
425 * host and the URL-path removed, but any additional leading slashes
426 * in the URL-path are retained (because they imply that we should
427 * later do "CWD" with a null argument).
430 * input URL output path
431 * --------- -----------
434 * "http://host/path" "/path"
435 * "file://host/dir/file" "dir/file"
439 * "ftp://host/dir/file" "dir/file"
440 * "ftp://host//dir/file" "/dir/file"
444 parse_url(const char *url
, const char *desc
, struct urlinfo
*ui
,
445 struct authinfo
*auth
)
447 const char *origurl
, *tport
;
448 char *cp
, *ep
, *thost
;
451 if (url
== NULL
|| desc
== NULL
|| ui
== NULL
|| auth
== NULL
)
452 errx(1, "parse_url: invoked with NULL argument!");
453 DPRINTF("parse_url: %s `%s'\n", desc
, url
);
458 if (STRNEQUAL(url
, HTTP_URL
)) {
459 url
+= sizeof(HTTP_URL
) - 1;
460 ui
->utype
= HTTP_URL_T
;
461 ui
->portnum
= HTTP_PORT
;
463 } else if (STRNEQUAL(url
, FTP_URL
)) {
464 url
+= sizeof(FTP_URL
) - 1;
465 ui
->utype
= FTP_URL_T
;
466 ui
->portnum
= FTP_PORT
;
468 } else if (STRNEQUAL(url
, FILE_URL
)) {
469 url
+= sizeof(FILE_URL
) - 1;
470 ui
->utype
= FILE_URL_T
;
473 } else if (STRNEQUAL(url
, HTTPS_URL
)) {
474 url
+= sizeof(HTTPS_URL
) - 1;
475 ui
->utype
= HTTPS_URL_T
;
476 ui
->portnum
= HTTPS_PORT
;
480 warnx("Invalid %s `%s'", desc
, url
);
490 /* find [user[:pass]@]host[:port] */
491 ep
= strchr(url
, '/');
493 thost
= ftp_strdup(url
);
496 thost
= (char *)ftp_malloc(len
+ 1);
497 (void)strlcpy(thost
, url
, len
+ 1);
498 if (ui
->utype
== FTP_URL_T
) /* skip first / for ftp URLs */
500 ui
->path
= ftp_strdup(ep
);
503 cp
= strchr(thost
, '@'); /* look for user[:pass]@ in URLs */
505 if (ui
->utype
== FTP_URL_T
)
506 anonftp
= 0; /* disable anonftp */
509 thost
= ftp_strdup(cp
+ 1);
510 cp
= strchr(auth
->user
, ':');
513 auth
->pass
= ftp_strdup(cp
+ 1);
515 url_decode(auth
->user
);
517 url_decode(auth
->pass
);
522 * Check if thost is an encoded IPv6 address, as per
524 * `[' ipv6-address ']'
528 if ((ep
= strchr(cp
, ']')) == NULL
||
529 (ep
[1] != '\0' && ep
[1] != ':')) {
530 warnx("Invalid address `%s' in %s `%s'",
531 thost
, desc
, origurl
);
532 goto cleanup_parse_url
;
534 len
= ep
- cp
; /* change `[xyz]' -> `xyz' */
535 memmove(thost
, thost
+ 1, len
);
537 if (! isipv6addr(thost
)) {
538 warnx("Invalid IPv6 address `%s' in %s `%s'",
539 thost
, desc
, origurl
);
540 goto cleanup_parse_url
;
549 if ((cp
= strchr(thost
, ':')) != NULL
)
553 /* look for [:port] */
557 nport
= strtoul(cp
, &ep
, 10);
558 if (*cp
== '\0' || *ep
!= '\0' ||
559 nport
< 1 || nport
> MAX_IN_PORT_T
) {
560 warnx("Unknown port `%s' in %s `%s'",
562 goto cleanup_parse_url
;
569 ui
->port
= ftp_strdup(tport
);
570 if (ui
->path
== NULL
) {
571 const char *emptypath
= "/";
572 if (ui
->utype
== FTP_URL_T
) /* skip first / for ftp URLs */
574 ui
->path
= ftp_strdup(emptypath
);
577 DPRINTF("parse_url: user `%s' pass `%s' host %s port %s(%d) "
579 STRorNULL(auth
->user
), STRorNULL(auth
->pass
),
580 STRorNULL(ui
->host
), STRorNULL(ui
->port
),
581 ui
->portnum
? ui
->portnum
: -1, STRorNULL(ui
->path
));
586 sigjmp_buf httpabort
;
589 ftp_socket(const struct urlinfo
*ui
, void **ssl
)
591 struct addrinfo hints
, *res
, *res0
= NULL
;
594 const char *host
= ui
->host
;
595 const char *port
= ui
->port
;
597 if (ui
->utype
!= HTTPS_URL_T
)
600 memset(&hints
, 0, sizeof(hints
));
602 hints
.ai_family
= family
;
603 hints
.ai_socktype
= SOCK_STREAM
;
604 hints
.ai_protocol
= 0;
606 error
= getaddrinfo(host
, port
, &hints
, &res0
);
608 warnx("Can't LOOKUP `%s:%s': %s", host
, port
,
609 (error
== EAI_SYSTEM
) ? strerror(errno
)
610 : gai_strerror(error
));
614 if (res0
->ai_canonname
)
615 host
= res0
->ai_canonname
;
620 for (res
= res0
; res
; res
= res
->ai_next
) {
621 char hname
[NI_MAXHOST
], sname
[NI_MAXSERV
];
624 if (getnameinfo(res
->ai_addr
, res
->ai_addrlen
,
625 hname
, sizeof(hname
), sname
, sizeof(sname
),
626 NI_NUMERICHOST
| NI_NUMERICSERV
) != 0) {
627 strlcpy(hname
, "?", sizeof(hname
));
628 strlcpy(sname
, "?", sizeof(sname
));
631 if (verbose
&& res0
->ai_next
) {
633 if(res
->ai_family
== AF_INET6
) {
634 fprintf(ttyout
, "Trying [%s]:%s ...\n",
638 fprintf(ttyout
, "Trying %s:%s ...\n",
645 s
= socket(res
->ai_family
, SOCK_STREAM
, res
->ai_protocol
);
648 "Can't create socket for connection to "
649 "`%s:%s'", hname
, sname
);
653 if (ftp_connect(s
, res
->ai_addr
, res
->ai_addrlen
,
654 verbose
|| !res
->ai_next
) < 0) {
662 if ((*ssl
= fetch_start_ssl(s
, host
)) == NULL
) {
677 handle_noproxy(const char *host
, in_port_t portnum
)
680 char *cp
, *ep
, *np
, *np_copy
, *np_iter
, *no_proxy
;
681 unsigned long np_port
;
685 /* check URL against list of no_proxied sites */
686 no_proxy
= getoptionvalue("no_proxy");
687 if (EMPTYSTRING(no_proxy
))
690 np_iter
= np_copy
= ftp_strdup(no_proxy
);
692 while ((cp
= strsep(&np_iter
, " ,")) != NULL
) {
695 if ((np
= strrchr(cp
, ':')) != NULL
) {
697 np_port
= strtoul(np
, &ep
, 10);
698 if (*np
== '\0' || *ep
!= '\0')
700 if (np_port
!= portnum
)
706 if (strncasecmp(host
+ hlen
- plen
, cp
, plen
) == 0) {
716 handle_proxy(const char *url
, const char *penv
, struct urlinfo
*ui
,
717 struct authinfo
*pauth
)
721 if (isipv6addr(ui
->host
) && strchr(ui
->host
, '%') != NULL
) {
722 warnx("Scoped address notation `%s' disallowed via web proxy",
728 if (parse_url(penv
, "proxy URL", &pui
, pauth
) == -1)
731 if ((!IS_HTTP_TYPE(pui
.utype
) && pui
.utype
!= FTP_URL_T
) ||
732 EMPTYSTRING(pui
.host
) ||
733 (! EMPTYSTRING(pui
.path
) && strcmp(pui
.path
, "/") != 0)) {
734 warnx("Malformed proxy URL `%s'", penv
);
740 pui
.path
= ftp_strdup(url
);
749 print_host(FETCH
*fin
, const struct urlinfo
*ui
)
753 if (strchr(ui
->host
, ':') == NULL
) {
754 fetch_printf(fin
, "Host: %s", ui
->host
);
757 * strip off IPv6 scope identifier, since it is
760 h
= ftp_strdup(ui
->host
);
761 if (isipv6addr(h
) && (p
= strchr(h
, '%')) != NULL
)
764 fetch_printf(fin
, "Host: [%s]", h
);
768 if ((ui
->utype
== HTTP_URL_T
&& ui
->portnum
!= HTTP_PORT
) ||
769 (ui
->utype
== HTTPS_URL_T
&& ui
->portnum
!= HTTPS_PORT
))
770 fetch_printf(fin
, ":%u", ui
->portnum
);
771 fetch_printf(fin
, "\r\n");
775 print_agent(FETCH
*fin
)
777 const char *useragent
;
778 if ((useragent
= getenv("FTPUSERAGENT")) != NULL
) {
779 fetch_printf(fin
, "User-Agent: %s\r\n", useragent
);
781 fetch_printf(fin
, "User-Agent: %s/%s\r\n",
782 FTP_PRODUCT
, FTP_VERSION
);
787 print_cache(FETCH
*fin
, int isproxy
)
789 fetch_printf(fin
, isproxy
?
790 "Pragma: no-cache\r\n" :
791 "Cache-Control: no-cache\r\n");
795 print_get(FETCH
*fin
, int hasleading
, int isproxy
, const struct urlinfo
*oui
,
796 const struct urlinfo
*ui
)
798 const char *leading
= hasleading
? ", " : " (";
802 fprintf(ttyout
, "%svia %s:%u", leading
,
803 ui
->host
, ui
->portnum
);
807 fetch_printf(fin
, "GET %s HTTP/1.0\r\n", ui
->path
);
808 print_host(fin
, oui
);
812 fetch_printf(fin
, "GET %s HTTP/1.1\r\n", ui
->path
);
814 fetch_printf(fin
, "Accept: */*\r\n");
815 fetch_printf(fin
, "Connection: close\r\n");
817 fputs(leading
, ttyout
);
818 fetch_printf(fin
, "Range: bytes=" LLF
"-\r\n",
820 fprintf(ttyout
, "restarting at " LLF
, (LLT
)restart_point
);
827 getmtime(const char *cp
, time_t *mtime
)
832 memset(&parsed
, 0, sizeof(parsed
));
833 t
= parse_rfc2616time(&parsed
, cp
);
838 parsed
.tm_isdst
= -1;
840 *mtime
= timegm(&parsed
);
843 if (ftp_debug
&& *mtime
!= -1) {
844 fprintf(ttyout
, "parsed time as: %s",
845 rfc2822time(localtime(mtime
)));
851 print_proxy(FETCH
*fin
, int hasleading
, const char *wwwauth
,
852 const char *proxyauth
)
854 const char *leading
= hasleading
? ", " : " (";
858 fprintf(ttyout
, "%swith authorization", leading
);
861 fetch_printf(fin
, "Authorization: %s\r\n", wwwauth
);
865 fprintf(ttyout
, "%swith proxy authorization", leading
);
868 fetch_printf(fin
, "Proxy-Authorization: %s\r\n", proxyauth
);
875 print_connect(FETCH
*fin
, const struct urlinfo
*ui
)
877 char hname
[NI_MAXHOST
], *p
;
880 if (isipv6addr(ui
->host
)) {
882 * strip off IPv6 scope identifier,
883 * since it is local to the node
885 if ((p
= strchr(ui
->host
, '%')) == NULL
)
886 snprintf(hname
, sizeof(hname
), "[%s]", ui
->host
);
888 snprintf(hname
, sizeof(hname
), "[%.*s]",
889 (int)(p
- ui
->host
), ui
->host
);
894 fetch_printf(fin
, "CONNECT %s:%d HTTP/1.1\r\n", h
, ui
->portnum
);
895 fetch_printf(fin
, "Host: %s:%d\r\n", h
, ui
->portnum
);
904 getresponseline(FETCH
*fin
, char *buf
, size_t buflen
, int *len
)
906 const char *errormsg
;
908 alarmtimer(quit_time
? quit_time
: 60);
909 *len
= fetch_getline(fin
, buf
, buflen
, &errormsg
);
912 if (*errormsg
== '\n')
914 warnx("Receiving HTTP reply: %s", errormsg
);
917 while (*len
> 0 && (ISLWS(buf
[*len
-1])))
921 DPRINTF("%s: received `%s'\n", __func__
, buf
);
926 getresponse(FETCH
*fin
, char **cp
, size_t buflen
, int *hcode
)
929 char *ep
, *buf
= *cp
;
932 if ((rv
= getresponseline(fin
, buf
, buflen
, &len
)) != C_OK
)
935 /* Determine HTTP response code */
936 *cp
= strchr(buf
, ' ');
942 *hcode
= strtol(*cp
, &ep
, 10);
943 if (*ep
!= '\0' && !isspace((unsigned char)*ep
))
950 parse_posinfo(const char **cp
, struct posinfo
*pi
)
953 if (!match_token(cp
, "bytes"))
959 pi
->rangestart
= STRTOLL(*cp
, &ep
, 10);
960 if (pi
->rangestart
< 0 || *ep
!= '-')
963 pi
->rangeend
= STRTOLL(*cp
, &ep
, 10);
964 if (pi
->rangeend
< 0 || pi
->rangeend
< pi
->rangestart
)
974 pi
->entitylen
= STRTOLL(*cp
, &ep
, 10);
975 if (pi
->entitylen
< 0)
984 fprintf(ttyout
, "parsed range as: ");
985 if (pi
->rangestart
== -1)
986 fprintf(ttyout
, "*");
988 fprintf(ttyout
, LLF
"-" LLF
, (LLT
)pi
->rangestart
,
990 fprintf(ttyout
, "/" LLF
"\n", (LLT
)pi
->entitylen
);
998 do_auth(int hcode
, const char *url
, const char *penv
, struct authinfo
*wauth
,
999 struct authinfo
*pauth
, char **auth
, const char *message
,
1002 struct authinfo aauth
;
1010 if (verbose
|| aauth
.auth
== NULL
||
1011 aauth
.user
== NULL
|| aauth
.pass
== NULL
)
1012 fprintf(ttyout
, "%s\n", message
);
1013 if (EMPTYSTRING(*auth
)) {
1014 warnx("No authentication challenge provided by server");
1018 if (aauth
.auth
!= NULL
) {
1021 fprintf(ttyout
, "Authorization failed. Retry (y/n)? ");
1022 if (get_line(stdin
, reply
, sizeof(reply
), NULL
) < 0) {
1025 if (tolower((unsigned char)reply
[0]) != 'y')
1032 if (auth_url(*auth
, &response
, &aauth
) == 0) {
1033 *rval
= fetch_url(url
, penv
,
1034 hcode
== 401 ? pauth
->auth
: response
,
1035 hcode
== 401 ? response
: wauth
->auth
);
1036 memset(response
, 0, strlen(response
));
1043 negotiate_connection(FETCH
*fin
, const char *url
, const char *penv
,
1044 struct posinfo
*pi
, time_t *mtime
, struct authinfo
*wauth
,
1045 struct authinfo
*pauth
, volatile int *rval
, volatile int *ischunked
,
1049 char buf
[FTPBUFLEN
], *ep
;
1050 const char *cp
, *token
;
1051 char *location
, *message
;
1053 *auth
= message
= location
= NULL
;
1055 /* Read the response */
1057 switch (getresponse(fin
, &ep
, sizeof(buf
), &hcode
)) {
1059 goto cleanup_fetch_url
;
1063 message
= ftp_strdup(ep
);
1067 /* Read the rest of the header. */
1070 if ((rv
= getresponseline(fin
, buf
, sizeof(buf
), &len
)) != C_OK
)
1071 goto cleanup_fetch_url
;
1076 * Look for some headers
1081 if (match_token(&cp
, "Content-Length:")) {
1082 filesize
= STRTOLL(cp
, &ep
, 10);
1083 if (filesize
< 0 || *ep
!= '\0')
1085 DPRINTF("%s: parsed len as: " LLF
"\n",
1086 __func__
, (LLT
)filesize
);
1088 } else if (match_token(&cp
, "Content-Range:")) {
1089 if (parse_posinfo(&cp
, pi
) == -1)
1091 if (! restart_point
) {
1093 "Received unexpected Content-Range header");
1094 goto cleanup_fetch_url
;
1097 } else if (match_token(&cp
, "Last-Modified:")) {
1098 getmtime(cp
, mtime
);
1100 } else if (match_token(&cp
, "Location:")) {
1101 location
= ftp_strdup(cp
);
1102 DPRINTF("%s: parsed location as `%s'\n",
1105 } else if (match_token(&cp
, "Transfer-Encoding:")) {
1106 if (match_token(&cp
, "binary")) {
1108 "Bogus transfer encoding `binary' (fetching anyway)");
1111 if (! (token
= match_token(&cp
, "chunked"))) {
1113 "Unsupported transfer encoding `%s'",
1115 goto cleanup_fetch_url
;
1118 DPRINTF("%s: using chunked encoding\n",
1121 } else if (match_token(&cp
, "Proxy-Authenticate:")
1122 || match_token(&cp
, "WWW-Authenticate:")) {
1123 if (! (token
= match_token(&cp
, "Basic"))) {
1124 DPRINTF("%s: skipping unknown auth "
1125 "scheme `%s'\n", __func__
, token
);
1129 *auth
= ftp_strdup(token
);
1130 DPRINTF("%s: parsed auth as `%s'\n",
1135 /* finished parsing header */
1141 if (! restart_point
) {
1142 warnx("Not expecting partial content header");
1143 goto cleanup_fetch_url
;
1152 if (EMPTYSTRING(location
)) {
1154 "No redirection Location provided by server");
1155 goto cleanup_fetch_url
;
1157 if (redirect_loop
++ > 5) {
1158 warnx("Too many redirections requested");
1159 goto cleanup_fetch_url
;
1163 fprintf(ttyout
, "Redirected via %s\n",
1165 *rval
= fetch_url(url
, location
,
1166 pauth
->auth
, wauth
->auth
);
1169 fprintf(ttyout
, "Redirected to %s\n",
1171 *rval
= go_fetch(location
);
1173 goto cleanup_fetch_url
;
1177 do_auth(hcode
, url
, penv
, wauth
, pauth
, auth
, message
, rval
);
1178 goto cleanup_fetch_url
;
1182 warnx("Error retrieving file `%s'", message
);
1184 warnx("Unknown error retrieving file");
1185 goto cleanup_fetch_url
;
1200 } /* end of ftp:// or http:// specific setup */
1204 connectmethod(FETCH
*fin
, const char *url
, const char *penv
,
1205 struct urlinfo
*oui
, struct urlinfo
*ui
, struct authinfo
*wauth
,
1206 struct authinfo
*pauth
, char **auth
, int *hasleading
, volatile int *rval
)
1211 char buf
[FTPBUFLEN
], *ep
;
1212 char *message
= NULL
;
1214 print_connect(fin
, oui
);
1217 *hasleading
= print_proxy(fin
, *hasleading
, NULL
, pauth
->auth
);
1219 if (verbose
&& *hasleading
)
1220 fputs(")\n", ttyout
);
1223 fetch_printf(fin
, "\r\n");
1224 if (fetch_flush(fin
) == EOF
) {
1225 warn("Writing HTTP request");
1227 goto cleanup_fetch_url
;
1231 /* Read the response */
1233 switch (getresponse(fin
, &ep
, sizeof(buf
), &hcode
)) {
1235 goto cleanup_fetch_url
;
1239 message
= ftp_strdup(ep
);
1245 if (getresponseline(fin
, buf
, sizeof(buf
), &len
) != C_OK
)
1246 goto cleanup_fetch_url
;
1251 if (match_token(&cp
, "Proxy-Authenticate:")) {
1253 if (!(token
= match_token(&cp
, "Basic"))) {
1255 "%s: skipping unknown auth scheme `%s'\n",
1260 *auth
= ftp_strdup(token
);
1261 DPRINTF("%s: parsed auth as " "`%s'\n", __func__
, cp
);
1265 /* finished parsing header */
1271 do_auth(hcode
, url
, penv
, wauth
, pauth
, auth
, message
, rval
);
1272 goto cleanup_fetch_url
;
1276 warnx("Error proxy connect " "`%s'", message
);
1278 warnx("Unknown error proxy " "connect");
1279 goto cleanup_fetch_url
;
1282 if ((ssl
= fetch_start_ssl(fetch_fileno(fin
), oui
->host
)) == NULL
)
1283 goto cleanup_fetch_url
;
1284 fetch_set_ssl(fin
, ssl
);
1301 * Retrieve URL, via a proxy if necessary, using HTTP.
1302 * If proxyenv is set, use that for the proxy, otherwise try ftp_proxy or
1303 * http_proxy/https_proxy as appropriate.
1304 * Supports HTTP redirects.
1305 * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection
1306 * is still open (e.g, ftp xfer with trailing /)
1309 fetch_url(const char *url
, const char *proxyenv
, char *proxyauth
, char *wwwauth
)
1311 sigfunc
volatile oldint
;
1312 sigfunc
volatile oldpipe
;
1313 sigfunc
volatile oldalrm
;
1314 sigfunc
volatile oldquit
;
1317 int volatile isproxy
;
1318 int volatile rval
, ischunked
;
1320 static size_t bufsize
;
1321 static char *xferbuf
;
1324 char *volatile auth
;
1325 char *volatile savefile
;
1326 char *volatile location
;
1327 char *volatile message
;
1328 char *volatile decodedpath
;
1329 struct authinfo wauth
, pauth
;
1332 int (*volatile closefunc
)(FILE *);
1333 FETCH
*volatile fin
;
1334 FILE *volatile fout
;
1335 const char *volatile penv
= proxyenv
;
1336 struct urlinfo ui
, oui
;
1340 DPRINTF("%s: `%s' proxyenv `%s'\n", __func__
, url
, STRorNULL(penv
));
1342 oldquit
= oldalrm
= oldint
= oldpipe
= SIG_ERR
;
1348 auth
= location
= message
= NULL
;
1349 ischunked
= isproxy
= 0;
1354 initauthinfo(&wauth
, wwwauth
);
1355 initauthinfo(&pauth
, proxyauth
);
1359 if (sigsetjmp(httpabort
, 1))
1360 goto cleanup_fetch_url
;
1362 if (parse_url(url
, "URL", &ui
, &wauth
) == -1)
1363 goto cleanup_fetch_url
;
1365 copyurlinfo(&oui
, &ui
);
1367 if (ui
.utype
== FILE_URL_T
&& ! EMPTYSTRING(ui
.host
)
1368 && strcasecmp(ui
.host
, "localhost") != 0) {
1369 warnx("No support for non local file URL `%s'", url
);
1370 goto cleanup_fetch_url
;
1373 if (EMPTYSTRING(ui
.path
)) {
1374 if (ui
.utype
== FTP_URL_T
) {
1375 rval
= fetch_ftp(url
);
1376 goto cleanup_fetch_url
;
1378 if (!IS_HTTP_TYPE(ui
.utype
) || outfile
== NULL
) {
1379 warnx("Invalid URL (no file after host) `%s'", url
);
1380 goto cleanup_fetch_url
;
1384 decodedpath
= ftp_strdup(ui
.path
);
1385 url_decode(decodedpath
);
1390 cp
= strrchr(decodedpath
, '/'); /* find savefile */
1392 savefile
= ftp_strdup(cp
+ 1);
1394 savefile
= ftp_strdup(decodedpath
);
1396 * Use the first URL we requested not the name after a
1397 * possible redirect, but careful to save it because our
1398 * "safety" check is the match to outfile.
1400 outfile
= ftp_strdup(savefile
);
1402 DPRINTF("%s: savefile `%s'\n", __func__
, savefile
);
1403 if (EMPTYSTRING(savefile
)) {
1404 if (ui
.utype
== FTP_URL_T
) {
1405 rval
= fetch_ftp(url
);
1406 goto cleanup_fetch_url
;
1408 warnx("No file after directory (you must specify an "
1409 "output file) `%s'", url
);
1410 goto cleanup_fetch_url
;
1417 if (restartautofetch
) {
1418 if (stat(savefile
, &sb
) == 0)
1419 restart_point
= sb
.st_size
;
1421 if (ui
.utype
== FILE_URL_T
) { /* file:// URLs */
1422 direction
= "copied";
1423 fin
= fetch_open(decodedpath
, "r");
1425 warn("Can't open `%s'", decodedpath
);
1426 goto cleanup_fetch_url
;
1428 if (fstat(fetch_fileno(fin
), &sb
) == 0) {
1429 mtime
= sb
.st_mtime
;
1430 filesize
= sb
.st_size
;
1432 if (restart_point
) {
1433 if (lseek(fetch_fileno(fin
), restart_point
, SEEK_SET
) < 0) {
1434 warn("Can't seek to restart `%s'",
1436 goto cleanup_fetch_url
;
1440 fprintf(ttyout
, "Copying %s", decodedpath
);
1442 fprintf(ttyout
, " (restarting at " LLF
")",
1443 (LLT
)restart_point
);
1444 fputs("\n", ttyout
);
1446 if (0 == rcvbuf_size
) {
1447 rcvbuf_size
= 8 * 1024; /* XXX */
1449 } else { /* ftp:// or http:// URLs */
1454 if (ui
.utype
== HTTPS_URL_T
)
1455 penv
= getoptionvalue("https_proxy");
1457 if (penv
== NULL
&& IS_HTTP_TYPE(ui
.utype
))
1458 penv
= getoptionvalue("http_proxy");
1459 else if (ui
.utype
== FTP_URL_T
)
1460 penv
= getoptionvalue("ftp_proxy");
1462 direction
= "retrieved";
1463 if (! EMPTYSTRING(penv
)) { /* use proxy */
1465 isproxy
= handle_noproxy(ui
.host
, ui
.portnum
);
1467 if (isproxy
== 0 && ui
.utype
== FTP_URL_T
) {
1468 rval
= fetch_ftp(url
);
1469 goto cleanup_fetch_url
;
1473 if (restart_point
) {
1475 "Can't restart via proxy URL `%s'",
1477 goto cleanup_fetch_url
;
1479 if (handle_proxy(url
, penv
, &ui
, &pauth
) < 0)
1480 goto cleanup_fetch_url
;
1482 } /* ! EMPTYSTRING(penv) */
1484 s
= ftp_socket(&ui
, &ssl
);
1486 warnx("Can't connect to `%s:%s'", ui
.host
, ui
.port
);
1487 goto cleanup_fetch_url
;
1490 oldalrm
= xsignal(SIGALRM
, timeouthttp
);
1491 alarmtimer(quit_time
? quit_time
: 60);
1492 fin
= fetch_fdopen(s
, "r+");
1493 fetch_set_ssl(fin
, ssl
);
1496 alarmtimer(quit_time
? quit_time
: 60);
1498 * Construct and send the request.
1501 fprintf(ttyout
, "Requesting %s\n", url
);
1505 if (isproxy
&& oui
.utype
== HTTPS_URL_T
) {
1506 switch (connectmethod(fin
, url
, penv
, &oui
, &ui
,
1507 &wauth
, &pauth
, __UNVOLATILE(&auth
), &hasleading
,
1510 goto cleanup_fetch_url
;
1521 hasleading
= print_get(fin
, hasleading
, isproxy
, &oui
, &ui
);
1524 print_cache(fin
, isproxy
);
1527 hasleading
= print_proxy(fin
, hasleading
, wauth
.auth
,
1528 auth
? NULL
: pauth
.auth
);
1532 fputs(")\n", ttyout
);
1535 fetch_printf(fin
, "\r\n");
1536 if (fetch_flush(fin
) == EOF
) {
1537 warn("Writing HTTP request");
1539 goto cleanup_fetch_url
;
1543 switch (negotiate_connection(fin
, url
, penv
, &pi
,
1544 &mtime
, &wauth
, &pauth
, &rval
, &ischunked
,
1545 __UNVOLATILE(&auth
))) {
1549 goto cleanup_fetch_url
;
1557 /* Open the output file. */
1560 * Only trust filenames with special meaning if they came from
1563 if (outfile
== savefile
) {
1564 if (strcmp(savefile
, "-") == 0) {
1566 } else if (*savefile
== '|') {
1568 oldpipe
= xsignal(SIGPIPE
, SIG_IGN
);
1569 fout
= popen(savefile
+ 1, "w");
1571 warn("Can't execute `%s'", savefile
+ 1);
1572 goto cleanup_fetch_url
;
1579 if ((pi
.rangeend
!= -1 && pi
.rangeend
<= restart_point
) ||
1580 (pi
.rangestart
== -1 &&
1581 filesize
!= -1 && filesize
<= restart_point
)) {
1584 fprintf(ttyout
, "already done\n");
1586 goto cleanup_fetch_url
;
1588 if (restart_point
&& pi
.rangestart
!= -1) {
1589 if (pi
.entitylen
!= -1)
1590 filesize
= pi
.entitylen
;
1591 if (pi
.rangestart
!= restart_point
) {
1593 "Size of `%s' differs from save file `%s'",
1595 goto cleanup_fetch_url
;
1597 fout
= fopen(savefile
, "a");
1599 fout
= fopen(savefile
, "w");
1601 warn("Can't open `%s'", savefile
);
1602 goto cleanup_fetch_url
;
1608 oldquit
= xsignal(SIGQUIT
, psummary
);
1609 oldint
= xsignal(SIGINT
, aborthttp
);
1611 assert(rcvbuf_size
> 0);
1612 if ((size_t)rcvbuf_size
> bufsize
) {
1614 (void)free(xferbuf
);
1615 bufsize
= rcvbuf_size
;
1616 xferbuf
= ftp_malloc(bufsize
);
1621 if (oldalrm
!= SIG_ERR
) {
1622 (void)xsignal(SIGALRM
, oldalrm
);
1627 /* Finally, suck down the file. */
1634 /* read chunk-size */
1636 if (fetch_getln(xferbuf
, bufsize
, fin
) == NULL
) {
1637 warnx("Unexpected EOF reading chunk-size");
1638 goto cleanup_fetch_url
;
1641 chunksize
= strtol(xferbuf
, &ep
, 16);
1642 if (ep
== xferbuf
) {
1643 warnx("Invalid chunk-size");
1644 goto cleanup_fetch_url
;
1646 if (errno
== ERANGE
|| chunksize
< 0) {
1648 warn("Chunk-size `%.*s'",
1649 (int)(ep
-xferbuf
), xferbuf
);
1650 goto cleanup_fetch_url
;
1654 * XXX: Work around bug in Apache 1.3.9 and
1655 * 1.3.11, which incorrectly put trailing
1656 * space after the chunk-size.
1661 /* skip [ chunk-ext ] */
1663 while (*ep
&& *ep
!= '\r')
1667 if (strcmp(ep
, "\r\n") != 0) {
1668 warnx("Unexpected data following chunk-size");
1669 goto cleanup_fetch_url
;
1671 DPRINTF("%s: got chunk-size of " LLF
"\n", __func__
,
1673 if (chunksize
== 0) {
1678 /* transfer file or chunk */
1680 struct timeval then
, now
, td
;
1681 volatile off_t bufrem
;
1684 (void)gettimeofday(&then
, NULL
);
1685 bufrem
= rate_get
? rate_get
: (off_t
)bufsize
;
1687 bufrem
= MIN(chunksize
, bufrem
);
1688 while (bufrem
> 0) {
1689 size_t nr
= MIN((off_t
)bufsize
, bufrem
);
1690 flen
= fetch_read(xferbuf
, sizeof(char),
1693 if (fetch_error(fin
))
1699 if (maxwrite(xferbuf
, sizeof(char), flen
, fout
)
1701 warn("Writing `%s'", savefile
);
1702 goto cleanup_fetch_url
;
1704 if (hash
&& !progress
) {
1705 while (bytes
>= hashbytes
) {
1706 (void)putc('#', ttyout
);
1709 (void)fflush(ttyout
);
1719 (void)gettimeofday(&now
, NULL
);
1720 timersub(&now
, &then
, &td
);
1723 usleep(1000000 - td
.tv_usec
);
1726 if (ischunked
&& chunksize
<= 0)
1729 /* read CRLF after chunk*/
1732 if (fetch_getln(xferbuf
, bufsize
, fin
) == NULL
) {
1734 warnx("Unexpected EOF reading chunk CRLF");
1735 goto cleanup_fetch_url
;
1737 if (strcmp(xferbuf
, "\r\n") != 0) {
1738 warnx("Unexpected data following chunk");
1739 goto cleanup_fetch_url
;
1744 } while (ischunked
);
1746 /* XXX: deal with optional trailer & CRLF here? */
1748 if (hash
&& !progress
&& bytes
> 0) {
1750 (void)putc('#', ttyout
);
1751 (void)putc('\n', ttyout
);
1753 if (fetch_error(fin
)) {
1754 warn("Reading file");
1755 goto cleanup_fetch_url
;
1759 if (closefunc
== fclose
&& mtime
!= -1) {
1760 struct timeval tval
[2];
1762 (void)gettimeofday(&tval
[0], NULL
);
1763 tval
[1].tv_sec
= mtime
;
1764 tval
[1].tv_usec
= 0;
1768 if (utimes(savefile
, tval
) == -1) {
1770 "Can't change modification time to %s",
1771 rfc2822time(localtime(&mtime
)));
1779 goto cleanup_fetch_url
;
1782 warnx("Improper response from `%s:%s'", ui
.host
, ui
.port
);
1785 if (oldint
!= SIG_ERR
)
1786 (void)xsignal(SIGINT
, oldint
);
1787 if (oldpipe
!= SIG_ERR
)
1788 (void)xsignal(SIGPIPE
, oldpipe
);
1789 if (oldalrm
!= SIG_ERR
)
1790 (void)xsignal(SIGALRM
, oldalrm
);
1791 if (oldquit
!= SIG_ERR
)
1792 (void)xsignal(SIGQUIT
, oldquit
);
1797 if (closefunc
!= NULL
&& fout
!= NULL
)
1799 if (savefile
!= outfile
)
1803 freeauthinfo(&wauth
);
1804 freeauthinfo(&pauth
);
1805 FREEPTR(decodedpath
);
1813 * Abort a HTTP retrieval
1816 aborthttp(int notused
)
1824 len
= snprintf(msgbuf
, sizeof(msgbuf
),
1825 "\n%s: HTTP fetch aborted.\n", getprogname());
1827 write(fileno(ttyout
), msgbuf
, len
);
1829 siglongjmp(httpabort
, 1);
1833 timeouthttp(int notused
)
1840 len
= snprintf(msgbuf
, sizeof(msgbuf
),
1841 "\n%s: HTTP fetch timeout.\n", getprogname());
1843 write(fileno(ttyout
), msgbuf
, len
);
1845 siglongjmp(httpabort
, 1);
1849 * Retrieve ftp URL or classic ftp argument using FTP.
1850 * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection
1851 * is still open (e.g, ftp xfer with trailing /)
1854 fetch_ftp(const char *url
)
1856 char *cp
, *xargv
[5], rempath
[MAXPATHLEN
];
1858 char cmdbuf
[MAXPATHLEN
];
1860 int dirhasglob
, filehasglob
, rval
, transtype
, xargc
;
1861 int oanonftp
, oautologin
;
1862 struct authinfo auth
;
1865 DPRINTF("fetch_ftp: `%s'\n", url
);
1871 initauthinfo(&auth
, NULL
);
1873 if (STRNEQUAL(url
, FTP_URL
)) {
1874 if ((parse_url(url
, "URL", &ui
, &auth
) == -1) ||
1875 (auth
.user
!= NULL
&& *auth
.user
== '\0') ||
1876 EMPTYSTRING(ui
.host
)) {
1877 warnx("Invalid URL `%s'", url
);
1878 goto cleanup_fetch_ftp
;
1881 * Note: Don't url_decode(path) here. We need to keep the
1882 * distinction between "/" and "%2F" until later.
1885 /* check for trailing ';type=[aid]' */
1886 if (! EMPTYSTRING(ui
.path
) && (cp
= strrchr(ui
.path
, ';')) != NULL
) {
1887 if (strcasecmp(cp
, ";type=a") == 0)
1889 else if (strcasecmp(cp
, ";type=i") == 0)
1891 else if (strcasecmp(cp
, ";type=d") == 0) {
1893 "Directory listing via a URL is not supported");
1894 goto cleanup_fetch_ftp
;
1896 warnx("Invalid suffix `%s' in URL `%s'", cp
,
1898 goto cleanup_fetch_ftp
;
1902 } else { /* classic style `[user@]host:[file]' */
1903 ui
.utype
= CLASSIC_URL_T
;
1904 ui
.host
= ftp_strdup(url
);
1905 cp
= strchr(ui
.host
, '@');
1908 auth
.user
= ui
.host
;
1909 anonftp
= 0; /* disable anonftp */
1910 ui
.host
= ftp_strdup(cp
+ 1);
1912 cp
= strchr(ui
.host
, ':');
1915 ui
.path
= ftp_strdup(cp
+ 1);
1918 if (EMPTYSTRING(ui
.host
))
1919 goto cleanup_fetch_ftp
;
1921 /* Extract the file and (if present) directory name. */
1923 if (! EMPTYSTRING(dir
)) {
1925 * If we are dealing with classic `[user@]host:[path]' syntax,
1926 * then a path of the form `/file' (resulting from input of the
1927 * form `host:/file') means that we should do "CWD /" before
1928 * retrieving the file. So we set dir="/" and file="file".
1930 * But if we are dealing with URLs like `ftp://host/path' then
1931 * a path of the form `/file' (resulting from a URL of the form
1932 * `ftp://host//file') means that we should do `CWD ' (with an
1933 * empty argument) before retrieving the file. So we set
1934 * dir="" and file="file".
1936 * If the path does not contain / at all, we set dir=NULL.
1937 * (We get a path without any slashes if we are dealing with
1938 * classic `[user@]host:[file]' or URL `ftp://host/file'.)
1940 * In all other cases, we set dir to a string that does not
1941 * include the final '/' that separates the dir part from the
1942 * file part of the path. (This will be the empty string if
1943 * and only if we are dealing with a path of the form `/file'
1944 * resulting from an URL of the form `ftp://host//file'.)
1946 cp
= strrchr(dir
, '/');
1947 if (cp
== dir
&& ui
.utype
== CLASSIC_URL_T
) {
1949 (void)strlcpy(dirbuf
, "/", sizeof(dirbuf
));
1951 } else if (cp
!= NULL
) {
1960 if (ui
.utype
== FTP_URL_T
&& file
!= NULL
) {
1962 /* but still don't url_decode(dir) */
1964 DPRINTF("fetch_ftp: user `%s' pass `%s' host %s port %s "
1965 "path `%s' dir `%s' file `%s'\n",
1966 STRorNULL(auth
.user
), STRorNULL(auth
.pass
),
1967 STRorNULL(ui
.host
), STRorNULL(ui
.port
),
1968 STRorNULL(ui
.path
), STRorNULL(dir
), STRorNULL(file
));
1970 dirhasglob
= filehasglob
= 0;
1972 (ui
.utype
== CLASSIC_URL_T
|| ui
.utype
== FTP_URL_T
)) {
1973 if (! EMPTYSTRING(dir
) && strpbrk(dir
, "*?[]{}") != NULL
)
1975 if (! EMPTYSTRING(file
) && strpbrk(file
, "*?[]{}") != NULL
)
1979 /* Set up the connection */
1982 disconnect(0, NULL
);
1984 (void)strlcpy(cmdbuf
, getprogname(), sizeof(cmdbuf
));
1994 oautologin
= autologin
;
1995 /* don't autologin in setpeer(), use ftp_login() below */
1997 setpeer(xargc
, xargv
);
1998 autologin
= oautologin
;
1999 if ((connected
== 0) ||
2000 (connected
== 1 && !ftp_login(ui
.host
, auth
.user
, auth
.pass
))) {
2001 warnx("Can't connect or login to host `%s:%s'",
2002 ui
.host
, ui
.port
? ui
.port
: "?");
2003 goto cleanup_fetch_ftp
;
2006 switch (transtype
) {
2011 setbinary(1, xargv
);
2014 errx(1, "fetch_ftp: unknown transfer type %d", transtype
);
2018 * Change directories, if necessary.
2020 * Note: don't use EMPTYSTRING(dir) below, because
2021 * dir=="" means something different from dir==NULL.
2023 if (dir
!= NULL
&& !dirhasglob
) {
2027 * If we are dealing with a classic `[user@]host:[path]'
2028 * (urltype is CLASSIC_URL_T) then we have a raw directory
2029 * name (not encoded in any way) and we can change
2030 * directories in one step.
2032 * If we are dealing with an `ftp://host/path' URL
2033 * (urltype is FTP_URL_T), then RFC 3986 says we need to
2034 * send a separate CWD command for each unescaped "/"
2035 * in the path, and we have to interpret %hex escaping
2036 * *after* we find the slashes. It's possible to get
2037 * empty components here, (from multiple adjacent
2038 * slashes in the path) and RFC 3986 says that we should
2039 * still do `CWD ' (with a null argument) in such cases.
2041 * Many ftp servers don't support `CWD ', so if there's an
2042 * error performing that command, bail out with a descriptive
2047 * host: dir="", urltype=CLASSIC_URL_T
2048 * logged in (to default directory)
2049 * host:file dir=NULL, urltype=CLASSIC_URL_T
2051 * host:dir/ dir="dir", urltype=CLASSIC_URL_T
2052 * "CWD dir", logged in
2053 * ftp://host/ dir="", urltype=FTP_URL_T
2054 * logged in (to default directory)
2055 * ftp://host/dir/ dir="dir", urltype=FTP_URL_T
2056 * "CWD dir", logged in
2057 * ftp://host/file dir=NULL, urltype=FTP_URL_T
2059 * ftp://host//file dir="", urltype=FTP_URL_T
2060 * "CWD ", "RETR file"
2061 * host:/file dir="/", urltype=CLASSIC_URL_T
2062 * "CWD /", "RETR file"
2063 * ftp://host///file dir="/", urltype=FTP_URL_T
2064 * "CWD ", "CWD ", "RETR file"
2065 * ftp://host/%2F/file dir="%2F", urltype=FTP_URL_T
2066 * "CWD /", "RETR file"
2067 * ftp://host/foo/file dir="foo", urltype=FTP_URL_T
2068 * "CWD foo", "RETR file"
2069 * ftp://host/foo/bar/file dir="foo/bar"
2070 * "CWD foo", "CWD bar", "RETR file"
2071 * ftp://host//foo/bar/file dir="/foo/bar"
2072 * "CWD ", "CWD foo", "CWD bar", "RETR file"
2073 * ftp://host/foo//bar/file dir="foo//bar"
2074 * "CWD foo", "CWD ", "CWD bar", "RETR file"
2075 * ftp://host/%2F/foo/bar/file dir="%2F/foo/bar"
2076 * "CWD /", "CWD foo", "CWD bar", "RETR file"
2077 * ftp://host/%2Ffoo/bar/file dir="%2Ffoo/bar"
2078 * "CWD /foo", "CWD bar", "RETR file"
2079 * ftp://host/%2Ffoo%2Fbar/file dir="%2Ffoo%2Fbar"
2080 * "CWD /foo/bar", "RETR file"
2081 * ftp://host/%2Ffoo%2Fbar%2Ffile dir=NULL
2082 * "RETR /foo/bar/file"
2084 * Note that we don't need `dir' after this point.
2087 if (ui
.utype
== FTP_URL_T
) {
2088 nextpart
= strchr(dir
, '/');
2096 DPRINTF("fetch_ftp: dir `%s', nextpart `%s'\n",
2097 STRorNULL(dir
), STRorNULL(nextpart
));
2098 if (ui
.utype
== FTP_URL_T
|| *dir
!= '\0') {
2099 (void)strlcpy(cmdbuf
, "cd", sizeof(cmdbuf
));
2106 if (*dir
== '\0' && code
== 500)
2109 "ftp: The `CWD ' command (without a directory), which is required by\n"
2110 " RFC 3986 to support the empty directory in the URL pathname (`//'),\n"
2111 " conflicts with the server's conformance to RFC 959.\n"
2112 " Try the same URL without the `//' in the URL pathname.\n"
2114 goto cleanup_fetch_ftp
;
2118 } while (dir
!= NULL
);
2121 if (EMPTYSTRING(file
)) {
2123 goto cleanup_fetch_ftp
;
2127 (void)strlcpy(rempath
, dir
, sizeof(rempath
));
2128 (void)strlcat(rempath
, "/", sizeof(rempath
));
2129 (void)strlcat(rempath
, file
, sizeof(rempath
));
2133 /* Fetch the file(s). */
2135 (void)strlcpy(cmdbuf
, "get", sizeof(cmdbuf
));
2139 if (dirhasglob
|| filehasglob
) {
2142 ointeractive
= interactive
;
2144 if (restartautofetch
)
2145 (void)strlcpy(cmdbuf
, "mreget", sizeof(cmdbuf
));
2147 (void)strlcpy(cmdbuf
, "mget", sizeof(cmdbuf
));
2150 interactive
= ointeractive
;
2152 char *destfile
= outfile
;
2153 if (destfile
== NULL
) {
2154 cp
= strrchr(file
, '/'); /* find savefile */
2160 xargv
[2] = (char *)destfile
;
2163 if (restartautofetch
)
2164 reget(xargc
, xargv
);
2169 if ((code
/ 100) == COMPLETE
)
2174 freeauthinfo(&auth
);
2179 * Retrieve the given file to outfile.
2180 * Supports arguments of the form:
2181 * "host:path", "ftp://host/path" if $ftpproxy, call fetch_url() else
2183 * "http://host/path" call fetch_url() to use HTTP
2184 * "file:///path" call fetch_url() to copy
2185 * "about:..." print a message
2187 * Returns 1 on failure, 0 on completed xfer, -1 if ftp connection
2188 * is still open (e.g, ftp xfer with trailing /)
2191 go_fetch(const char *url
)
2200 if (STRNEQUAL(url
, ABOUT_URL
)) {
2201 url
+= sizeof(ABOUT_URL
) -1;
2202 if (strcasecmp(url
, "ftp") == 0 ||
2203 strcasecmp(url
, "tnftp") == 0) {
2205 "This version of ftp has been enhanced by Luke Mewburn <lukem@NetBSD.org>\n"
2206 "for the NetBSD project. Execute `man ftp' for more details.\n", ttyout
);
2207 } else if (strcasecmp(url
, "lukem") == 0) {
2209 "Luke Mewburn is the author of most of the enhancements in this ftp client.\n"
2210 "Please email feedback to <lukem@NetBSD.org>.\n", ttyout
);
2211 } else if (strcasecmp(url
, "netbsd") == 0) {
2213 "NetBSD is a freely available and redistributable UNIX-like operating system.\n"
2214 "For more information, see http://www.NetBSD.org/\n", ttyout
);
2215 } else if (strcasecmp(url
, "version") == 0) {
2216 fprintf(ttyout
, "Version: %s %s%s\n",
2217 FTP_PRODUCT
, FTP_VERSION
,
2225 fprintf(ttyout
, "`%s' is an interesting topic.\n", url
);
2227 fputs("\n", ttyout
);
2233 * Check for file:// and http:// URLs.
2235 if (STRNEQUAL(url
, HTTP_URL
)
2237 || STRNEQUAL(url
, HTTPS_URL
)
2239 || STRNEQUAL(url
, FILE_URL
))
2240 return (fetch_url(url
, NULL
, NULL
, NULL
));
2243 * If it contains "://" but does not begin with ftp://
2244 * or something that was already handled, then it's
2247 * If it contains ":" but not "://" then we assume the
2248 * part before the colon is a host name, not an URL scheme,
2249 * so we don't try to match that here.
2251 if ((p
= strstr(url
, "://")) != NULL
&& ! STRNEQUAL(url
, FTP_URL
))
2252 errx(1, "Unsupported URL scheme `%.*s'", (int)(p
- url
), url
);
2255 * Try FTP URL-style and host:file arguments next.
2256 * If ftpproxy is set with an FTP URL, use fetch_url()
2257 * Otherwise, use fetch_ftp().
2259 proxyenv
= getoptionvalue("ftp_proxy");
2260 if (!EMPTYSTRING(proxyenv
) && STRNEQUAL(url
, FTP_URL
))
2261 return (fetch_url(url
, NULL
, NULL
, NULL
));
2263 return (fetch_ftp(url
));
2267 * Retrieve multiple files from the command line,
2268 * calling go_fetch() for each file.
2270 * If an ftp path has a trailing "/", the path will be cd-ed into and
2271 * the connection remains open, and the function will return -1
2272 * (to indicate the connection is alive).
2273 * If an error occurs the return value will be the offset+1 in
2274 * argv[] of the file that caused a problem (i.e, argv[x]
2276 * Otherwise, 0 is returned if all files retrieved successfully.
2279 auto_fetch(int argc
, char *argv
[])
2281 volatile int argpos
, rval
;
2285 if (sigsetjmp(toplevel
, 1)) {
2287 disconnect(0, NULL
);
2292 (void)xsignal(SIGINT
, intr
);
2293 (void)xsignal(SIGPIPE
, lostpeer
);
2296 * Loop through as long as there's files to fetch.
2298 for (; (rval
== 0) && (argpos
< argc
); argpos
++) {
2299 if (strchr(argv
[argpos
], ':') == NULL
)
2303 anonftp
= 2; /* Handle "automatic" transfers. */
2304 rval
= go_fetch(argv
[argpos
]);
2305 if (outfile
!= NULL
&& strcmp(outfile
, "-") != 0
2306 && outfile
[0] != '|') {
2313 if (connected
&& rval
!= -1)
2314 disconnect(0, NULL
);
2320 * Upload multiple files from the command line.
2322 * If an error occurs the return value will be the offset+1 in
2323 * argv[] of the file that caused a problem (i.e, argv[x]
2325 * Otherwise, 0 is returned if all files uploaded successfully.
2328 auto_put(int argc
, char **argv
, const char *uploadserver
)
2330 char *uargv
[4], *path
, *pathsep
;
2331 int uargc
, rval
, argpos
;
2333 char cmdbuf
[MAX_C_NAME
];
2335 (void)strlcpy(cmdbuf
, "mput", sizeof(cmdbuf
));
2339 uargv
[2] = uargv
[3] = NULL
;
2343 DPRINTF("auto_put: target `%s'\n", uploadserver
);
2345 path
= ftp_strdup(uploadserver
);
2347 if (path
[len
- 1] != '/' && path
[len
- 1] != ':') {
2349 * make sure we always pass a directory to auto_fetch
2351 if (argc
> 1) { /* more than one file to upload */
2352 len
= strlen(uploadserver
) + 2; /* path + "/" + "\0" */
2354 path
= (char *)ftp_malloc(len
);
2355 (void)strlcpy(path
, uploadserver
, len
);
2356 (void)strlcat(path
, "/", len
);
2357 } else { /* single file to upload */
2358 (void)strlcpy(cmdbuf
, "put", sizeof(cmdbuf
));
2360 pathsep
= strrchr(path
, '/');
2361 if (pathsep
== NULL
) {
2362 pathsep
= strrchr(path
, ':');
2363 if (pathsep
== NULL
) {
2364 warnx("Invalid URL `%s'", path
);
2365 goto cleanup_auto_put
;
2368 uargv
[2] = ftp_strdup(pathsep
);
2371 uargv
[2] = ftp_strdup(pathsep
+ 1);
2376 DPRINTF("auto_put: URL `%s' argv[2] `%s'\n",
2377 path
, STRorNULL(uargv
[2]));
2379 /* connect and cwd */
2380 rval
= auto_fetch(1, &path
);
2382 goto cleanup_auto_put
;
2386 /* target filename provided; upload 1 file */
2387 /* XXX : is this the best way? */
2391 if ((code
/ 100) != COMPLETE
)
2393 } else { /* otherwise a target dir: upload all files to it */
2394 for(argpos
= 0; argv
[argpos
] != NULL
; argpos
++) {
2395 uargv
[1] = argv
[argpos
];
2397 if ((code
/ 100) != COMPLETE
) {