2 * Copyright (c) 2000 Dag-Erling Coïdan Smørgrav
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer
10 * in this position and unchanged.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 * 3. The name of the author may not be used to endorse or promote products
15 * derived from this software without specific prior written permission
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28 * $FreeBSD: src/usr.bin/fetch/fetch.c,v 1.10.2.21 2003/06/06 06:48:42 des Exp $
29 * $DragonFly: src/usr.bin/fetch/fetch.c,v 1.7 2006/03/06 03:21:26 swildner Exp $
32 #include <sys/param.h>
33 #include <sys/socket.h>
50 #define MINBUFSIZE 4096
53 int A_flag
; /* -A: do not follow 302 redirects */
54 int a_flag
; /* -a: auto retry */
55 off_t B_size
; /* -B: buffer size */
56 int b_flag
; /*! -b: workaround TCP bug */
57 char *c_dirname
; /* -c: remote directory */
58 int d_flag
; /* -d: direct connection */
59 int F_flag
; /* -F: restart without checking mtime */
60 char *f_filename
; /* -f: file to fetch */
61 char *h_hostname
; /* -h: host to fetch from */
62 int l_flag
; /* -l: link rather than copy file: URLs */
63 int m_flag
; /* -[Mm]: mirror mode */
64 char *N_filename
; /* -N: netrc file name */
65 int n_flag
; /* -n: do not preserve modification time */
66 int o_flag
; /* -o: specify output file */
67 int o_directory
; /* output file is a directory */
68 char *o_filename
; /* name of output file */
69 int o_stdout
; /* output file is stdout */
70 int once_flag
; /* -1: stop at first successful file */
71 int p_flag
; /* -[Pp]: use passive FTP */
72 int R_flag
; /* -R: don't delete partially transferred files */
73 int r_flag
; /* -r: restart previously interrupted transfer */
74 off_t S_size
; /* -S: require size to match */
75 int s_flag
; /* -s: show size, don't fetch */
76 long T_secs
= 120; /* -T: transfer timeout in seconds */
77 int t_flag
; /*! -t: workaround TCP bug */
78 int U_flag
; /* -U: do not use high ports */
79 int v_level
= 1; /* -v: verbosity level */
80 int v_tty
; /* stdout is a tty */
81 pid_t pgrp
; /* our process group */
82 long w_secs
; /* -w: retry delay */
83 int family
= PF_UNSPEC
; /* -[46]: address family to use */
85 int sigalrm
; /* SIGALRM received */
86 int siginfo
; /* SIGINFO received */
87 int sigint
; /* SIGINT received */
89 long ftp_timeout
; /* default timeout for FTP transfers */
90 long http_timeout
; /* default timeout for HTTP transfers */
91 u_char
*buf
; /* transfer buffer */
115 struct timeval start
;
123 * Compute and display ETA
126 stat_eta(struct xferstat
*xs
)
128 long elapsed
, received
, expected
, eta
;
130 elapsed
= xs
->last
.tv_sec
- xs
->start
.tv_sec
;
131 received
= xs
->rcvd
- xs
->offset
;
132 expected
= xs
->size
- xs
->rcvd
;
133 eta
= (long)((double)elapsed
* expected
/ received
);
135 fprintf(stderr
, "%02ld:", eta
/ 3600);
138 fprintf(stderr
, "%02ld:%02ld", eta
/ 60, eta
% 60);
142 * Compute and display transfer rate
145 stat_bps(struct xferstat
*xs
)
149 delta
= (xs
->last
.tv_sec
+ (xs
->last
.tv_usec
/ 1.e6
))
150 - (xs
->start
.tv_sec
+ (xs
->start
.tv_usec
/ 1.e6
));
152 fprintf(stderr
, "?? Bps");
155 bps
= (xs
->rcvd
- xs
->offset
) / delta
;
157 fprintf(stderr
, "%.2f MBps", bps
/ (1024*1024));
159 fprintf(stderr
, "%.2f kBps", bps
/ 1024);
161 fprintf(stderr
, "%.2f Bps", bps
);
165 * Update the stats display
168 stat_display(struct xferstat
*xs
, int force
)
173 if (!v_tty
|| !v_level
)
176 /* check if we're the foreground process */
177 if (ioctl(STDERR_FILENO
, TIOCGPGRP
, &ctty_pgrp
) == -1 ||
178 (pid_t
)ctty_pgrp
!= pgrp
)
181 gettimeofday(&now
, NULL
);
182 if (!force
&& now
.tv_sec
<= xs
->last
.tv_sec
)
186 fprintf(stderr
, "\rReceiving %s", xs
->name
);
188 fprintf(stderr
, ": %lld bytes", (long long)xs
->rcvd
);
190 fprintf(stderr
, " (%lld bytes): %d%%", (long long)xs
->size
,
191 (int)((100.0 * xs
->rcvd
) / xs
->size
));
192 if (xs
->rcvd
> 0 && xs
->last
.tv_sec
>= xs
->start
.tv_sec
+ 30) {
193 fprintf(stderr
, " (ETA ");
196 fprintf(stderr
, " at ");
199 fprintf(stderr
, ") ");
205 * Initialize the transfer statistics
208 stat_start(struct xferstat
*xs
, const char *name
, off_t size
, off_t offset
)
210 snprintf(xs
->name
, sizeof xs
->name
, "%s", name
);
211 gettimeofday(&xs
->start
, NULL
);
212 xs
->last
.tv_sec
= xs
->last
.tv_usec
= 0;
220 * Update the transfer statistics
223 stat_update(struct xferstat
*xs
, off_t rcvd
)
230 * Finalize the transfer statistics
233 stat_end(struct xferstat
*xs
)
240 gettimeofday(&xs
->last
, NULL
);
244 delta
= (xs
->last
.tv_sec
+ (xs
->last
.tv_usec
/ 1.e6
))
245 - (xs
->start
.tv_sec
+ (xs
->start
.tv_usec
/ 1.e6
));
246 fprintf(stderr
, "%lld bytes transferred in %.1f seconds (",
247 (long long)(xs
->rcvd
- xs
->offset
), delta
);
249 fprintf(stderr
, ")\n");
253 * Ask the user for authentication details
256 query_auth(struct url
*URL
)
259 tcflag_t saved_flags
;
263 fprintf(stderr
, "Authentication required for <%s://%s:%d/>!\n",
264 URL
->scheme
, URL
->host
, URL
->port
);
266 fprintf(stderr
, "Login: ");
267 if (fgets(URL
->user
, sizeof URL
->user
, stdin
) == NULL
)
269 for (i
= 0; URL
->user
[i
]; ++i
)
270 if (isspace(URL
->user
[i
]))
273 fprintf(stderr
, "Password: ");
274 if (tcgetattr(STDIN_FILENO
, &tios
) == 0) {
275 saved_flags
= tios
.c_lflag
;
276 tios
.c_lflag
&= ~ECHO
;
277 tios
.c_lflag
|= ECHONL
|ICANON
;
278 tcsetattr(STDIN_FILENO
, TCSAFLUSH
|TCSASOFT
, &tios
);
279 nopwd
= (fgets(URL
->pwd
, sizeof URL
->pwd
, stdin
) == NULL
);
280 tios
.c_lflag
= saved_flags
;
281 tcsetattr(STDIN_FILENO
, TCSANOW
|TCSASOFT
, &tios
);
283 nopwd
= (fgets(URL
->pwd
, sizeof URL
->pwd
, stdin
) == NULL
);
288 for (i
= 0; URL
->pwd
[i
]; ++i
)
289 if (isspace(URL
->pwd
[i
]))
298 fetch(char *URL
, const char *path
)
321 /* set verbosity level */
328 if ((url
= fetchParseURL(URL
)) == NULL
) {
329 warnx("%s: parse error", URL
);
333 /* if no scheme was specified, take a guess */
334 if (*url
->scheme
== 0) {
336 strcpy(url
->scheme
, SCHEME_FILE
);
337 else if (strncasecmp(url
->host
, "ftp.", 4) == 0)
338 strcpy(url
->scheme
, SCHEME_FTP
);
339 else if (strncasecmp(url
->host
, "www.", 4) == 0)
340 strcpy(url
->scheme
, SCHEME_HTTP
);
353 /* FTP specific flags */
354 if (strcmp(url
->scheme
, "ftp") == 0) {
361 timeout
= T_secs
? T_secs
: ftp_timeout
;
364 /* HTTP specific flags */
365 if (strcmp(url
->scheme
, "http") == 0) {
370 timeout
= T_secs
? T_secs
: http_timeout
;
373 /* set the protocol timeout. */
374 fetchTimeout
= timeout
;
376 /* just print size */
380 r
= fetchStat(url
, &us
, flags
);
383 if (sigalrm
|| sigint
)
386 warnx("%s", fetchLastErrString
);
392 printf("%lld\n", (long long)us
.size
);
397 * If the -r flag was specified, we have to compare the local
398 * and remote files, so we should really do a fetchStat()
399 * first, but I know of at least one HTTP server that only
400 * sends the content size in response to GET requests, and
401 * leaves it out of replies to HEAD requests. Also, in the
402 * (frequent) case that the local and remote files match but
403 * the local file is truncated, we have sufficient information
404 * before the compare to issue a correct request. Therefore,
405 * we always issue a GET request as if we were sure the local
406 * file was a truncated copy of the remote file; we can drop
407 * the connection later if we change our minds.
410 if (!o_stdout
&& stat(path
, &sb
) == -1 && errno
!= ENOENT
) {
411 warnx("%s: stat()", path
);
414 if (!o_stdout
&& r_flag
&& S_ISREG(sb
.st_mode
))
415 url
->offset
= sb
.st_size
;
417 /* start the transfer */
420 f
= fetchXGet(url
, &us
, flags
);
423 if (sigalrm
|| sigint
)
426 warnx("%s: %s", URL
, fetchLastErrString
);
432 /* check that size is as expected */
435 warnx("%s: size unknown", URL
);
437 } else if (us
.size
!= S_size
) {
438 warnx("%s: size mismatch: expected %lld, actual %lld",
439 URL
, (long long)S_size
, (long long)us
.size
);
444 /* symlink instead of copy */
445 if (l_flag
&& strcmp(url
->scheme
, "file") == 0 && !o_stdout
) {
446 if (symlink(url
->doc
, path
) == -1) {
447 warn("%s: symlink()", path
);
453 if (us
.size
== -1 && !o_stdout
&& v_level
> 0)
454 warnx("%s: size of remote file is not known", URL
);
456 if (sb
.st_size
!= -1)
457 fprintf(stderr
, "local size / mtime: %lld / %ld\n",
458 (long long)sb
.st_size
, (long)sb
.st_mtime
);
460 fprintf(stderr
, "remote size / mtime: %lld / %ld\n",
461 (long long)us
.size
, (long)us
.mtime
);
464 /* open output file */
466 /* output to stdout */
468 } else if (r_flag
&& sb
.st_size
!= -1) {
469 /* resume mode, local file exists */
470 if (!F_flag
&& us
.mtime
&& sb
.st_mtime
!= us
.mtime
) {
471 /* no match! have to refetch */
473 /* if precious, warn the user and give up */
475 warnx("%s: local modification time "
476 "does not match remote", path
);
480 if (us
.size
== sb
.st_size
)
483 if (sb
.st_size
> us
.size
) {
484 /* local file too long! */
485 warnx("%s: local file (%lld bytes) is longer "
486 "than remote file (%lld bytes)", path
,
487 (long long)sb
.st_size
, (long long)us
.size
);
490 /* we got it, open local file */
491 if ((of
= fopen(path
, "a")) == NULL
) {
492 warn("%s: fopen()", path
);
495 /* check that it didn't move under our feet */
496 if (fstat(fileno(of
), &nsb
) == -1) {
498 warn("%s: fstat()", path
);
501 if (nsb
.st_dev
!= sb
.st_dev
||
502 nsb
.st_ino
!= nsb
.st_ino
||
503 nsb
.st_size
!= sb
.st_size
) {
504 warnx("%s: file has changed", URL
);
510 } else if (m_flag
&& sb
.st_size
!= -1) {
511 /* mirror mode, local file exists */
512 if (sb
.st_size
== us
.size
&& sb
.st_mtime
== us
.mtime
)
518 * We don't yet have an output file; either this is a
519 * vanilla run with no special flags, or the local and
520 * remote files didn't match.
523 if (url
->offset
> 0) {
525 * We tried to restart a transfer, but for
526 * some reason gave up - so we have to restart
527 * from scratch if we want the whole file
530 if ((f
= fetchXGet(url
, &us
, flags
)) == NULL
) {
531 warnx("%s: %s", URL
, fetchLastErrString
);
538 /* construct a temporary file name */
539 if (sb
.st_size
!= -1 && S_ISREG(sb
.st_mode
)) {
540 if ((slash
= strrchr(path
, '/')) == NULL
)
544 asprintf(&tmppath
, "%.*s.fetch.XXXXXX.%s",
545 (int)(slash
- path
), path
, slash
);
546 if (tmppath
!= NULL
) {
547 if (mkstemps(tmppath
, strlen(slash
)+1) == -1) {
548 warn("%s: mkstemps()", path
);
552 of
= fopen(tmppath
, "w");
557 if ((of
= fopen(path
, "w")) == NULL
) {
558 warn("%s: fopen()", path
);
564 /* start the counter */
565 stat_start(&xs
, path
, us
.size
, count
);
567 sigalrm
= siginfo
= sigint
= 0;
569 /* suck in the data */
570 signal(SIGINFO
, sig_handler
);
572 if (us
.size
!= -1 && us
.size
- count
< B_size
&&
573 us
.size
- count
>= 0)
574 size
= us
.size
- count
;
581 if ((size
= fread(buf
, 1, size
, f
)) == 0) {
582 if (ferror(f
) && errno
== EINTR
&& !sigint
)
587 stat_update(&xs
, count
+= size
);
588 for (ptr
= buf
; size
> 0; ptr
+= wr
, size
-= wr
)
589 if ((wr
= fwrite(ptr
, 1, size
, of
)) < size
) {
590 if (ferror(of
) && errno
== EINTR
&& !sigint
)
599 sigalrm
= ferror(f
) && errno
== ETIMEDOUT
;
600 signal(SIGINFO
, SIG_DFL
);
605 * If the transfer timed out or was interrupted, we still want to
606 * set the mtime in case the file is not removed (-r or -R) and
607 * the user later restarts the transfer.
610 /* set mtime of local file */
611 if (!n_flag
&& us
.mtime
&& !o_stdout
&& of
!= NULL
&&
612 (stat(path
, &sb
) != -1) && sb
.st_mode
& S_IFREG
) {
613 struct timeval tv
[2];
616 tv
[0].tv_sec
= (long)(us
.atime
? us
.atime
: us
.mtime
);
617 tv
[1].tv_sec
= (long)us
.mtime
;
618 tv
[0].tv_usec
= tv
[1].tv_usec
= 0;
619 if (utimes(tmppath
? tmppath
: path
, tv
))
620 warn("%s: utimes()", tmppath
? tmppath
: path
);
623 /* timed out or interrupted? */
625 warnx("transfer timed out");
627 warnx("transfer interrupted");
631 /* timeout / interrupt before connection completley established? */
636 /* check the status of our files */
641 if (ferror(f
) || ferror(of
))
645 /* did the transfer complete normally? */
646 if (us
.size
!= -1 && count
< us
.size
) {
647 warnx("%s appears to be truncated: %lld/%lld bytes",
648 path
, (long long)count
, (long long)us
.size
);
653 * If the transfer timed out and we didn't know how much to
654 * expect, assume the worst (i.e. we didn't get all of it)
656 if (sigalrm
&& us
.size
== -1) {
657 warnx("%s may be truncated", path
);
663 if (tmppath
!= NULL
&& rename(tmppath
, path
) == -1) {
664 warn("%s: rename()", path
);
669 if (of
&& of
!= stdout
&& !R_flag
&& !r_flag
)
670 if (stat(path
, &sb
) != -1 && (sb
.st_mode
& S_IFREG
))
671 unlink(tmppath
? tmppath
: path
);
672 if (R_flag
&& tmppath
!= NULL
&& sb
.st_size
== -1)
673 rename(tmppath
, path
); /* ignore errors here */
680 if (of
&& of
!= stdout
)
692 fprintf(stderr
, "%s\n%s\n%s\n",
693 "usage: fetch [-146AFMPRUadlmnpqrsv] [-N netrc] [-o outputfile]",
694 " [-S bytes] [-B bytes] [-T seconds] [-w seconds]",
695 " [-h host -f file [-c dir] | URL ...]");
703 main(int argc
, char *argv
[])
711 while ((c
= getopt(argc
, argv
,
712 "146AaB:bc:dFf:Hh:lMmN:nPpo:qRrS:sT:tUvw:")) != -1)
730 B_size
= (off_t
)strtol(optarg
, &end
, 10);
731 if (*optarg
== '\0' || *end
!= '\0')
732 errx(1, "invalid buffer size (%s)", optarg
);
735 warnx("warning: the -b option is deprecated");
751 warnx("the -H option is now implicit, "
752 "use -U to disable");
767 errx(1, "the -m and -r flags "
768 "are mutually exclusive");
789 errx(1, "the -m and -r flags "
790 "are mutually exclusive");
794 S_size
= (off_t
)strtol(optarg
, &end
, 10);
795 if (*optarg
== '\0' || *end
!= '\0')
796 errx(1, "invalid size (%s)", optarg
);
802 T_secs
= strtol(optarg
, &end
, 10);
803 if (*optarg
== '\0' || *end
!= '\0')
804 errx(1, "invalid timeout (%s)", optarg
);
808 warnx("warning: the -t option is deprecated");
818 w_secs
= strtol(optarg
, &end
, 10);
819 if (*optarg
== '\0' || *end
!= '\0')
820 errx(1, "invalid delay (%s)", optarg
);
830 if (h_hostname
|| f_filename
|| c_dirname
) {
831 if (!h_hostname
|| !f_filename
|| argc
) {
835 /* XXX this is a hack. */
836 if (strcspn(h_hostname
, "@:/") != strlen(h_hostname
))
837 errx(1, "invalid hostname");
838 if (asprintf(argv
, "ftp://%s/%s/%s", h_hostname
,
839 c_dirname
? c_dirname
: "", f_filename
) == -1)
840 errx(1, "%s", strerror(ENOMEM
));
849 /* allocate buffer */
850 if (B_size
< MINBUFSIZE
)
852 if ((buf
= malloc(B_size
)) == NULL
)
853 errx(1, "%s", strerror(ENOMEM
));
856 if ((s
= getenv("FTP_TIMEOUT")) != NULL
) {
857 ftp_timeout
= strtol(s
, &end
, 10);
858 if (*s
== '\0' || *end
!= '\0' || ftp_timeout
< 0) {
859 warnx("FTP_TIMEOUT (%s) is not a positive integer", s
);
863 if ((s
= getenv("HTTP_TIMEOUT")) != NULL
) {
864 http_timeout
= strtol(s
, &end
, 10);
865 if (*s
== '\0' || *end
!= '\0' || http_timeout
< 0) {
866 warnx("HTTP_TIMEOUT (%s) is not a positive integer", s
);
871 /* signal handling */
873 sa
.sa_handler
= sig_handler
;
874 sigemptyset(&sa
.sa_mask
);
875 sigaction(SIGALRM
, &sa
, NULL
);
876 sa
.sa_flags
= SA_RESETHAND
;
877 sigaction(SIGINT
, &sa
, NULL
);
878 fetchRestartCalls
= 0;
882 if (strcmp(o_filename
, "-") == 0) {
884 } else if (stat(o_filename
, &sb
) == -1) {
885 if (errno
== ENOENT
) {
887 errx(EX_USAGE
, "%s is not a directory",
890 err(EX_IOERR
, "%s", o_filename
);
893 if (sb
.st_mode
& S_IFDIR
)
898 /* check if output is to a tty (for progress report) */
899 v_tty
= isatty(STDERR_FILENO
);
907 fetchAuthMethod
= query_auth
;
908 if (N_filename
!= NULL
) {
909 if (setenv("NETRC", N_filename
, 1) == -1)
910 err(1, "setenv: cannot set NETRC=%s", N_filename
);
914 if ((p
= strrchr(*argv
, '/')) == NULL
)
922 fetchLastErrCode
= 0;
926 e
= fetch(*argv
, "-");
927 } else if (o_directory
) {
928 asprintf(&q
, "%s/%s", o_filename
, p
);
932 e
= fetch(*argv
, o_filename
);
939 kill(getpid(), SIGINT
);
941 if (e
== 0 && once_flag
)
946 if ((fetchLastErrCode
947 && fetchLastErrCode
!= FETCH_UNAVAIL
948 && fetchLastErrCode
!= FETCH_MOVED
949 && fetchLastErrCode
!= FETCH_URL
950 && fetchLastErrCode
!= FETCH_RESOLV
951 && fetchLastErrCode
!= FETCH_UNKNOWN
)) {
952 if (w_secs
&& v_level
)
953 fprintf(stderr
, "Waiting %ld seconds "
954 "before retrying\n", w_secs
);