1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171#include "libbb.h"
172
173#if 0
174# define log_io(...) bb_error_msg(__VA_ARGS__)
175# define SENDFMT(fp, fmt, ...) \
176 do { \
177 log_io("> " fmt, ##__VA_ARGS__); \
178 fprintf(fp, fmt, ##__VA_ARGS__); \
179 } while (0);
180#else
181# define log_io(...) ((void)0)
182# define SENDFMT(fp, fmt, ...) fprintf(fp, fmt, ##__VA_ARGS__)
183#endif
184
185
186#define SSL_SUPPORTED (ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_HTTPS)
187#define FTPS_SUPPORTED (ENABLE_FEATURE_WGET_FTP && ENABLE_FEATURE_WGET_HTTPS)
188
189struct host_info {
190 char *allocated;
191 const char *path;
192 char *user;
193 const char *protocol;
194 char *host;
195 int port;
196};
197static const char P_HTTP[] ALIGN1 = "http";
198#if SSL_SUPPORTED
199static const char P_HTTPS[] ALIGN1 = "https";
200#endif
201#if ENABLE_FEATURE_WGET_FTP
202static const char P_FTP[] ALIGN1 = "ftp";
203#endif
204#if FTPS_SUPPORTED
205static const char P_FTPS[] ALIGN1 = "ftps";
206#endif
207
208#if ENABLE_FEATURE_WGET_LONG_OPTIONS
209
210enum {
211 HDR_HOST = (1<<0),
212 HDR_USER_AGENT = (1<<1),
213 HDR_RANGE = (1<<2),
214 HDR_CONTENT_TYPE = (1<<3),
215 HDR_AUTH = (1<<4) * ENABLE_FEATURE_WGET_AUTHENTICATION,
216 HDR_PROXY_AUTH = (1<<5) * ENABLE_FEATURE_WGET_AUTHENTICATION,
217};
218static const char wget_user_headers[] ALIGN1 =
219 "Host:\0"
220 "User-Agent:\0"
221 "Range:\0"
222 "Content-Type:\0"
223# if ENABLE_FEATURE_WGET_AUTHENTICATION
224 "Authorization:\0"
225 "Proxy-Authorization:\0"
226# endif
227 ;
228# define USR_HEADER_HOST (G.user_headers & HDR_HOST)
229# define USR_HEADER_USER_AGENT (G.user_headers & HDR_USER_AGENT)
230# define USR_HEADER_RANGE (G.user_headers & HDR_RANGE)
231# define USR_HEADER_CONTENT_TYPE (G.user_headers & HDR_CONTENT_TYPE)
232# define USR_HEADER_AUTH (G.user_headers & HDR_AUTH)
233# define USR_HEADER_PROXY_AUTH (G.user_headers & HDR_PROXY_AUTH)
234#else
235# define USR_HEADER_HOST 0
236# define USR_HEADER_USER_AGENT 0
237# define USR_HEADER_RANGE 0
238# define USR_HEADER_CONTENT_TYPE 0
239# define USR_HEADER_AUTH 0
240# define USR_HEADER_PROXY_AUTH 0
241#endif
242
243
244struct globals {
245 off_t content_len;
246 off_t beg_range;
247#if ENABLE_FEATURE_WGET_STATUSBAR
248 off_t transferred;
249 const char *curfile;
250 bb_progress_t pmt;
251#endif
252 char *dir_prefix;
253#if ENABLE_FEATURE_WGET_LONG_OPTIONS
254 char *post_data;
255 char *post_file;
256 char *extra_headers;
257 unsigned char user_headers;
258#endif
259 char *fname_out;
260 char *fname_log;
261 const char *proxy_flag;
262 const char *user_agent;
263 int output_fd;
264 int log_fd;
265 int o_flags;
266#if ENABLE_FEATURE_WGET_TIMEOUT
267 unsigned timeout_seconds;
268 smallint die_if_timed_out;
269#endif
270 smallint chunked;
271 smallint got_clen;
272
273
274
275
276 char wget_buf[CONFIG_FEATURE_COPYBUF_KB*1024] ALIGNED(16);
277} FIX_ALIASING;
278#define G (*ptr_to_globals)
279#define INIT_G() do { \
280 SET_PTR_TO_GLOBALS(xzalloc(sizeof(G))); \
281} while (0)
282#define FINI_G() do { \
283 FREE_PTR_TO_GLOBALS(); \
284} while (0)
285
286
287
288enum {
289 WGET_OPT_CONTINUE = (1 << 0),
290 WGET_OPT_QUIET = (1 << 1),
291 WGET_OPT_SERVER_RESPONSE = (1 << 2),
292 WGET_OPT_OUTNAME = (1 << 3),
293 WGET_OPT_LOGNAME = (1 << 4),
294 WGET_OPT_PREFIX = (1 << 5),
295 WGET_OPT_PROXY = (1 << 6),
296 WGET_OPT_USER_AGENT = (1 << 7),
297 WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 8),
298 WGET_OPT_RETRIES = (1 << 9),
299 WGET_OPT_nsomething = (1 << 10),
300 WGET_OPT_HEADER = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
301 WGET_OPT_POST_DATA = (1 << 12) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
302 WGET_OPT_SPIDER = (1 << 13) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
303 WGET_OPT_NO_CHECK_CERT = (1 << 14) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
304 WGET_OPT_POST_FILE = (1 << 15) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
305
306 WGET_NO_FTRUNCATE = (1 << 31)
307};
308
309#define WGET_OPT_POST (WGET_OPT_POST_DATA | WGET_OPT_POST_FILE)
310
311enum {
312 PROGRESS_START = -1,
313 PROGRESS_END = 0,
314 PROGRESS_BUMP = 1,
315};
316#if ENABLE_FEATURE_WGET_STATUSBAR
317static void progress_meter(int flag)
318{
319 int notty;
320
321 if (option_mask32 & WGET_OPT_QUIET)
322 return;
323
324
325 if (G.log_fd >= 0)
326 return;
327
328 if (flag == PROGRESS_START)
329 bb_progress_init(&G.pmt, G.curfile);
330
331 notty = bb_progress_update(&G.pmt,
332 G.beg_range,
333 G.transferred,
334 (G.chunked || !G.got_clen) ? 0 : G.beg_range + G.transferred + G.content_len
335 );
336
337 if (flag == PROGRESS_END) {
338 bb_progress_free(&G.pmt);
339 if (notty == 0)
340 bb_putchar_stderr('\n');
341 G.transferred = 0;
342 }
343}
344#else
345static ALWAYS_INLINE void progress_meter(int flag UNUSED_PARAM) {}
346#endif
347
348
349
350
351
352
353
354
355
356
357
358
359static void strip_ipv6_scope_id(char *host)
360{
361 char *scope, *cp;
362
363
364
365
366
367 if (host[0] != '[')
368 return;
369
370 scope = strchr(host, '%');
371 if (!scope)
372 return;
373
374
375 cp = strchr(host, ']');
376 if (!cp || (cp[1] != ':' && cp[1] != '\0')) {
377
378 return;
379 }
380
381
382 overlapping_strcpy(scope, cp);
383}
384
385#if ENABLE_FEATURE_WGET_AUTHENTICATION
386
387static char *base64enc(const char *str)
388{
389
390 unsigned len = strnlen(str, sizeof(G.wget_buf)/4*3 - 10);
391 bb_uuencode(G.wget_buf, str, len, bb_uuenc_tbl_base64);
392 return G.wget_buf;
393}
394#endif
395
396#if ENABLE_FEATURE_WGET_TIMEOUT
397static void alarm_handler(int sig UNUSED_PARAM)
398{
399
400 if (G.die_if_timed_out)
401 bb_simple_error_msg_and_die("download timed out");
402}
403static void set_alarm(void)
404{
405 if (G.timeout_seconds) {
406 alarm(G.timeout_seconds);
407 G.die_if_timed_out = 1;
408 }
409}
410# define clear_alarm() ((void)(G.die_if_timed_out = 0))
411#else
412# define set_alarm() ((void)0)
413# define clear_alarm() ((void)0)
414#endif
415
416#if ENABLE_FEATURE_WGET_OPENSSL
417
418
419
420
421
422static int is_ip_address(const char *string)
423{
424 struct sockaddr_in sa;
425
426 int result = inet_pton(AF_INET, string, &(sa.sin_addr));
427# if ENABLE_FEATURE_IPV6
428 if (result == 0) {
429 struct sockaddr_in6 sa6;
430 result = inet_pton(AF_INET6, string, &(sa6.sin6_addr));
431 }
432# endif
433 return (result == 1);
434}
435#endif
436
437static FILE *open_socket(len_and_sockaddr *lsa)
438{
439 int fd;
440 FILE *fp;
441
442 set_alarm();
443 fd = xconnect_stream(lsa);
444 clear_alarm();
445
446
447
448 fp = fdopen(fd, "r+");
449 if (!fp)
450 bb_die_memory_exhausted();
451
452 return fp;
453}
454
455
456
457
458
459
460
461
462static char* sanitize_string(char *s)
463{
464 unsigned char *p = (void *) s;
465 while (*p) {
466 if (*p < ' ') {
467 if (*p != '\t')
468 break;
469 *p = ' ';
470 }
471 p++;
472 }
473 *p = '\0';
474 return s;
475}
476
477
478static char fgets_trim_sanitize(FILE *fp, const char *fmt)
479{
480 char c;
481 char *buf_ptr;
482
483 set_alarm();
484 if (fgets(G.wget_buf, sizeof(G.wget_buf), fp) == NULL)
485 bb_simple_perror_msg_and_die("error getting response");
486 clear_alarm();
487
488 buf_ptr = strchrnul(G.wget_buf, '\n');
489 c = *buf_ptr;
490#if 1
491
492 sanitize_string(G.wget_buf);
493#else
494 *buf_ptr = '\0';
495 buf_ptr = strchrnul(G.wget_buf, '\r');
496 *buf_ptr = '\0';
497#endif
498
499 log_io("< %s", G.wget_buf);
500
501 if (fmt && (option_mask32 & WGET_OPT_SERVER_RESPONSE))
502 fprintf(stderr, fmt, G.wget_buf);
503
504 return c;
505}
506
507#if ENABLE_FEATURE_WGET_FTP
508static int ftpcmd(const char *s1, const char *s2, FILE *fp)
509{
510 int result;
511 if (s1) {
512 if (!s2)
513 s2 = "";
514 fprintf(fp, "%s%s\r\n", s1, s2);
515
516 if (option_mask32 & WGET_OPT_SERVER_RESPONSE)
517 fprintf(stderr, "--> %s%s\n\n", s1, s2);
518 fflush(fp);
519 log_io("> %s%s", s1, s2);
520 }
521
522
523 G.wget_buf[3] = 0;
524 do {
525 fgets_trim_sanitize(fp, "%s\n");
526 } while (!isdigit(G.wget_buf[0]) || G.wget_buf[3] != ' ');
527
528 G.wget_buf[3] = '\0';
529 result = xatoi_positive(G.wget_buf);
530 G.wget_buf[3] = ' ';
531 return result;
532}
533#endif
534
535static void parse_url(const char *src_url, struct host_info *h)
536{
537 char *url, *p, *sp;
538
539 free(h->allocated);
540 h->allocated = url = xstrdup(src_url);
541
542 h->protocol = P_HTTP;
543 p = strstr(url, "://");
544 if (p) {
545 *p = '\0';
546 h->host = p + 3;
547#if ENABLE_FEATURE_WGET_FTP
548 if (strcmp(url, P_FTP) == 0) {
549 h->port = bb_lookup_std_port(P_FTP, "tcp", 21);
550 h->protocol = P_FTP;
551 } else
552#endif
553#if FTPS_SUPPORTED
554 if (strcmp(url, P_FTPS) == 0) {
555 h->port = bb_lookup_std_port(P_FTPS, "tcp", 990);
556 h->protocol = P_FTPS;
557 } else
558#endif
559#if SSL_SUPPORTED
560 if (strcmp(url, P_HTTPS) == 0) {
561 h->port = bb_lookup_std_port(P_HTTPS, "tcp", 443);
562 h->protocol = P_HTTPS;
563 } else
564#endif
565 if (strcmp(url, P_HTTP) == 0) {
566 goto http;
567 } else {
568 *p = ':';
569 bb_error_msg_and_die("not an http or ftp url: %s", url);
570 }
571 } else {
572
573 h->host = url;
574 http:
575 h->port = bb_lookup_std_port(P_HTTP, "tcp", 80);
576 }
577
578
579
580
581
582
583
584
585
586
587
588
589
590 sp = strchr(h->host, '/');
591 p = strchr(h->host, '?'); if (!sp || (p && sp > p)) sp = p;
592 p = strchr(h->host, '#'); if (!sp || (p && sp > p)) sp = p;
593 if (!sp) {
594 h->path = "";
595 } else if (*sp == '/') {
596 *sp = '\0';
597 h->path = sp + 1;
598 } else {
599
600
601
602
603
604 *sp++ = '\0';
605 h->path = sp;
606 }
607
608 sp = strrchr(h->host, '@');
609 if (sp != NULL) {
610
611
612
613
614
615 *sp = '\0';
616 free(h->user);
617 h->user = xstrdup(percent_decode_in_place(h->host, 0));
618 h->host = sp + 1;
619 }
620
621
622
623}
624
625static char *get_sanitized_hdr(FILE *fp)
626{
627 char *s, *hdrval;
628 int c;
629
630
631 c = fgets_trim_sanitize(fp, " %s\n");
632
633
634 if (G.wget_buf[0] == '\0')
635 return NULL;
636
637
638 for (s = G.wget_buf; isalnum(*s) || *s == '-' || *s == '.' || *s == '_'; ++s) {
639
640
641
642
643
644
645
646
647
648 *s |= 0x20;
649 }
650
651
652 if (*s != ':')
653 bb_error_msg_and_die("bad header line: %s", G.wget_buf);
654
655
656 *s++ = '\0';
657 hdrval = skip_whitespace(s);
658
659 if (c != '\n') {
660
661 while (c = getc(fp), c != EOF && c != '\n')
662 continue;
663 }
664
665 return hdrval;
666}
667
668static void reset_beg_range_to_zero(void)
669{
670 bb_simple_error_msg("restart failed");
671 G.beg_range = 0;
672 xlseek(G.output_fd, 0, SEEK_SET);
673
674
675}
676
677#if ENABLE_FEATURE_WGET_OPENSSL
678static int spawn_https_helper_openssl(const char *host, unsigned port)
679{
680 char *allocated = NULL;
681 char *servername;
682 int sp[2];
683 int pid;
684 IF_FEATURE_WGET_HTTPS(volatile int child_failed = 0;)
685
686 if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) != 0)
687
688 bb_simple_perror_msg_and_die("socketpair");
689
690 if (!strchr(host, ':'))
691 host = allocated = xasprintf("%s:%u", host, port);
692 servername = xstrdup(host);
693 strrchr(servername, ':')[0] = '\0';
694
695 fflush_all();
696 pid = xvfork();
697 if (pid == 0) {
698
699 char *argv[13];
700 char **argp;
701
702 close(sp[0]);
703 xmove_fd(sp[1], 0);
704 xdup2(0, 1);
705
706
707
708
709
710 xmove_fd(2, 3);
711 xopen("/dev/null", O_RDWR);
712 memset(&argv, 0, sizeof(argv));
713 argv[0] = (char*)"openssl";
714 argv[1] = (char*)"s_client";
715 argv[2] = (char*)"-quiet";
716 argv[3] = (char*)"-connect";
717 argv[4] = (char*)host;
718
719
720
721
722
723 argp = &argv[5];
724 if (!is_ip_address(servername)) {
725 *argp++ = (char*)"-servername";
726 *argp++ = (char*)servername;
727 }
728 if (!(option_mask32 & WGET_OPT_NO_CHECK_CERT)) {
729
730 *argp++ = (char*)"-verify";
731 *argp++ = (char*)"100";
732 *argp++ = (char*)"-verify_return_error";
733 if (!is_ip_address(servername)) {
734 *argp++ = (char*)"-verify_hostname";
735 *argp++ = (char*)servername;
736 } else {
737 *argp++ = (char*)"-verify_ip";
738 *argp++ = (char*)host;
739 }
740 }
741
742
743 BB_EXECVP(argv[0], argv);
744 xmove_fd(3, 2);
745# if ENABLE_FEATURE_WGET_HTTPS
746 child_failed = 1;
747 xfunc_die();
748# else
749 bb_perror_msg_and_die("can't execute '%s'", argv[0]);
750# endif
751
752 }
753
754
755 free(servername);
756 free(allocated);
757 close(sp[1]);
758# if ENABLE_FEATURE_WGET_HTTPS
759 if (child_failed) {
760 close(sp[0]);
761 return -1;
762 }
763# endif
764 return sp[0];
765}
766#endif
767
768#if ENABLE_FEATURE_WGET_HTTPS
769static void spawn_ssl_client(const char *host, int network_fd, int flags)
770{
771 int sp[2];
772 int pid;
773 char *servername, *p;
774
775 if (!(option_mask32 & WGET_OPT_NO_CHECK_CERT)) {
776 option_mask32 |= WGET_OPT_NO_CHECK_CERT;
777 bb_simple_error_msg("note: TLS certificate validation not implemented");
778 }
779
780 servername = xstrdup(host);
781 p = strrchr(servername, ':');
782 if (p) *p = '\0';
783
784 if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) != 0)
785
786 bb_simple_perror_msg_and_die("socketpair");
787
788 fflush_all();
789 pid = BB_MMU ? xfork() : xvfork();
790 if (pid == 0) {
791
792 close(sp[0]);
793 xmove_fd(sp[1], 0);
794 xdup2(0, 1);
795 if (BB_MMU) {
796 tls_state_t *tls = new_tls_state();
797 tls->ifd = tls->ofd = network_fd;
798 tls_handshake(tls, servername);
799 tls_run_copy_loop(tls, flags);
800 exit(0);
801 } else {
802 char *argv[6];
803
804 xmove_fd(network_fd, 3);
805 argv[0] = (char*)"ssl_client";
806 argv[1] = (char*)"-s3";
807
808 argv[2] = (char*)"-n";
809 argv[3] = servername;
810 argv[4] = (flags & TLSLOOP_EXIT_ON_LOCAL_EOF ? (char*)"-e" : NULL);
811 argv[5] = NULL;
812 BB_EXECVP(argv[0], argv);
813 bb_perror_msg_and_die("can't execute '%s'", argv[0]);
814 }
815
816 }
817
818
819 free(servername);
820 close(sp[1]);
821 xmove_fd(sp[0], network_fd);
822}
823#endif
824
825#if ENABLE_FEATURE_WGET_FTP
826static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_sockaddr *lsa)
827{
828 FILE *sfp;
829 char *pass;
830 int port;
831
832 sfp = open_socket(lsa);
833#if FTPS_SUPPORTED
834 if (target->protocol == P_FTPS)
835 spawn_ssl_client(target->host, fileno(sfp), TLSLOOP_EXIT_ON_LOCAL_EOF);
836#endif
837
838 if (ftpcmd(NULL, NULL, sfp) != 220)
839 bb_simple_error_msg_and_die(G.wget_buf);
840
841
842
843 pass = (char*)"busybox";
844 if (target->user) {
845 pass = strchr(target->user, ':');
846 if (pass)
847 *pass++ = '\0';
848 }
849
850
851 switch (ftpcmd("USER ", target->user ?: "anonymous", sfp)) {
852 case 230:
853 break;
854 case 331:
855 if (ftpcmd("PASS ", pass, sfp) == 230)
856 break;
857
858 default:
859 bb_error_msg_and_die("ftp login: %s", G.wget_buf);
860 }
861
862 ftpcmd("TYPE I", NULL, sfp);
863
864
865 if (ftpcmd("SIZE ", target->path, sfp) == 213) {
866 G.content_len = BB_STRTOOFF(G.wget_buf + 4, NULL, 10);
867 if (G.content_len < 0 || errno) {
868 bb_error_msg_and_die("bad SIZE value '%s'", G.wget_buf + 4);
869 }
870 G.got_clen = 1;
871 }
872
873
874 if (ENABLE_FEATURE_IPV6 && ftpcmd("EPSV", NULL, sfp) == 229) {
875
876 } else
877 if (ftpcmd("PASV", NULL, sfp) != 227) {
878 pasv_error:
879 bb_error_msg_and_die("bad response to %s: %s", "PASV", G.wget_buf);
880 }
881 port = parse_pasv_epsv(G.wget_buf);
882 if (port < 0)
883 goto pasv_error;
884
885 set_nport(&lsa->u.sa, htons(port));
886
887 *dfpp = open_socket(lsa);
888
889#if FTPS_SUPPORTED
890 if (target->protocol == P_FTPS) {
891
892
893
894 if (ftpcmd("PROT P", NULL, sfp) == 200)
895 spawn_ssl_client(target->host, fileno(*dfpp), 0);
896 }
897#endif
898
899 if (G.beg_range != 0) {
900 sprintf(G.wget_buf, "REST %"OFF_FMT"u", G.beg_range);
901 if (ftpcmd(G.wget_buf, NULL, sfp) == 350)
902 G.content_len -= G.beg_range;
903 else
904 reset_beg_range_to_zero();
905 }
906
907
908
909
910 if (ftpcmd("RETR ", target->path, sfp) > 150)
911 bb_error_msg_and_die("bad response to %s: %s", "RETR", G.wget_buf);
912
913 return sfp;
914}
915#endif
916
917static void NOINLINE retrieve_file_data(FILE *dfp)
918{
919#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
920# if ENABLE_FEATURE_WGET_TIMEOUT
921 unsigned second_cnt = G.timeout_seconds;
922# endif
923 struct pollfd polldata;
924
925 polldata.fd = fileno(dfp);
926 polldata.events = POLLIN | POLLPRI;
927#endif
928 if (!(option_mask32 & WGET_OPT_QUIET)) {
929 if (G.output_fd == 1)
930 fprintf(stderr, "writing to stdout\n");
931 else
932 fprintf(stderr, "saving to '%s'\n", G.fname_out);
933 }
934 progress_meter(PROGRESS_START);
935
936 if (G.chunked)
937 goto get_clen;
938
939
940 while (1) {
941
942#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
943
944
945
946
947
948
949
950 ndelay_on(polldata.fd);
951#endif
952 while (1) {
953 int n;
954 unsigned rdsz;
955
956#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
957
958
959
960
961
962
963
964
965 clearerr(dfp);
966#endif
967 errno = 0;
968 rdsz = sizeof(G.wget_buf);
969 if (G.got_clen) {
970 if (G.content_len < (off_t)sizeof(G.wget_buf)) {
971 if ((int)G.content_len <= 0)
972 break;
973 rdsz = (unsigned)G.content_len;
974 }
975 }
976 n = fread(G.wget_buf, 1, rdsz, dfp);
977
978 if (n > 0) {
979 xwrite(G.output_fd, G.wget_buf, n);
980#if ENABLE_FEATURE_WGET_STATUSBAR
981 G.transferred += n;
982#endif
983 if (G.got_clen) {
984 G.content_len -= n;
985 if (G.content_len == 0)
986 break;
987 }
988#if ENABLE_FEATURE_WGET_TIMEOUT
989 second_cnt = G.timeout_seconds;
990#endif
991 goto bump;
992 }
993
994
995
996
997
998
999
1000 if (errno != EAGAIN) {
1001 if (ferror(dfp)) {
1002 progress_meter(PROGRESS_END);
1003 bb_simple_perror_msg_and_die(bb_msg_read_error);
1004 }
1005 break;
1006 }
1007
1008#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
1009
1010
1011
1012 if (safe_poll(&polldata, 1, 1000) == 0) {
1013# if ENABLE_FEATURE_WGET_TIMEOUT
1014 if (second_cnt != 0 && --second_cnt == 0) {
1015 progress_meter(PROGRESS_END);
1016 bb_simple_error_msg_and_die("download timed out");
1017 }
1018# endif
1019
1020
1021
1022
1023 }
1024#endif
1025 bump:
1026
1027
1028
1029 progress_meter(PROGRESS_BUMP);
1030 }
1031
1032#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
1033 clearerr(dfp);
1034 ndelay_off(polldata.fd);
1035#endif
1036 if (!G.chunked)
1037 break;
1038
1039
1040 fgets_trim_sanitize(dfp, NULL);
1041 get_clen:
1042
1043 fgets_trim_sanitize(dfp, NULL);
1044 errno = 0;
1045 G.content_len = STRTOOFF(G.wget_buf, NULL, 16);
1046
1047
1048
1049
1050 if (G.content_len < 0 || errno)
1051 bb_error_msg_and_die("bad chunk length '%s'", G.wget_buf);
1052 if (G.content_len == 0)
1053 break;
1054 G.got_clen = 1;
1055
1056
1057
1058
1059
1060
1061
1062 }
1063
1064
1065 G.chunked = 0;
1066 G.got_clen = 1;
1067 progress_meter(PROGRESS_END);
1068 if (G.content_len != 0) {
1069 bb_simple_perror_msg_and_die("connection closed prematurely");
1070
1071 }
1072
1073
1074
1075
1076
1077
1078 {
1079 off_t pos = lseek(G.output_fd, 0, SEEK_CUR);
1080 if (pos != (off_t)-1) {
1081
1082
1083
1084 if (!(option_mask32 & WGET_NO_FTRUNCATE))
1085 ftruncate(G.output_fd, pos);
1086 }
1087 }
1088
1089 if (!(option_mask32 & WGET_OPT_QUIET)) {
1090 if (G.output_fd == 1)
1091 fprintf(stderr, "written to stdout\n");
1092 else
1093 fprintf(stderr, "'%s' saved\n", G.fname_out);
1094 }
1095}
1096
1097static void download_one_url(const char *url)
1098{
1099 bool use_proxy;
1100 int redir_limit;
1101 len_and_sockaddr *lsa;
1102 FILE *sfp;
1103 FILE *dfp;
1104 char *fname_out_alloc;
1105 char *redirected_path = NULL;
1106 struct host_info server;
1107 struct host_info target;
1108
1109 server.allocated = NULL;
1110 target.allocated = NULL;
1111 server.user = NULL;
1112 target.user = NULL;
1113
1114 parse_url(url, &target);
1115
1116
1117 use_proxy = (strcmp(G.proxy_flag, "off") != 0);
1118 if (use_proxy) {
1119 char *proxy = getenv(target.protocol[0] == 'f' ? "ftp_proxy" : "http_proxy");
1120
1121 use_proxy = (proxy && proxy[0]);
1122 if (use_proxy)
1123 parse_url(proxy, &server);
1124 }
1125 if (!use_proxy) {
1126 server.protocol = target.protocol;
1127 server.port = target.port;
1128 if (ENABLE_FEATURE_IPV6) {
1129
1130 server.host = server.allocated = xstrdup(target.host);
1131 } else {
1132 server.host = target.host;
1133 }
1134 }
1135
1136 if (ENABLE_FEATURE_IPV6)
1137 strip_ipv6_scope_id(target.host);
1138
1139
1140 fname_out_alloc = NULL;
1141 if (!(option_mask32 & WGET_OPT_OUTNAME)) {
1142 G.fname_out = bb_get_last_path_component_nostrip(target.path);
1143
1144 if (G.fname_out[0] == '/' || !G.fname_out[0])
1145 G.fname_out = (char*)"index.html";
1146
1147 if (G.dir_prefix)
1148 G.fname_out = fname_out_alloc = concat_path_file(G.dir_prefix, G.fname_out);
1149 else {
1150
1151 G.fname_out = fname_out_alloc = xstrdup(G.fname_out);
1152 }
1153 }
1154#if ENABLE_FEATURE_WGET_STATUSBAR
1155 G.curfile = bb_get_last_path_component_nostrip(G.fname_out);
1156#endif
1157
1158
1159 G.beg_range = 0;
1160 if (option_mask32 & WGET_OPT_CONTINUE) {
1161 G.output_fd = open(G.fname_out, O_WRONLY);
1162 if (G.output_fd >= 0) {
1163 G.beg_range = xlseek(G.output_fd, 0, SEEK_END);
1164 }
1165
1166
1167 }
1168
1169 redir_limit = 16;
1170 resolve_lsa:
1171 lsa = xhost2sockaddr(server.host, server.port);
1172 if (!(option_mask32 & WGET_OPT_QUIET)) {
1173 char *s = xmalloc_sockaddr2dotted(&lsa->u.sa);
1174 fprintf(stderr, "Connecting to %s (%s)\n", server.host, s);
1175 free(s);
1176 }
1177 establish_session:
1178
1179 G.got_clen = 0;
1180 G.chunked = 0;
1181 if (use_proxy || target.protocol[0] != 'f' ) {
1182
1183
1184
1185 char *str;
1186 int status;
1187
1188
1189#if ENABLE_FEATURE_WGET_OPENSSL
1190
1191 if (server.protocol == P_HTTPS) {
1192
1193
1194
1195 int fd = spawn_https_helper_openssl(server.host, server.port);
1196# if ENABLE_FEATURE_WGET_HTTPS
1197 if (fd < 0) {
1198 sfp = open_socket(lsa);
1199 spawn_ssl_client(server.host, fileno(sfp), 0);
1200 goto socket_opened;
1201 }
1202# else
1203
1204# endif
1205 sfp = fdopen(fd, "r+");
1206 if (!sfp)
1207 bb_die_memory_exhausted();
1208 goto socket_opened;
1209 }
1210 sfp = open_socket(lsa);
1211 socket_opened:
1212#elif ENABLE_FEATURE_WGET_HTTPS
1213
1214 sfp = open_socket(lsa);
1215 if (server.protocol == P_HTTPS)
1216 spawn_ssl_client(server.host, fileno(sfp), 0);
1217#else
1218
1219 sfp = open_socket(lsa);
1220#endif
1221
1222 if (use_proxy) {
1223 SENDFMT(sfp, "GET %s://%s/%s HTTP/1.1\r\n",
1224 target.protocol, target.host,
1225 target.path);
1226 } else {
1227 SENDFMT(sfp, "%s /%s HTTP/1.1\r\n",
1228 (option_mask32 & WGET_OPT_POST) ? "POST" : "GET",
1229 target.path);
1230 }
1231 if (!USR_HEADER_HOST)
1232 SENDFMT(sfp, "Host: %s\r\n", target.host);
1233 if (!USR_HEADER_USER_AGENT)
1234 SENDFMT(sfp, "User-Agent: %s\r\n", G.user_agent);
1235
1236
1237
1238
1239 SENDFMT(sfp, "Connection: close\r\n");
1240
1241#if ENABLE_FEATURE_WGET_AUTHENTICATION
1242 if (target.user && !USR_HEADER_AUTH) {
1243 SENDFMT(sfp, "Proxy-Authorization: Basic %s\r\n"+6,
1244 base64enc(target.user));
1245 }
1246 if (use_proxy && server.user && !USR_HEADER_PROXY_AUTH) {
1247 SENDFMT(sfp, "Proxy-Authorization: Basic %s\r\n",
1248 base64enc(server.user));
1249 }
1250#endif
1251
1252 if (G.beg_range != 0 && !USR_HEADER_RANGE)
1253 SENDFMT(sfp, "Range: bytes=%"OFF_FMT"u-\r\n", G.beg_range);
1254
1255#if ENABLE_FEATURE_WGET_LONG_OPTIONS
1256 if (G.extra_headers) {
1257 log_io(G.extra_headers);
1258 fputs(G.extra_headers, sfp);
1259 }
1260
1261 if (option_mask32 & WGET_OPT_POST_FILE) {
1262 int fd = xopen_stdin(G.post_file);
1263 G.post_data = xmalloc_read(fd, NULL);
1264 close(fd);
1265 }
1266
1267 if (G.post_data) {
1268
1269 if (!USR_HEADER_CONTENT_TYPE) {
1270 SENDFMT(sfp,
1271 "Content-Type: application/x-www-form-urlencoded\r\n"
1272 );
1273 }
1274 SENDFMT(sfp,
1275 "Content-Length: %u\r\n"
1276 "\r\n"
1277 "%s",
1278 (int) strlen(G.post_data), G.post_data
1279 );
1280 } else
1281#endif
1282 {
1283 SENDFMT(sfp, "\r\n");
1284 }
1285
1286 fflush(sfp);
1287
1288
1289
1290
1291#if SSL_SUPPORTED
1292 if (target.protocol == P_HTTPS) {
1293
1294
1295
1296
1297
1298 shutdown(fileno(sfp), SHUT_WR);
1299 }
1300#endif
1301
1302
1303
1304
1305 read_response:
1306 fgets_trim_sanitize(sfp, " %s\n");
1307
1308 str = G.wget_buf;
1309 str = skip_non_whitespace(str);
1310 str = skip_whitespace(str);
1311
1312
1313 status = atoi(str);
1314 switch (status) {
1315 case 0:
1316 case 100:
1317 while (get_sanitized_hdr(sfp) != NULL)
1318 ;
1319 goto read_response;
1320
1321
1322 case 200:
1323
1324 case 201:
1325
1326
1327
1328 case 202:
1329
1330
1331 case 203:
1332
1333
1334 case 204:
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359 if (G.beg_range != 0) {
1360
1361
1362
1363 reset_beg_range_to_zero();
1364 }
1365 break;
1366
1367
1368 case 300:
1369 case 301:
1370 case 302:
1371 case 303:
1372 case 307:
1373 case 308:
1374 break;
1375
1376 case 206:
1377 if (G.beg_range != 0)
1378
1379 break;
1380
1381
1382 default:
1383 bb_error_msg_and_die("server returned error: %s", G.wget_buf);
1384 }
1385
1386
1387
1388
1389 while ((str = get_sanitized_hdr(sfp)) != NULL) {
1390 static const char keywords[] ALIGN1 =
1391 "content-length\0""transfer-encoding\0""location\0";
1392 enum {
1393 KEY_content_length = 1, KEY_transfer_encoding, KEY_location
1394 };
1395 smalluint key;
1396
1397
1398
1399
1400 char *s = strchrnul(str, '\0') - 1;
1401 while (s >= str && (*s == ' ' || *s == '\t')) {
1402 *s = '\0';
1403 s--;
1404 }
1405 key = index_in_strings(keywords, G.wget_buf) + 1;
1406 if (key == KEY_content_length) {
1407 G.content_len = BB_STRTOOFF(str, NULL, 10);
1408 if (G.content_len < 0 || errno) {
1409 bb_error_msg_and_die("content-length %s is garbage", str);
1410 }
1411 G.got_clen = 1;
1412 continue;
1413 }
1414 if (key == KEY_transfer_encoding) {
1415 if (strcmp(str_tolower(str), "chunked") != 0)
1416 bb_error_msg_and_die("transfer encoding '%s' is not supported", str);
1417 G.chunked = 1;
1418 }
1419 if (key == KEY_location && status >= 300) {
1420 if (--redir_limit == 0)
1421 bb_simple_error_msg_and_die("too many redirections");
1422 fclose(sfp);
1423 if (str[0] == '/') {
1424 free(redirected_path);
1425 target.path = redirected_path = xstrdup(str + 1);
1426
1427 } else {
1428 parse_url(str, &target);
1429 if (!use_proxy) {
1430
1431 free(server.allocated);
1432 server.allocated = NULL;
1433 server.protocol = target.protocol;
1434 server.host = target.host;
1435
1436
1437 server.port = target.port;
1438 free(lsa);
1439 goto resolve_lsa;
1440 }
1441 }
1442 goto establish_session;
1443 }
1444 }
1445
1446
1447
1448
1449 dfp = sfp;
1450 } else {
1451#if ENABLE_FEATURE_WGET_FTP
1452
1453
1454
1455 sfp = prepare_ftp_session(&dfp, &target, lsa);
1456#endif
1457 }
1458
1459 free(lsa);
1460
1461 if (!(option_mask32 & WGET_OPT_SPIDER)) {
1462 if (G.output_fd < 0)
1463 G.output_fd = xopen(G.fname_out, G.o_flags);
1464 retrieve_file_data(dfp);
1465 if (!(option_mask32 & WGET_OPT_OUTNAME)) {
1466 xclose(G.output_fd);
1467 G.output_fd = -1;
1468 }
1469 } else {
1470 if (!(option_mask32 & WGET_OPT_QUIET))
1471 fprintf(stderr, "remote file exists\n");
1472 }
1473
1474#if ENABLE_FEATURE_WGET_FTP
1475 if (dfp != sfp) {
1476
1477 fclose(dfp);
1478 if (ftpcmd(NULL, NULL, sfp) != 226)
1479 bb_error_msg_and_die("ftp error: %s", G.wget_buf);
1480
1481 }
1482#endif
1483 fclose(sfp);
1484
1485 free(server.allocated);
1486 free(target.allocated);
1487 free(server.user);
1488 free(target.user);
1489 free(fname_out_alloc);
1490 free(redirected_path);
1491}
1492
1493int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
1494int wget_main(int argc UNUSED_PARAM, char **argv)
1495{
1496#if ENABLE_FEATURE_WGET_LONG_OPTIONS
1497 static const char wget_longopts[] ALIGN1 =
1498
1499 "continue\0" No_argument "c"
1500 "quiet\0" No_argument "q"
1501 "server-response\0" No_argument "S"
1502 "output-document\0" Required_argument "O"
1503 "output-file\0" Required_argument "o"
1504 "directory-prefix\0" Required_argument "P"
1505 "proxy\0" Required_argument "Y"
1506 "user-agent\0" Required_argument "U"
1507IF_FEATURE_WGET_TIMEOUT(
1508 "timeout\0" Required_argument "T")
1509
1510IF_DESKTOP( "tries\0" Required_argument "t")
1511 "header\0" Required_argument "\xff"
1512 "post-data\0" Required_argument "\xfe"
1513 "spider\0" No_argument "\xfd"
1514 "no-check-certificate\0" No_argument "\xfc"
1515 "post-file\0" Required_argument "\xfb"
1516
1517IF_DESKTOP( "passive-ftp\0" No_argument "\xf0")
1518
1519IF_DESKTOP( "no-cache\0" No_argument "\xf0")
1520IF_DESKTOP( "no-verbose\0" No_argument "\xf0")
1521IF_DESKTOP( "no-clobber\0" No_argument "\xf0")
1522IF_DESKTOP( "no-host-directories\0" No_argument "\xf0")
1523IF_DESKTOP( "no-parent\0" No_argument "\xf0")
1524 ;
1525# define GETOPT32 getopt32long
1526# define LONGOPTS ,wget_longopts
1527#else
1528# define GETOPT32 getopt32
1529# define LONGOPTS
1530#endif
1531
1532#if ENABLE_FEATURE_WGET_LONG_OPTIONS
1533 llist_t *headers_llist = NULL;
1534#endif
1535
1536 INIT_G();
1537
1538#if ENABLE_FEATURE_WGET_TIMEOUT
1539 G.timeout_seconds = 900;
1540 signal(SIGALRM, alarm_handler);
1541#endif
1542 G.proxy_flag = "on";
1543 G.user_agent = "Wget";
1544
1545 GETOPT32(argv, "^"
1546 "cqSO:o:P:Y:U:T:+"
1547 "t:"
1548 "n::"
1549
1550
1551
1552
1553
1554
1555
1556
1557 "\0"
1558 "-1"
1559 IF_FEATURE_WGET_LONG_OPTIONS(":\xfe--\xfb")
1560 IF_FEATURE_WGET_LONG_OPTIONS(":\xfe--\xfe")
1561 IF_FEATURE_WGET_LONG_OPTIONS(":\xfb--\xfb")
1562 IF_FEATURE_WGET_LONG_OPTIONS(":\xff::")
1563 LONGOPTS
1564 , &G.fname_out, &G.fname_log, &G.dir_prefix,
1565 &G.proxy_flag, &G.user_agent,
1566 IF_FEATURE_WGET_TIMEOUT(&G.timeout_seconds) IF_NOT_FEATURE_WGET_TIMEOUT(NULL),
1567 NULL,
1568 NULL
1569 IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist)
1570 IF_FEATURE_WGET_LONG_OPTIONS(, &G.post_data)
1571 IF_FEATURE_WGET_LONG_OPTIONS(, &G.post_file)
1572 );
1573#if 0
1574 if (option_mask32 & WGET_OPT_RETRIES) bb_error_msg("-t NUM");
1575 if (option_mask32 & WGET_OPT_nsomething) bb_error_msg("-nsomething");
1576 if (option_mask32 & WGET_OPT_HEADER) bb_error_msg("--header");
1577 if (option_mask32 & WGET_OPT_POST_DATA) bb_error_msg("--post-data");
1578 if (option_mask32 & WGET_OPT_SPIDER) bb_error_msg("--spider");
1579 if (option_mask32 & WGET_OPT_NO_CHECK_CERT) bb_error_msg("--no-check-certificate");
1580 if (option_mask32 & WGET_OPT_POST_FILE) bb_error_msg("--post-file");
1581 exit(0);
1582#endif
1583 argv += optind;
1584
1585#if ENABLE_FEATURE_WGET_LONG_OPTIONS
1586 if (headers_llist) {
1587 int size = 0;
1588 char *hdr;
1589 llist_t *ll = headers_llist;
1590 while (ll) {
1591 size += strlen(ll->data) + 2;
1592 ll = ll->link;
1593 }
1594 G.extra_headers = hdr = xmalloc(size + 1);
1595 while (headers_llist) {
1596 int bit;
1597 const char *words;
1598
1599 size = sprintf(hdr, "%s\r\n",
1600 (char*)llist_pop(&headers_llist));
1601
1602 bit = 1;
1603 words = wget_user_headers;
1604 while (*words) {
1605 if (strstr(hdr, words) == hdr) {
1606 G.user_headers |= bit;
1607 break;
1608 }
1609 bit <<= 1;
1610 words += strlen(words) + 1;
1611 }
1612 hdr += size;
1613 }
1614 }
1615#endif
1616
1617 G.output_fd = -1;
1618 G.o_flags = O_WRONLY | O_CREAT | O_TRUNC | O_EXCL;
1619 if (G.fname_out) {
1620 if (LONE_DASH(G.fname_out)) {
1621 G.output_fd = 1;
1622 option_mask32 = (option_mask32 & (~WGET_OPT_CONTINUE)) | WGET_NO_FTRUNCATE;
1623 }
1624
1625 G.o_flags = O_WRONLY | O_CREAT | O_TRUNC;
1626 }
1627
1628 G.log_fd = -1;
1629 if (G.fname_log) {
1630 if (!LONE_DASH(G.fname_log)) {
1631
1632 G.log_fd = xopen(G.fname_log, O_WRONLY | O_CREAT | O_TRUNC);
1633
1634 xdup2(G.log_fd, STDERR_FILENO);
1635 }
1636 }
1637
1638 while (*argv)
1639 download_one_url(*argv++);
1640
1641 if (G.output_fd >= 0)
1642 xclose(G.output_fd);
1643
1644 if (G.log_fd >= 0)
1645 xclose(G.log_fd);
1646
1647#if ENABLE_FEATURE_CLEAN_UP && ENABLE_FEATURE_WGET_LONG_OPTIONS
1648 free(G.extra_headers);
1649#endif
1650 FINI_G();
1651
1652 return EXIT_SUCCESS;
1653}
1654