1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153#include "libbb.h"
154
155#if 0
156# define log_io(...) bb_error_msg(__VA_ARGS__)
157# define SENDFMT(fp, fmt, ...) \
158 do { \
159 log_io("> " fmt, ##__VA_ARGS__); \
160 fprintf(fp, fmt, ##__VA_ARGS__); \
161 } while (0);
162#else
163# define log_io(...) ((void)0)
164# define SENDFMT(fp, fmt, ...) fprintf(fp, fmt, ##__VA_ARGS__)
165#endif
166
167
168#define SSL_SUPPORTED (ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_HTTPS)
169
170struct host_info {
171 char *allocated;
172 const char *path;
173 char *user;
174 const char *protocol;
175 char *host;
176 int port;
177};
178static const char P_FTP[] ALIGN1 = "ftp";
179static const char P_HTTP[] ALIGN1 = "http";
180#if SSL_SUPPORTED
181# if ENABLE_FEATURE_WGET_HTTPS
182static const char P_FTPS[] ALIGN1 = "ftps";
183# endif
184static const char P_HTTPS[] ALIGN1 = "https";
185#endif
186
187#if ENABLE_FEATURE_WGET_LONG_OPTIONS
188
189enum {
190 HDR_HOST = (1<<0),
191 HDR_USER_AGENT = (1<<1),
192 HDR_RANGE = (1<<2),
193 HDR_AUTH = (1<<3) * ENABLE_FEATURE_WGET_AUTHENTICATION,
194 HDR_PROXY_AUTH = (1<<4) * ENABLE_FEATURE_WGET_AUTHENTICATION,
195};
196static const char wget_user_headers[] ALIGN1 =
197 "Host:\0"
198 "User-Agent:\0"
199 "Range:\0"
200# if ENABLE_FEATURE_WGET_AUTHENTICATION
201 "Authorization:\0"
202 "Proxy-Authorization:\0"
203# endif
204 ;
205# define USR_HEADER_HOST (G.user_headers & HDR_HOST)
206# define USR_HEADER_USER_AGENT (G.user_headers & HDR_USER_AGENT)
207# define USR_HEADER_RANGE (G.user_headers & HDR_RANGE)
208# define USR_HEADER_AUTH (G.user_headers & HDR_AUTH)
209# define USR_HEADER_PROXY_AUTH (G.user_headers & HDR_PROXY_AUTH)
210#else
211# define USR_HEADER_HOST 0
212# define USR_HEADER_USER_AGENT 0
213# define USR_HEADER_RANGE 0
214# define USR_HEADER_AUTH 0
215# define USR_HEADER_PROXY_AUTH 0
216#endif
217
218
219struct globals {
220 off_t content_len;
221 off_t beg_range;
222#if ENABLE_FEATURE_WGET_STATUSBAR
223 off_t transferred;
224 const char *curfile;
225 bb_progress_t pmt;
226#endif
227 char *dir_prefix;
228#if ENABLE_FEATURE_WGET_LONG_OPTIONS
229 char *post_data;
230 char *extra_headers;
231 unsigned char user_headers;
232#endif
233 char *fname_out;
234 const char *proxy_flag;
235 const char *user_agent;
236 int output_fd;
237 int o_flags;
238#if ENABLE_FEATURE_WGET_TIMEOUT
239 unsigned timeout_seconds;
240 smallint die_if_timed_out;
241#endif
242 smallint chunked;
243 smallint got_clen;
244
245
246
247
248 char wget_buf[CONFIG_FEATURE_COPYBUF_KB*1024] ALIGNED(sizeof(long));
249} FIX_ALIASING;
250#define G (*ptr_to_globals)
251#define INIT_G() do { \
252 SET_PTR_TO_GLOBALS(xzalloc(sizeof(G))); \
253} while (0)
254#define FINI_G() do { \
255 FREE_PTR_TO_GLOBALS(); \
256} while (0)
257
258
259
260enum {
261 WGET_OPT_CONTINUE = (1 << 0),
262 WGET_OPT_QUIET = (1 << 1),
263 WGET_OPT_SERVER_RESPONSE = (1 << 2),
264 WGET_OPT_OUTNAME = (1 << 3),
265 WGET_OPT_PREFIX = (1 << 4),
266 WGET_OPT_PROXY = (1 << 5),
267 WGET_OPT_USER_AGENT = (1 << 6),
268 WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 7),
269 WGET_OPT_RETRIES = (1 << 8),
270 WGET_OPT_nsomething = (1 << 9),
271 WGET_OPT_HEADER = (1 << 10) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
272 WGET_OPT_POST_DATA = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
273 WGET_OPT_SPIDER = (1 << 12) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
274 WGET_OPT_NO_CHECK_CERT = (1 << 13) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
275};
276
277enum {
278 PROGRESS_START = -1,
279 PROGRESS_END = 0,
280 PROGRESS_BUMP = 1,
281};
282#if ENABLE_FEATURE_WGET_STATUSBAR
283static void progress_meter(int flag)
284{
285 int notty;
286
287 if (option_mask32 & WGET_OPT_QUIET)
288 return;
289
290 if (flag == PROGRESS_START)
291 bb_progress_init(&G.pmt, G.curfile);
292
293 notty = bb_progress_update(&G.pmt,
294 G.beg_range,
295 G.transferred,
296 (G.chunked || !G.got_clen) ? 0 : G.beg_range + G.transferred + G.content_len
297 );
298
299 if (flag == PROGRESS_END) {
300 bb_progress_free(&G.pmt);
301 if (notty == 0)
302 bb_putchar_stderr('\n');
303 G.transferred = 0;
304 }
305}
306#else
307static ALWAYS_INLINE void progress_meter(int flag UNUSED_PARAM) { }
308#endif
309
310
311
312
313
314
315
316
317
318
319
320
321static void strip_ipv6_scope_id(char *host)
322{
323 char *scope, *cp;
324
325
326
327
328
329 if (host[0] != '[')
330 return;
331
332 scope = strchr(host, '%');
333 if (!scope)
334 return;
335
336
337 cp = strchr(host, ']');
338 if (!cp || (cp[1] != ':' && cp[1] != '\0')) {
339
340 return;
341 }
342
343
344 overlapping_strcpy(scope, cp);
345}
346
347#if ENABLE_FEATURE_WGET_AUTHENTICATION
348
349static char *base64enc(const char *str)
350{
351
352 unsigned len = strnlen(str, sizeof(G.wget_buf)/4*3 - 10);
353 bb_uuencode(G.wget_buf, str, len, bb_uuenc_tbl_base64);
354 return G.wget_buf;
355}
356#endif
357
358#if ENABLE_FEATURE_WGET_TIMEOUT
359static void alarm_handler(int sig UNUSED_PARAM)
360{
361
362 if (G.die_if_timed_out)
363 bb_error_msg_and_die("download timed out");
364}
365static void set_alarm(void)
366{
367 if (G.timeout_seconds) {
368 alarm(G.timeout_seconds);
369 G.die_if_timed_out = 1;
370 }
371}
372# define clear_alarm() ((void)(G.die_if_timed_out = 0))
373#else
374# define set_alarm() ((void)0)
375# define clear_alarm() ((void)0)
376#endif
377
378#if ENABLE_FEATURE_WGET_OPENSSL
379
380
381
382
383
384
385
386
387static int is_ip_address(const char *string)
388{
389 struct sockaddr_in sa;
390
391 int result = inet_pton(AF_INET, string, &(sa.sin_addr));
392# if ENABLE_FEATURE_IPV6
393 if (result == 0) {
394 struct sockaddr_in6 sa6;
395 result = inet_pton(AF_INET6, string, &(sa6.sin6_addr));
396 }
397# endif
398 return (result == 1);
399}
400#endif
401
402static FILE *open_socket(len_and_sockaddr *lsa)
403{
404 int fd;
405 FILE *fp;
406
407 set_alarm();
408 fd = xconnect_stream(lsa);
409 clear_alarm();
410
411
412
413 fp = fdopen(fd, "r+");
414 if (!fp)
415 bb_die_memory_exhausted();
416
417 return fp;
418}
419
420
421
422
423
424
425
426
427static char* sanitize_string(char *s)
428{
429 unsigned char *p = (void *) s;
430 while (*p) {
431 if (*p < ' ') {
432 if (*p != '\t')
433 break;
434 *p = ' ';
435 }
436 p++;
437 }
438 *p = '\0';
439 return s;
440}
441
442
443static char fgets_trim_sanitize(FILE *fp, const char *fmt)
444{
445 char c;
446 char *buf_ptr;
447
448 set_alarm();
449 if (fgets(G.wget_buf, sizeof(G.wget_buf), fp) == NULL)
450 bb_perror_msg_and_die("error getting response");
451 clear_alarm();
452
453 buf_ptr = strchrnul(G.wget_buf, '\n');
454 c = *buf_ptr;
455#if 1
456
457 sanitize_string(G.wget_buf);
458#else
459 *buf_ptr = '\0';
460 buf_ptr = strchrnul(G.wget_buf, '\r');
461 *buf_ptr = '\0';
462#endif
463
464 log_io("< %s", G.wget_buf);
465
466 if (fmt && (option_mask32 & WGET_OPT_SERVER_RESPONSE))
467 fprintf(stderr, fmt, G.wget_buf);
468
469 return c;
470}
471
472static int ftpcmd(const char *s1, const char *s2, FILE *fp)
473{
474 int result;
475 if (s1) {
476 if (!s2)
477 s2 = "";
478 fprintf(fp, "%s%s\r\n", s1, s2);
479
480 if (option_mask32 & WGET_OPT_SERVER_RESPONSE)
481 fprintf(stderr, "--> %s%s\n\n", s1, s2);
482 fflush(fp);
483 log_io("> %s%s", s1, s2);
484 }
485
486
487 G.wget_buf[3] = 0;
488 do {
489 fgets_trim_sanitize(fp, "%s\n");
490 } while (!isdigit(G.wget_buf[0]) || G.wget_buf[3] != ' ');
491
492 G.wget_buf[3] = '\0';
493 result = xatoi_positive(G.wget_buf);
494 G.wget_buf[3] = ' ';
495 return result;
496}
497
498static void parse_url(const char *src_url, struct host_info *h)
499{
500 char *url, *p, *sp;
501
502 free(h->allocated);
503 h->allocated = url = xstrdup(src_url);
504
505 h->protocol = P_FTP;
506 p = strstr(url, "://");
507 if (p) {
508 *p = '\0';
509 h->host = p + 3;
510 if (strcmp(url, P_FTP) == 0) {
511 h->port = bb_lookup_std_port(P_FTP, "tcp", 21);
512 } else
513#if SSL_SUPPORTED
514# if ENABLE_FEATURE_WGET_HTTPS
515 if (strcmp(url, P_FTPS) == 0) {
516 h->port = bb_lookup_std_port(P_FTPS, "tcp", 990);
517 h->protocol = P_FTPS;
518 } else
519# endif
520 if (strcmp(url, P_HTTPS) == 0) {
521 h->port = bb_lookup_std_port(P_HTTPS, "tcp", 443);
522 h->protocol = P_HTTPS;
523 } else
524#endif
525 if (strcmp(url, P_HTTP) == 0) {
526 http:
527 h->port = bb_lookup_std_port(P_HTTP, "tcp", 80);
528 h->protocol = P_HTTP;
529 } else {
530 *p = ':';
531 bb_error_msg_and_die("not an http or ftp url: %s", url);
532 }
533 } else {
534
535 h->host = url;
536 goto http;
537 }
538
539
540
541
542
543
544
545
546
547
548
549
550
551 sp = strchr(h->host, '/');
552 p = strchr(h->host, '?'); if (!sp || (p && sp > p)) sp = p;
553 p = strchr(h->host, '#'); if (!sp || (p && sp > p)) sp = p;
554 if (!sp) {
555 h->path = "";
556 } else if (*sp == '/') {
557 *sp = '\0';
558 h->path = sp + 1;
559 } else {
560
561
562
563
564
565 *sp++ = '\0';
566 h->path = sp;
567 }
568
569 sp = strrchr(h->host, '@');
570 if (sp != NULL) {
571
572
573
574
575
576 *sp = '\0';
577 free(h->user);
578 h->user = xstrdup(percent_decode_in_place(h->host, 0));
579 h->host = sp + 1;
580 }
581
582
583
584}
585
586static char *get_sanitized_hdr(FILE *fp)
587{
588 char *s, *hdrval;
589 int c;
590
591
592 c = fgets_trim_sanitize(fp, " %s\n");
593
594
595 if (G.wget_buf[0] == '\0')
596 return NULL;
597
598
599 for (s = G.wget_buf; isalnum(*s) || *s == '-' || *s == '.' || *s == '_'; ++s) {
600
601
602
603
604
605
606
607
608
609 *s |= 0x20;
610 }
611
612
613 if (*s != ':')
614 bb_error_msg_and_die("bad header line: %s", G.wget_buf);
615
616
617 *s++ = '\0';
618 hdrval = skip_whitespace(s);
619
620 if (c != '\n') {
621
622 while (c = getc(fp), c != EOF && c != '\n')
623 continue;
624 }
625
626 return hdrval;
627}
628
629static void reset_beg_range_to_zero(void)
630{
631 bb_error_msg("restart failed");
632 G.beg_range = 0;
633 xlseek(G.output_fd, 0, SEEK_SET);
634
635
636}
637
638#if ENABLE_FEATURE_WGET_OPENSSL
639static int spawn_https_helper_openssl(const char *host, unsigned port)
640{
641 char *allocated = NULL;
642 char *servername;
643 int sp[2];
644 int pid;
645 IF_FEATURE_WGET_HTTPS(volatile int child_failed = 0;)
646
647 if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) != 0)
648
649 bb_perror_msg_and_die("socketpair");
650
651 if (!strchr(host, ':'))
652 host = allocated = xasprintf("%s:%u", host, port);
653 servername = xstrdup(host);
654 strrchr(servername, ':')[0] = '\0';
655
656 fflush_all();
657 pid = xvfork();
658 if (pid == 0) {
659
660 char *argv[8];
661
662 close(sp[0]);
663 xmove_fd(sp[1], 0);
664 xdup2(0, 1);
665
666
667
668
669
670 xmove_fd(2, 3);
671 xopen("/dev/null", O_RDWR);
672 memset(&argv, 0, sizeof(argv));
673 argv[0] = (char*)"openssl";
674 argv[1] = (char*)"s_client";
675 argv[2] = (char*)"-quiet";
676 argv[3] = (char*)"-connect";
677 argv[4] = (char*)host;
678
679
680
681
682
683 if (!is_ip_address(servername)) {
684 argv[5] = (char*)"-servername";
685 argv[6] = (char*)servername;
686 }
687
688 BB_EXECVP(argv[0], argv);
689 xmove_fd(3, 2);
690# if ENABLE_FEATURE_WGET_HTTPS
691 child_failed = 1;
692 xfunc_die();
693# else
694 bb_perror_msg_and_die("can't execute '%s'", argv[0]);
695# endif
696
697 }
698
699
700 free(servername);
701 free(allocated);
702 close(sp[1]);
703# if ENABLE_FEATURE_WGET_HTTPS
704 if (child_failed) {
705 close(sp[0]);
706 return -1;
707 }
708# endif
709 return sp[0];
710}
711#endif
712
713#if ENABLE_FEATURE_WGET_HTTPS
714static void spawn_ssl_client(const char *host, int network_fd, int flags)
715{
716 int sp[2];
717 int pid;
718 char *servername, *p;
719
720 if (!(option_mask32 & WGET_OPT_NO_CHECK_CERT)) {
721 option_mask32 |= WGET_OPT_NO_CHECK_CERT;
722 bb_error_msg("note: TLS certificate validation not implemented");
723 }
724
725 servername = xstrdup(host);
726 p = strrchr(servername, ':');
727 if (p) *p = '\0';
728
729 if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) != 0)
730
731 bb_perror_msg_and_die("socketpair");
732
733 fflush_all();
734 pid = BB_MMU ? xfork() : xvfork();
735 if (pid == 0) {
736
737 close(sp[0]);
738 xmove_fd(sp[1], 0);
739 xdup2(0, 1);
740 if (BB_MMU) {
741 tls_state_t *tls = new_tls_state();
742 tls->ifd = tls->ofd = network_fd;
743 tls_handshake(tls, servername);
744 tls_run_copy_loop(tls, flags);
745 exit(0);
746 } else {
747 char *argv[6];
748
749 xmove_fd(network_fd, 3);
750 argv[0] = (char*)"ssl_client";
751 argv[1] = (char*)"-s3";
752
753 argv[2] = (char*)"-n";
754 argv[3] = servername;
755 argv[4] = (flags & TLSLOOP_EXIT_ON_LOCAL_EOF ? (char*)"-e" : NULL);
756 argv[5] = NULL;
757 BB_EXECVP(argv[0], argv);
758 bb_perror_msg_and_die("can't execute '%s'", argv[0]);
759 }
760
761 }
762
763
764 free(servername);
765 close(sp[1]);
766 xmove_fd(sp[0], network_fd);
767}
768#endif
769
770static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_sockaddr *lsa)
771{
772 FILE *sfp;
773 char *pass;
774 int port;
775
776 sfp = open_socket(lsa);
777#if ENABLE_FEATURE_WGET_HTTPS
778 if (target->protocol == P_FTPS)
779 spawn_ssl_client(target->host, fileno(sfp), TLSLOOP_EXIT_ON_LOCAL_EOF);
780#endif
781
782 if (ftpcmd(NULL, NULL, sfp) != 220)
783 bb_error_msg_and_die("%s", G.wget_buf);
784
785
786
787 pass = (char*)"busybox";
788 if (target->user) {
789 pass = strchr(target->user, ':');
790 if (pass)
791 *pass++ = '\0';
792 }
793
794
795 switch (ftpcmd("USER ", target->user ?: "anonymous", sfp)) {
796 case 230:
797 break;
798 case 331:
799 if (ftpcmd("PASS ", pass, sfp) == 230)
800 break;
801
802 default:
803 bb_error_msg_and_die("ftp login: %s", G.wget_buf);
804 }
805
806 ftpcmd("TYPE I", NULL, sfp);
807
808
809 if (ftpcmd("SIZE ", target->path, sfp) == 213) {
810 G.content_len = BB_STRTOOFF(G.wget_buf + 4, NULL, 10);
811 if (G.content_len < 0 || errno) {
812 bb_error_msg_and_die("bad SIZE value '%s'", G.wget_buf + 4);
813 }
814 G.got_clen = 1;
815 }
816
817
818 if (ENABLE_FEATURE_IPV6 && ftpcmd("EPSV", NULL, sfp) == 229) {
819
820 } else
821 if (ftpcmd("PASV", NULL, sfp) != 227) {
822 pasv_error:
823 bb_error_msg_and_die("bad response to %s: %s", "PASV", G.wget_buf);
824 }
825 port = parse_pasv_epsv(G.wget_buf);
826 if (port < 0)
827 goto pasv_error;
828
829 set_nport(&lsa->u.sa, htons(port));
830
831 *dfpp = open_socket(lsa);
832
833#if ENABLE_FEATURE_WGET_HTTPS
834 if (target->protocol == P_FTPS) {
835
836
837
838 if (ftpcmd("PROT P", NULL, sfp) == 200)
839 spawn_ssl_client(target->host, fileno(*dfpp), 0);
840 }
841#endif
842
843 if (G.beg_range != 0) {
844 sprintf(G.wget_buf, "REST %"OFF_FMT"u", G.beg_range);
845 if (ftpcmd(G.wget_buf, NULL, sfp) == 350)
846 G.content_len -= G.beg_range;
847 else
848 reset_beg_range_to_zero();
849 }
850
851
852
853
854 if (ftpcmd("RETR ", target->path, sfp) > 150)
855 bb_error_msg_and_die("bad response to %s: %s", "RETR", G.wget_buf);
856
857 return sfp;
858}
859
860static void NOINLINE retrieve_file_data(FILE *dfp)
861{
862#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
863# if ENABLE_FEATURE_WGET_TIMEOUT
864 unsigned second_cnt = G.timeout_seconds;
865# endif
866 struct pollfd polldata;
867
868 polldata.fd = fileno(dfp);
869 polldata.events = POLLIN | POLLPRI;
870#endif
871 progress_meter(PROGRESS_START);
872
873 if (G.chunked)
874 goto get_clen;
875
876
877 while (1) {
878
879#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
880
881
882
883
884
885
886
887 ndelay_on(polldata.fd);
888#endif
889 while (1) {
890 int n;
891 unsigned rdsz;
892
893#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
894
895
896
897
898
899
900
901
902 clearerr(dfp);
903#endif
904 errno = 0;
905 rdsz = sizeof(G.wget_buf);
906 if (G.got_clen) {
907 if (G.content_len < (off_t)sizeof(G.wget_buf)) {
908 if ((int)G.content_len <= 0)
909 break;
910 rdsz = (unsigned)G.content_len;
911 }
912 }
913 n = fread(G.wget_buf, 1, rdsz, dfp);
914
915 if (n > 0) {
916 xwrite(G.output_fd, G.wget_buf, n);
917#if ENABLE_FEATURE_WGET_STATUSBAR
918 G.transferred += n;
919#endif
920 if (G.got_clen) {
921 G.content_len -= n;
922 if (G.content_len == 0)
923 break;
924 }
925#if ENABLE_FEATURE_WGET_TIMEOUT
926 second_cnt = G.timeout_seconds;
927#endif
928 goto bump;
929 }
930
931
932
933
934
935
936
937 if (errno != EAGAIN) {
938 if (ferror(dfp)) {
939 progress_meter(PROGRESS_END);
940 bb_perror_msg_and_die(bb_msg_read_error);
941 }
942 break;
943 }
944
945#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
946
947
948
949 if (safe_poll(&polldata, 1, 1000) == 0) {
950# if ENABLE_FEATURE_WGET_TIMEOUT
951 if (second_cnt != 0 && --second_cnt == 0) {
952 progress_meter(PROGRESS_END);
953 bb_error_msg_and_die("download timed out");
954 }
955# endif
956
957
958
959
960 }
961#endif
962 bump:
963
964
965
966 progress_meter(PROGRESS_BUMP);
967 }
968
969#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
970 clearerr(dfp);
971 ndelay_off(polldata.fd);
972#endif
973 if (!G.chunked)
974 break;
975
976
977 fgets_trim_sanitize(dfp, NULL);
978 get_clen:
979
980 fgets_trim_sanitize(dfp, NULL);
981 errno = 0;
982 G.content_len = STRTOOFF(G.wget_buf, NULL, 16);
983
984
985
986
987 if (G.content_len < 0 || errno)
988 bb_error_msg_and_die("bad chunk length '%s'", G.wget_buf);
989 if (G.content_len == 0)
990 break;
991 G.got_clen = 1;
992
993
994
995
996
997
998
999 }
1000
1001
1002
1003
1004
1005
1006 {
1007 off_t pos = lseek(G.output_fd, 0, SEEK_CUR);
1008 if (pos != (off_t)-1)
1009 ftruncate(G.output_fd, pos);
1010 }
1011
1012
1013 G.chunked = 0;
1014 G.got_clen = 1;
1015 progress_meter(PROGRESS_END);
1016}
1017
1018static void download_one_url(const char *url)
1019{
1020 bool use_proxy;
1021 int redir_limit;
1022 len_and_sockaddr *lsa;
1023 FILE *sfp;
1024 FILE *dfp;
1025 char *fname_out_alloc;
1026 char *redirected_path = NULL;
1027 struct host_info server;
1028 struct host_info target;
1029
1030 server.allocated = NULL;
1031 target.allocated = NULL;
1032 server.user = NULL;
1033 target.user = NULL;
1034
1035 parse_url(url, &target);
1036
1037
1038 use_proxy = (strcmp(G.proxy_flag, "off") != 0);
1039 if (use_proxy) {
1040 char *proxy = getenv(target.protocol[0] == 'f' ? "ftp_proxy" : "http_proxy");
1041
1042 use_proxy = (proxy && proxy[0]);
1043 if (use_proxy)
1044 parse_url(proxy, &server);
1045 }
1046 if (!use_proxy) {
1047 server.protocol = target.protocol;
1048 server.port = target.port;
1049 if (ENABLE_FEATURE_IPV6) {
1050
1051 server.host = server.allocated = xstrdup(target.host);
1052 } else {
1053 server.host = target.host;
1054 }
1055 }
1056
1057 if (ENABLE_FEATURE_IPV6)
1058 strip_ipv6_scope_id(target.host);
1059
1060
1061 fname_out_alloc = NULL;
1062 if (!(option_mask32 & WGET_OPT_OUTNAME)) {
1063 G.fname_out = bb_get_last_path_component_nostrip(target.path);
1064
1065 if (G.fname_out[0] == '/' || !G.fname_out[0])
1066 G.fname_out = (char*)"index.html";
1067
1068 if (G.dir_prefix)
1069 G.fname_out = fname_out_alloc = concat_path_file(G.dir_prefix, G.fname_out);
1070 else {
1071
1072 G.fname_out = fname_out_alloc = xstrdup(G.fname_out);
1073 }
1074 }
1075#if ENABLE_FEATURE_WGET_STATUSBAR
1076 G.curfile = bb_get_last_path_component_nostrip(G.fname_out);
1077#endif
1078
1079
1080 G.beg_range = 0;
1081 if (option_mask32 & WGET_OPT_CONTINUE) {
1082 G.output_fd = open(G.fname_out, O_WRONLY);
1083 if (G.output_fd >= 0) {
1084 G.beg_range = xlseek(G.output_fd, 0, SEEK_END);
1085 }
1086
1087
1088 }
1089
1090 redir_limit = 5;
1091 resolve_lsa:
1092 lsa = xhost2sockaddr(server.host, server.port);
1093 if (!(option_mask32 & WGET_OPT_QUIET)) {
1094 char *s = xmalloc_sockaddr2dotted(&lsa->u.sa);
1095 fprintf(stderr, "Connecting to %s (%s)\n", server.host, s);
1096 free(s);
1097 }
1098 establish_session:
1099
1100 G.got_clen = 0;
1101 G.chunked = 0;
1102 if (use_proxy || target.protocol[0] != 'f' ) {
1103
1104
1105
1106 char *str;
1107 int status;
1108
1109
1110#if ENABLE_FEATURE_WGET_OPENSSL
1111
1112 if (server.protocol == P_HTTPS) {
1113
1114
1115
1116 int fd = spawn_https_helper_openssl(server.host, server.port);
1117# if ENABLE_FEATURE_WGET_HTTPS
1118 if (fd < 0) {
1119 sfp = open_socket(lsa);
1120 spawn_ssl_client(server.host, fileno(sfp), 0);
1121 goto socket_opened;
1122 }
1123# else
1124
1125# endif
1126 sfp = fdopen(fd, "r+");
1127 if (!sfp)
1128 bb_die_memory_exhausted();
1129 goto socket_opened;
1130 }
1131 sfp = open_socket(lsa);
1132 socket_opened:
1133#elif ENABLE_FEATURE_WGET_HTTPS
1134
1135 sfp = open_socket(lsa);
1136 if (server.protocol == P_HTTPS)
1137 spawn_ssl_client(server.host, fileno(sfp), 0);
1138#else
1139
1140 sfp = open_socket(lsa);
1141#endif
1142
1143 if (use_proxy) {
1144 SENDFMT(sfp, "GET %s://%s/%s HTTP/1.1\r\n",
1145 target.protocol, target.host,
1146 target.path);
1147 } else {
1148 SENDFMT(sfp, "%s /%s HTTP/1.1\r\n",
1149 (option_mask32 & WGET_OPT_POST_DATA) ? "POST" : "GET",
1150 target.path);
1151 }
1152 if (!USR_HEADER_HOST)
1153 SENDFMT(sfp, "Host: %s\r\n", target.host);
1154 if (!USR_HEADER_USER_AGENT)
1155 SENDFMT(sfp, "User-Agent: %s\r\n", G.user_agent);
1156
1157
1158
1159
1160 SENDFMT(sfp, "Connection: close\r\n");
1161
1162#if ENABLE_FEATURE_WGET_AUTHENTICATION
1163 if (target.user && !USR_HEADER_AUTH) {
1164 SENDFMT(sfp, "Proxy-Authorization: Basic %s\r\n"+6,
1165 base64enc(target.user));
1166 }
1167 if (use_proxy && server.user && !USR_HEADER_PROXY_AUTH) {
1168 SENDFMT(sfp, "Proxy-Authorization: Basic %s\r\n",
1169 base64enc(server.user));
1170 }
1171#endif
1172
1173 if (G.beg_range != 0 && !USR_HEADER_RANGE)
1174 SENDFMT(sfp, "Range: bytes=%"OFF_FMT"u-\r\n", G.beg_range);
1175
1176#if ENABLE_FEATURE_WGET_LONG_OPTIONS
1177 if (G.extra_headers) {
1178 log_io(G.extra_headers);
1179 fputs(G.extra_headers, sfp);
1180 }
1181
1182 if (option_mask32 & WGET_OPT_POST_DATA) {
1183 SENDFMT(sfp,
1184 "Content-Type: application/x-www-form-urlencoded\r\n"
1185 "Content-Length: %u\r\n"
1186 "\r\n"
1187 "%s",
1188 (int) strlen(G.post_data), G.post_data
1189 );
1190 } else
1191#endif
1192 {
1193 SENDFMT(sfp, "\r\n");
1194 }
1195
1196 fflush(sfp);
1197
1198
1199
1200
1201#if SSL_SUPPORTED
1202 if (target.protocol == P_HTTPS) {
1203
1204
1205
1206
1207
1208 shutdown(fileno(sfp), SHUT_WR);
1209 }
1210#endif
1211
1212
1213
1214
1215 read_response:
1216 fgets_trim_sanitize(sfp, " %s\n");
1217
1218 str = G.wget_buf;
1219 str = skip_non_whitespace(str);
1220 str = skip_whitespace(str);
1221
1222
1223 status = atoi(str);
1224 switch (status) {
1225 case 0:
1226 case 100:
1227 while (get_sanitized_hdr(sfp) != NULL)
1228 ;
1229 goto read_response;
1230
1231
1232 case 200:
1233
1234 case 201:
1235
1236
1237
1238 case 202:
1239
1240
1241 case 203:
1242
1243
1244 case 204:
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269 if (G.beg_range != 0) {
1270
1271
1272
1273 reset_beg_range_to_zero();
1274 }
1275 break;
1276
1277
1278 case 300:
1279 case 301:
1280 case 302:
1281 case 303:
1282 break;
1283
1284 case 206:
1285 if (G.beg_range != 0)
1286
1287 break;
1288
1289
1290 default:
1291 bb_error_msg_and_die("server returned error: %s", G.wget_buf);
1292 }
1293
1294
1295
1296
1297 while ((str = get_sanitized_hdr(sfp)) != NULL) {
1298 static const char keywords[] ALIGN1 =
1299 "content-length\0""transfer-encoding\0""location\0";
1300 enum {
1301 KEY_content_length = 1, KEY_transfer_encoding, KEY_location
1302 };
1303 smalluint key;
1304
1305
1306
1307
1308 char *s = strchrnul(str, '\0') - 1;
1309 while (s >= str && (*s == ' ' || *s == '\t')) {
1310 *s = '\0';
1311 s--;
1312 }
1313 key = index_in_strings(keywords, G.wget_buf) + 1;
1314 if (key == KEY_content_length) {
1315 G.content_len = BB_STRTOOFF(str, NULL, 10);
1316 if (G.content_len < 0 || errno) {
1317 bb_error_msg_and_die("content-length %s is garbage", str);
1318 }
1319 G.got_clen = 1;
1320 continue;
1321 }
1322 if (key == KEY_transfer_encoding) {
1323 if (strcmp(str_tolower(str), "chunked") != 0)
1324 bb_error_msg_and_die("transfer encoding '%s' is not supported", str);
1325 G.chunked = 1;
1326 }
1327 if (key == KEY_location && status >= 300) {
1328 if (--redir_limit == 0)
1329 bb_error_msg_and_die("too many redirections");
1330 fclose(sfp);
1331 if (str[0] == '/') {
1332 free(redirected_path);
1333 target.path = redirected_path = xstrdup(str + 1);
1334
1335 } else {
1336 parse_url(str, &target);
1337 if (!use_proxy) {
1338
1339 free(server.allocated);
1340 server.allocated = NULL;
1341 server.protocol = target.protocol;
1342 server.host = target.host;
1343
1344
1345 server.port = target.port;
1346 free(lsa);
1347 goto resolve_lsa;
1348 }
1349 }
1350 goto establish_session;
1351 }
1352 }
1353
1354
1355
1356
1357 dfp = sfp;
1358 } else {
1359
1360
1361
1362 sfp = prepare_ftp_session(&dfp, &target, lsa);
1363 }
1364
1365 free(lsa);
1366
1367 if (!(option_mask32 & WGET_OPT_SPIDER)) {
1368 if (G.output_fd < 0)
1369 G.output_fd = xopen(G.fname_out, G.o_flags);
1370 retrieve_file_data(dfp);
1371 if (!(option_mask32 & WGET_OPT_OUTNAME)) {
1372 xclose(G.output_fd);
1373 G.output_fd = -1;
1374 }
1375 }
1376
1377 if (dfp != sfp) {
1378
1379 fclose(dfp);
1380 if (ftpcmd(NULL, NULL, sfp) != 226)
1381 bb_error_msg_and_die("ftp error: %s", G.wget_buf);
1382
1383 }
1384 fclose(sfp);
1385
1386 free(server.allocated);
1387 free(target.allocated);
1388 free(server.user);
1389 free(target.user);
1390 free(fname_out_alloc);
1391 free(redirected_path);
1392}
1393
1394int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
1395int wget_main(int argc UNUSED_PARAM, char **argv)
1396{
1397#if ENABLE_FEATURE_WGET_LONG_OPTIONS
1398 static const char wget_longopts[] ALIGN1 =
1399
1400 "continue\0" No_argument "c"
1401 "quiet\0" No_argument "q"
1402 "server-response\0" No_argument "S"
1403 "output-document\0" Required_argument "O"
1404 "directory-prefix\0" Required_argument "P"
1405 "proxy\0" Required_argument "Y"
1406 "user-agent\0" Required_argument "U"
1407IF_FEATURE_WGET_TIMEOUT(
1408 "timeout\0" Required_argument "T")
1409
1410IF_DESKTOP( "tries\0" Required_argument "t")
1411 "header\0" Required_argument "\xff"
1412 "post-data\0" Required_argument "\xfe"
1413 "spider\0" No_argument "\xfd"
1414 "no-check-certificate\0" No_argument "\xfc"
1415
1416IF_DESKTOP( "passive-ftp\0" No_argument "\xf0")
1417
1418IF_DESKTOP( "no-cache\0" No_argument "\xf0")
1419IF_DESKTOP( "no-verbose\0" No_argument "\xf0")
1420IF_DESKTOP( "no-clobber\0" No_argument "\xf0")
1421IF_DESKTOP( "no-host-directories\0" No_argument "\xf0")
1422IF_DESKTOP( "no-parent\0" No_argument "\xf0")
1423 ;
1424# define GETOPT32 getopt32long
1425# define LONGOPTS ,wget_longopts
1426#else
1427# define GETOPT32 getopt32
1428# define LONGOPTS
1429#endif
1430
1431#if ENABLE_FEATURE_WGET_LONG_OPTIONS
1432 llist_t *headers_llist = NULL;
1433#endif
1434
1435 INIT_G();
1436
1437#if ENABLE_FEATURE_WGET_TIMEOUT
1438 G.timeout_seconds = 900;
1439 signal(SIGALRM, alarm_handler);
1440#endif
1441 G.proxy_flag = "on";
1442 G.user_agent = "Wget";
1443
1444#if ENABLE_FEATURE_WGET_LONG_OPTIONS
1445#endif
1446 GETOPT32(argv, "^"
1447 "cqSO:P:Y:U:T:+"
1448 "t:"
1449 "n::"
1450
1451
1452
1453
1454
1455
1456
1457
1458 "\0"
1459 "-1"
1460 IF_FEATURE_WGET_LONG_OPTIONS(":\xff::")
1461 LONGOPTS
1462 , &G.fname_out, &G.dir_prefix,
1463 &G.proxy_flag, &G.user_agent,
1464 IF_FEATURE_WGET_TIMEOUT(&G.timeout_seconds) IF_NOT_FEATURE_WGET_TIMEOUT(NULL),
1465 NULL,
1466 NULL
1467 IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist)
1468 IF_FEATURE_WGET_LONG_OPTIONS(, &G.post_data)
1469 );
1470#if 0
1471 if (option_mask32 & WGET_OPT_RETRIES) bb_error_msg("-t NUM");
1472 if (option_mask32 & WGET_OPT_nsomething) bb_error_msg("-nsomething");
1473 if (option_mask32 & WGET_OPT_HEADER) bb_error_msg("--header");
1474 if (option_mask32 & WGET_OPT_POST_DATA) bb_error_msg("--post-data");
1475 if (option_mask32 & WGET_OPT_SPIDER) bb_error_msg("--spider");
1476 if (option_mask32 & WGET_OPT_NO_CHECK_CERT) bb_error_msg("--no-check-certificate");
1477 exit(0);
1478#endif
1479 argv += optind;
1480
1481#if ENABLE_FEATURE_WGET_LONG_OPTIONS
1482 if (headers_llist) {
1483 int size = 0;
1484 char *hdr;
1485 llist_t *ll = headers_llist;
1486 while (ll) {
1487 size += strlen(ll->data) + 2;
1488 ll = ll->link;
1489 }
1490 G.extra_headers = hdr = xmalloc(size + 1);
1491 while (headers_llist) {
1492 int bit;
1493 const char *words;
1494
1495 size = sprintf(hdr, "%s\r\n",
1496 (char*)llist_pop(&headers_llist));
1497
1498 bit = 1;
1499 words = wget_user_headers;
1500 while (*words) {
1501 if (strstr(hdr, words) == hdr) {
1502 G.user_headers |= bit;
1503 break;
1504 }
1505 bit <<= 1;
1506 words += strlen(words) + 1;
1507 }
1508 hdr += size;
1509 }
1510 }
1511#endif
1512
1513 G.output_fd = -1;
1514 G.o_flags = O_WRONLY | O_CREAT | O_TRUNC | O_EXCL;
1515 if (G.fname_out) {
1516 if (LONE_DASH(G.fname_out)) {
1517 G.output_fd = 1;
1518 option_mask32 &= ~WGET_OPT_CONTINUE;
1519 }
1520
1521 G.o_flags = O_WRONLY | O_CREAT | O_TRUNC;
1522 }
1523
1524 while (*argv)
1525 download_one_url(*argv++);
1526
1527 if (G.output_fd >= 0)
1528 xclose(G.output_fd);
1529
1530#if ENABLE_FEATURE_CLEAN_UP && ENABLE_FEATURE_WGET_LONG_OPTIONS
1531 free(G.extra_headers);
1532#endif
1533 FINI_G();
1534
1535 return EXIT_SUCCESS;
1536}
1537