1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154#include "libbb.h"
155
156#if 0
157# define log_io(...) bb_error_msg(__VA_ARGS__)
158# define SENDFMT(fp, fmt, ...) \
159 do { \
160 log_io("> " fmt, ##__VA_ARGS__); \
161 fprintf(fp, fmt, ##__VA_ARGS__); \
162 } while (0);
163#else
164# define log_io(...) ((void)0)
165# define SENDFMT(fp, fmt, ...) fprintf(fp, fmt, ##__VA_ARGS__)
166#endif
167
168
169#define SSL_SUPPORTED (ENABLE_FEATURE_WGET_OPENSSL || ENABLE_FEATURE_WGET_HTTPS)
170
171struct host_info {
172 char *allocated;
173 const char *path;
174 char *user;
175 const char *protocol;
176 char *host;
177 int port;
178};
179static const char P_FTP[] ALIGN1 = "ftp";
180static const char P_HTTP[] ALIGN1 = "http";
181#if SSL_SUPPORTED
182# if ENABLE_FEATURE_WGET_HTTPS
183static const char P_FTPS[] ALIGN1 = "ftps";
184# endif
185static const char P_HTTPS[] ALIGN1 = "https";
186#endif
187
188#if ENABLE_FEATURE_WGET_LONG_OPTIONS
189
190enum {
191 HDR_HOST = (1<<0),
192 HDR_USER_AGENT = (1<<1),
193 HDR_RANGE = (1<<2),
194 HDR_AUTH = (1<<3) * ENABLE_FEATURE_WGET_AUTHENTICATION,
195 HDR_PROXY_AUTH = (1<<4) * ENABLE_FEATURE_WGET_AUTHENTICATION,
196};
197static const char wget_user_headers[] ALIGN1 =
198 "Host:\0"
199 "User-Agent:\0"
200 "Range:\0"
201# if ENABLE_FEATURE_WGET_AUTHENTICATION
202 "Authorization:\0"
203 "Proxy-Authorization:\0"
204# endif
205 ;
206# define USR_HEADER_HOST (G.user_headers & HDR_HOST)
207# define USR_HEADER_USER_AGENT (G.user_headers & HDR_USER_AGENT)
208# define USR_HEADER_RANGE (G.user_headers & HDR_RANGE)
209# define USR_HEADER_AUTH (G.user_headers & HDR_AUTH)
210# define USR_HEADER_PROXY_AUTH (G.user_headers & HDR_PROXY_AUTH)
211#else
212# define USR_HEADER_HOST 0
213# define USR_HEADER_USER_AGENT 0
214# define USR_HEADER_RANGE 0
215# define USR_HEADER_AUTH 0
216# define USR_HEADER_PROXY_AUTH 0
217#endif
218
219
220struct globals {
221 off_t content_len;
222 off_t beg_range;
223#if ENABLE_FEATURE_WGET_STATUSBAR
224 off_t transferred;
225 const char *curfile;
226 bb_progress_t pmt;
227#endif
228 char *dir_prefix;
229#if ENABLE_FEATURE_WGET_LONG_OPTIONS
230 char *post_data;
231 char *extra_headers;
232 unsigned char user_headers;
233#endif
234 char *fname_out;
235 char *fname_log;
236 const char *proxy_flag;
237 const char *user_agent;
238 int output_fd;
239 int log_fd;
240 int o_flags;
241#if ENABLE_FEATURE_WGET_TIMEOUT
242 unsigned timeout_seconds;
243 smallint die_if_timed_out;
244#endif
245 smallint chunked;
246 smallint got_clen;
247
248
249
250
251 char wget_buf[CONFIG_FEATURE_COPYBUF_KB*1024] ALIGNED(16);
252} FIX_ALIASING;
253#define G (*ptr_to_globals)
254#define INIT_G() do { \
255 SET_PTR_TO_GLOBALS(xzalloc(sizeof(G))); \
256} while (0)
257#define FINI_G() do { \
258 FREE_PTR_TO_GLOBALS(); \
259} while (0)
260
261
262
263enum {
264 WGET_OPT_CONTINUE = (1 << 0),
265 WGET_OPT_QUIET = (1 << 1),
266 WGET_OPT_SERVER_RESPONSE = (1 << 2),
267 WGET_OPT_OUTNAME = (1 << 3),
268 WGET_OPT_LOGNAME = (1 << 4),
269 WGET_OPT_PREFIX = (1 << 5),
270 WGET_OPT_PROXY = (1 << 6),
271 WGET_OPT_USER_AGENT = (1 << 7),
272 WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 8),
273 WGET_OPT_RETRIES = (1 << 9),
274 WGET_OPT_nsomething = (1 << 10),
275 WGET_OPT_HEADER = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
276 WGET_OPT_POST_DATA = (1 << 12) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
277 WGET_OPT_SPIDER = (1 << 13) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
278 WGET_OPT_NO_CHECK_CERT = (1 << 14) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
279};
280
281enum {
282 PROGRESS_START = -1,
283 PROGRESS_END = 0,
284 PROGRESS_BUMP = 1,
285};
286#if ENABLE_FEATURE_WGET_STATUSBAR
287static void progress_meter(int flag)
288{
289 int notty;
290
291 if (option_mask32 & WGET_OPT_QUIET)
292 return;
293
294
295 if (G.log_fd >= 0)
296 return;
297
298 if (flag == PROGRESS_START)
299 bb_progress_init(&G.pmt, G.curfile);
300
301 notty = bb_progress_update(&G.pmt,
302 G.beg_range,
303 G.transferred,
304 (G.chunked || !G.got_clen) ? 0 : G.beg_range + G.transferred + G.content_len
305 );
306
307 if (flag == PROGRESS_END) {
308 bb_progress_free(&G.pmt);
309 if (notty == 0)
310 bb_putchar_stderr('\n');
311 G.transferred = 0;
312 }
313}
314#else
315static ALWAYS_INLINE void progress_meter(int flag UNUSED_PARAM) {}
316#endif
317
318
319
320
321
322
323
324
325
326
327
328
329static void strip_ipv6_scope_id(char *host)
330{
331 char *scope, *cp;
332
333
334
335
336
337 if (host[0] != '[')
338 return;
339
340 scope = strchr(host, '%');
341 if (!scope)
342 return;
343
344
345 cp = strchr(host, ']');
346 if (!cp || (cp[1] != ':' && cp[1] != '\0')) {
347
348 return;
349 }
350
351
352 overlapping_strcpy(scope, cp);
353}
354
355#if ENABLE_FEATURE_WGET_AUTHENTICATION
356
357static char *base64enc(const char *str)
358{
359
360 unsigned len = strnlen(str, sizeof(G.wget_buf)/4*3 - 10);
361 bb_uuencode(G.wget_buf, str, len, bb_uuenc_tbl_base64);
362 return G.wget_buf;
363}
364#endif
365
366#if ENABLE_FEATURE_WGET_TIMEOUT
367static void alarm_handler(int sig UNUSED_PARAM)
368{
369
370 if (G.die_if_timed_out)
371 bb_error_msg_and_die("download timed out");
372}
373static void set_alarm(void)
374{
375 if (G.timeout_seconds) {
376 alarm(G.timeout_seconds);
377 G.die_if_timed_out = 1;
378 }
379}
380# define clear_alarm() ((void)(G.die_if_timed_out = 0))
381#else
382# define set_alarm() ((void)0)
383# define clear_alarm() ((void)0)
384#endif
385
386#if ENABLE_FEATURE_WGET_OPENSSL
387
388
389
390
391
392static int is_ip_address(const char *string)
393{
394 struct sockaddr_in sa;
395
396 int result = inet_pton(AF_INET, string, &(sa.sin_addr));
397# if ENABLE_FEATURE_IPV6
398 if (result == 0) {
399 struct sockaddr_in6 sa6;
400 result = inet_pton(AF_INET6, string, &(sa6.sin6_addr));
401 }
402# endif
403 return (result == 1);
404}
405#endif
406
407static FILE *open_socket(len_and_sockaddr *lsa)
408{
409 int fd;
410 FILE *fp;
411
412 set_alarm();
413 fd = xconnect_stream(lsa);
414 clear_alarm();
415
416
417
418 fp = fdopen(fd, "r+");
419 if (!fp)
420 bb_die_memory_exhausted();
421
422 return fp;
423}
424
425
426
427
428
429
430
431
432static char* sanitize_string(char *s)
433{
434 unsigned char *p = (void *) s;
435 while (*p) {
436 if (*p < ' ') {
437 if (*p != '\t')
438 break;
439 *p = ' ';
440 }
441 p++;
442 }
443 *p = '\0';
444 return s;
445}
446
447
448static char fgets_trim_sanitize(FILE *fp, const char *fmt)
449{
450 char c;
451 char *buf_ptr;
452
453 set_alarm();
454 if (fgets(G.wget_buf, sizeof(G.wget_buf), fp) == NULL)
455 bb_perror_msg_and_die("error getting response");
456 clear_alarm();
457
458 buf_ptr = strchrnul(G.wget_buf, '\n');
459 c = *buf_ptr;
460#if 1
461
462 sanitize_string(G.wget_buf);
463#else
464 *buf_ptr = '\0';
465 buf_ptr = strchrnul(G.wget_buf, '\r');
466 *buf_ptr = '\0';
467#endif
468
469 log_io("< %s", G.wget_buf);
470
471 if (fmt && (option_mask32 & WGET_OPT_SERVER_RESPONSE))
472 fprintf(stderr, fmt, G.wget_buf);
473
474 return c;
475}
476
477static int ftpcmd(const char *s1, const char *s2, FILE *fp)
478{
479 int result;
480 if (s1) {
481 if (!s2)
482 s2 = "";
483 fprintf(fp, "%s%s\r\n", s1, s2);
484
485 if (option_mask32 & WGET_OPT_SERVER_RESPONSE)
486 fprintf(stderr, "--> %s%s\n\n", s1, s2);
487 fflush(fp);
488 log_io("> %s%s", s1, s2);
489 }
490
491
492 G.wget_buf[3] = 0;
493 do {
494 fgets_trim_sanitize(fp, "%s\n");
495 } while (!isdigit(G.wget_buf[0]) || G.wget_buf[3] != ' ');
496
497 G.wget_buf[3] = '\0';
498 result = xatoi_positive(G.wget_buf);
499 G.wget_buf[3] = ' ';
500 return result;
501}
502
503static void parse_url(const char *src_url, struct host_info *h)
504{
505 char *url, *p, *sp;
506
507 free(h->allocated);
508 h->allocated = url = xstrdup(src_url);
509
510 h->protocol = P_FTP;
511 p = strstr(url, "://");
512 if (p) {
513 *p = '\0';
514 h->host = p + 3;
515 if (strcmp(url, P_FTP) == 0) {
516 h->port = bb_lookup_std_port(P_FTP, "tcp", 21);
517 } else
518#if SSL_SUPPORTED
519# if ENABLE_FEATURE_WGET_HTTPS
520 if (strcmp(url, P_FTPS) == 0) {
521 h->port = bb_lookup_std_port(P_FTPS, "tcp", 990);
522 h->protocol = P_FTPS;
523 } else
524# endif
525 if (strcmp(url, P_HTTPS) == 0) {
526 h->port = bb_lookup_std_port(P_HTTPS, "tcp", 443);
527 h->protocol = P_HTTPS;
528 } else
529#endif
530 if (strcmp(url, P_HTTP) == 0) {
531 http:
532 h->port = bb_lookup_std_port(P_HTTP, "tcp", 80);
533 h->protocol = P_HTTP;
534 } else {
535 *p = ':';
536 bb_error_msg_and_die("not an http or ftp url: %s", url);
537 }
538 } else {
539
540 h->host = url;
541 goto http;
542 }
543
544
545
546
547
548
549
550
551
552
553
554
555
556 sp = strchr(h->host, '/');
557 p = strchr(h->host, '?'); if (!sp || (p && sp > p)) sp = p;
558 p = strchr(h->host, '#'); if (!sp || (p && sp > p)) sp = p;
559 if (!sp) {
560 h->path = "";
561 } else if (*sp == '/') {
562 *sp = '\0';
563 h->path = sp + 1;
564 } else {
565
566
567
568
569
570 *sp++ = '\0';
571 h->path = sp;
572 }
573
574 sp = strrchr(h->host, '@');
575 if (sp != NULL) {
576
577
578
579
580
581 *sp = '\0';
582 free(h->user);
583 h->user = xstrdup(percent_decode_in_place(h->host, 0));
584 h->host = sp + 1;
585 }
586
587
588
589}
590
591static char *get_sanitized_hdr(FILE *fp)
592{
593 char *s, *hdrval;
594 int c;
595
596
597 c = fgets_trim_sanitize(fp, " %s\n");
598
599
600 if (G.wget_buf[0] == '\0')
601 return NULL;
602
603
604 for (s = G.wget_buf; isalnum(*s) || *s == '-' || *s == '.' || *s == '_'; ++s) {
605
606
607
608
609
610
611
612
613
614 *s |= 0x20;
615 }
616
617
618 if (*s != ':')
619 bb_error_msg_and_die("bad header line: %s", G.wget_buf);
620
621
622 *s++ = '\0';
623 hdrval = skip_whitespace(s);
624
625 if (c != '\n') {
626
627 while (c = getc(fp), c != EOF && c != '\n')
628 continue;
629 }
630
631 return hdrval;
632}
633
634static void reset_beg_range_to_zero(void)
635{
636 bb_error_msg("restart failed");
637 G.beg_range = 0;
638 xlseek(G.output_fd, 0, SEEK_SET);
639
640
641}
642
643#if ENABLE_FEATURE_WGET_OPENSSL
644static int spawn_https_helper_openssl(const char *host, unsigned port)
645{
646 char *allocated = NULL;
647 char *servername;
648 int sp[2];
649 int pid;
650 IF_FEATURE_WGET_HTTPS(volatile int child_failed = 0;)
651
652 if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) != 0)
653
654 bb_perror_msg_and_die("socketpair");
655
656 if (!strchr(host, ':'))
657 host = allocated = xasprintf("%s:%u", host, port);
658 servername = xstrdup(host);
659 strrchr(servername, ':')[0] = '\0';
660
661 fflush_all();
662 pid = xvfork();
663 if (pid == 0) {
664
665 char *argv[8];
666
667 close(sp[0]);
668 xmove_fd(sp[1], 0);
669 xdup2(0, 1);
670
671
672
673
674
675 xmove_fd(2, 3);
676 xopen("/dev/null", O_RDWR);
677 memset(&argv, 0, sizeof(argv));
678 argv[0] = (char*)"openssl";
679 argv[1] = (char*)"s_client";
680 argv[2] = (char*)"-quiet";
681 argv[3] = (char*)"-connect";
682 argv[4] = (char*)host;
683
684
685
686
687
688 if (!is_ip_address(servername)) {
689 argv[5] = (char*)"-servername";
690 argv[6] = (char*)servername;
691 }
692
693 BB_EXECVP(argv[0], argv);
694 xmove_fd(3, 2);
695# if ENABLE_FEATURE_WGET_HTTPS
696 child_failed = 1;
697 xfunc_die();
698# else
699 bb_perror_msg_and_die("can't execute '%s'", argv[0]);
700# endif
701
702 }
703
704
705 free(servername);
706 free(allocated);
707 close(sp[1]);
708# if ENABLE_FEATURE_WGET_HTTPS
709 if (child_failed) {
710 close(sp[0]);
711 return -1;
712 }
713# endif
714 return sp[0];
715}
716#endif
717
718#if ENABLE_FEATURE_WGET_HTTPS
719static void spawn_ssl_client(const char *host, int network_fd, int flags)
720{
721 int sp[2];
722 int pid;
723 char *servername, *p;
724
725 if (!(option_mask32 & WGET_OPT_NO_CHECK_CERT)) {
726 option_mask32 |= WGET_OPT_NO_CHECK_CERT;
727 bb_error_msg("note: TLS certificate validation not implemented");
728 }
729
730 servername = xstrdup(host);
731 p = strrchr(servername, ':');
732 if (p) *p = '\0';
733
734 if (socketpair(AF_UNIX, SOCK_STREAM, 0, sp) != 0)
735
736 bb_perror_msg_and_die("socketpair");
737
738 fflush_all();
739 pid = BB_MMU ? xfork() : xvfork();
740 if (pid == 0) {
741
742 close(sp[0]);
743 xmove_fd(sp[1], 0);
744 xdup2(0, 1);
745 if (BB_MMU) {
746 tls_state_t *tls = new_tls_state();
747 tls->ifd = tls->ofd = network_fd;
748 tls_handshake(tls, servername);
749 tls_run_copy_loop(tls, flags);
750 exit(0);
751 } else {
752 char *argv[6];
753
754 xmove_fd(network_fd, 3);
755 argv[0] = (char*)"ssl_client";
756 argv[1] = (char*)"-s3";
757
758 argv[2] = (char*)"-n";
759 argv[3] = servername;
760 argv[4] = (flags & TLSLOOP_EXIT_ON_LOCAL_EOF ? (char*)"-e" : NULL);
761 argv[5] = NULL;
762 BB_EXECVP(argv[0], argv);
763 bb_perror_msg_and_die("can't execute '%s'", argv[0]);
764 }
765
766 }
767
768
769 free(servername);
770 close(sp[1]);
771 xmove_fd(sp[0], network_fd);
772}
773#endif
774
775static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_sockaddr *lsa)
776{
777 FILE *sfp;
778 char *pass;
779 int port;
780
781 sfp = open_socket(lsa);
782#if ENABLE_FEATURE_WGET_HTTPS
783 if (target->protocol == P_FTPS)
784 spawn_ssl_client(target->host, fileno(sfp), TLSLOOP_EXIT_ON_LOCAL_EOF);
785#endif
786
787 if (ftpcmd(NULL, NULL, sfp) != 220)
788 bb_error_msg_and_die("%s", G.wget_buf);
789
790
791
792 pass = (char*)"busybox";
793 if (target->user) {
794 pass = strchr(target->user, ':');
795 if (pass)
796 *pass++ = '\0';
797 }
798
799
800 switch (ftpcmd("USER ", target->user ?: "anonymous", sfp)) {
801 case 230:
802 break;
803 case 331:
804 if (ftpcmd("PASS ", pass, sfp) == 230)
805 break;
806
807 default:
808 bb_error_msg_and_die("ftp login: %s", G.wget_buf);
809 }
810
811 ftpcmd("TYPE I", NULL, sfp);
812
813
814 if (ftpcmd("SIZE ", target->path, sfp) == 213) {
815 G.content_len = BB_STRTOOFF(G.wget_buf + 4, NULL, 10);
816 if (G.content_len < 0 || errno) {
817 bb_error_msg_and_die("bad SIZE value '%s'", G.wget_buf + 4);
818 }
819 G.got_clen = 1;
820 }
821
822
823 if (ENABLE_FEATURE_IPV6 && ftpcmd("EPSV", NULL, sfp) == 229) {
824
825 } else
826 if (ftpcmd("PASV", NULL, sfp) != 227) {
827 pasv_error:
828 bb_error_msg_and_die("bad response to %s: %s", "PASV", G.wget_buf);
829 }
830 port = parse_pasv_epsv(G.wget_buf);
831 if (port < 0)
832 goto pasv_error;
833
834 set_nport(&lsa->u.sa, htons(port));
835
836 *dfpp = open_socket(lsa);
837
838#if ENABLE_FEATURE_WGET_HTTPS
839 if (target->protocol == P_FTPS) {
840
841
842
843 if (ftpcmd("PROT P", NULL, sfp) == 200)
844 spawn_ssl_client(target->host, fileno(*dfpp), 0);
845 }
846#endif
847
848 if (G.beg_range != 0) {
849 sprintf(G.wget_buf, "REST %"OFF_FMT"u", G.beg_range);
850 if (ftpcmd(G.wget_buf, NULL, sfp) == 350)
851 G.content_len -= G.beg_range;
852 else
853 reset_beg_range_to_zero();
854 }
855
856
857
858
859 if (ftpcmd("RETR ", target->path, sfp) > 150)
860 bb_error_msg_and_die("bad response to %s: %s", "RETR", G.wget_buf);
861
862 return sfp;
863}
864
865static void NOINLINE retrieve_file_data(FILE *dfp)
866{
867#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
868# if ENABLE_FEATURE_WGET_TIMEOUT
869 unsigned second_cnt = G.timeout_seconds;
870# endif
871 struct pollfd polldata;
872
873 polldata.fd = fileno(dfp);
874 polldata.events = POLLIN | POLLPRI;
875#endif
876 if (!(option_mask32 & WGET_OPT_QUIET)) {
877 if (G.output_fd == 1)
878 fprintf(stderr, "writing to stdout\n");
879 else
880 fprintf(stderr, "saving to '%s'\n", G.fname_out);
881 }
882 progress_meter(PROGRESS_START);
883
884 if (G.chunked)
885 goto get_clen;
886
887
888 while (1) {
889
890#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
891
892
893
894
895
896
897
898 ndelay_on(polldata.fd);
899#endif
900 while (1) {
901 int n;
902 unsigned rdsz;
903
904#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
905
906
907
908
909
910
911
912
913 clearerr(dfp);
914#endif
915 errno = 0;
916 rdsz = sizeof(G.wget_buf);
917 if (G.got_clen) {
918 if (G.content_len < (off_t)sizeof(G.wget_buf)) {
919 if ((int)G.content_len <= 0)
920 break;
921 rdsz = (unsigned)G.content_len;
922 }
923 }
924 n = fread(G.wget_buf, 1, rdsz, dfp);
925
926 if (n > 0) {
927 xwrite(G.output_fd, G.wget_buf, n);
928#if ENABLE_FEATURE_WGET_STATUSBAR
929 G.transferred += n;
930#endif
931 if (G.got_clen) {
932 G.content_len -= n;
933 if (G.content_len == 0)
934 break;
935 }
936#if ENABLE_FEATURE_WGET_TIMEOUT
937 second_cnt = G.timeout_seconds;
938#endif
939 goto bump;
940 }
941
942
943
944
945
946
947
948 if (errno != EAGAIN) {
949 if (ferror(dfp)) {
950 progress_meter(PROGRESS_END);
951 bb_perror_msg_and_die(bb_msg_read_error);
952 }
953 break;
954 }
955
956#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
957
958
959
960 if (safe_poll(&polldata, 1, 1000) == 0) {
961# if ENABLE_FEATURE_WGET_TIMEOUT
962 if (second_cnt != 0 && --second_cnt == 0) {
963 progress_meter(PROGRESS_END);
964 bb_error_msg_and_die("download timed out");
965 }
966# endif
967
968
969
970
971 }
972#endif
973 bump:
974
975
976
977 progress_meter(PROGRESS_BUMP);
978 }
979
980#if ENABLE_FEATURE_WGET_STATUSBAR || ENABLE_FEATURE_WGET_TIMEOUT
981 clearerr(dfp);
982 ndelay_off(polldata.fd);
983#endif
984 if (!G.chunked)
985 break;
986
987
988 fgets_trim_sanitize(dfp, NULL);
989 get_clen:
990
991 fgets_trim_sanitize(dfp, NULL);
992 errno = 0;
993 G.content_len = STRTOOFF(G.wget_buf, NULL, 16);
994
995
996
997
998 if (G.content_len < 0 || errno)
999 bb_error_msg_and_die("bad chunk length '%s'", G.wget_buf);
1000 if (G.content_len == 0)
1001 break;
1002 G.got_clen = 1;
1003
1004
1005
1006
1007
1008
1009
1010 }
1011
1012
1013 G.chunked = 0;
1014 G.got_clen = 1;
1015 progress_meter(PROGRESS_END);
1016 if (G.content_len != 0) {
1017 bb_perror_msg_and_die("connection closed prematurely");
1018
1019 }
1020
1021
1022
1023
1024
1025
1026 {
1027 off_t pos = lseek(G.output_fd, 0, SEEK_CUR);
1028 if (pos != (off_t)-1)
1029 ftruncate(G.output_fd, pos);
1030 }
1031
1032 if (!(option_mask32 & WGET_OPT_QUIET)) {
1033 if (G.output_fd == 1)
1034 fprintf(stderr, "written to stdout\n");
1035 else
1036 fprintf(stderr, "'%s' saved\n", G.fname_out);
1037 }
1038}
1039
1040static void download_one_url(const char *url)
1041{
1042 bool use_proxy;
1043 int redir_limit;
1044 len_and_sockaddr *lsa;
1045 FILE *sfp;
1046 FILE *dfp;
1047 char *fname_out_alloc;
1048 char *redirected_path = NULL;
1049 struct host_info server;
1050 struct host_info target;
1051
1052 server.allocated = NULL;
1053 target.allocated = NULL;
1054 server.user = NULL;
1055 target.user = NULL;
1056
1057 parse_url(url, &target);
1058
1059
1060 use_proxy = (strcmp(G.proxy_flag, "off") != 0);
1061 if (use_proxy) {
1062 char *proxy = getenv(target.protocol[0] == 'f' ? "ftp_proxy" : "http_proxy");
1063
1064 use_proxy = (proxy && proxy[0]);
1065 if (use_proxy)
1066 parse_url(proxy, &server);
1067 }
1068 if (!use_proxy) {
1069 server.protocol = target.protocol;
1070 server.port = target.port;
1071 if (ENABLE_FEATURE_IPV6) {
1072
1073 server.host = server.allocated = xstrdup(target.host);
1074 } else {
1075 server.host = target.host;
1076 }
1077 }
1078
1079 if (ENABLE_FEATURE_IPV6)
1080 strip_ipv6_scope_id(target.host);
1081
1082
1083 fname_out_alloc = NULL;
1084 if (!(option_mask32 & WGET_OPT_OUTNAME)) {
1085 G.fname_out = bb_get_last_path_component_nostrip(target.path);
1086
1087 if (G.fname_out[0] == '/' || !G.fname_out[0])
1088 G.fname_out = (char*)"index.html";
1089
1090 if (G.dir_prefix)
1091 G.fname_out = fname_out_alloc = concat_path_file(G.dir_prefix, G.fname_out);
1092 else {
1093
1094 G.fname_out = fname_out_alloc = xstrdup(G.fname_out);
1095 }
1096 }
1097#if ENABLE_FEATURE_WGET_STATUSBAR
1098 G.curfile = bb_get_last_path_component_nostrip(G.fname_out);
1099#endif
1100
1101
1102 G.beg_range = 0;
1103 if (option_mask32 & WGET_OPT_CONTINUE) {
1104 G.output_fd = open(G.fname_out, O_WRONLY);
1105 if (G.output_fd >= 0) {
1106 G.beg_range = xlseek(G.output_fd, 0, SEEK_END);
1107 }
1108
1109
1110 }
1111
1112 redir_limit = 5;
1113 resolve_lsa:
1114 lsa = xhost2sockaddr(server.host, server.port);
1115 if (!(option_mask32 & WGET_OPT_QUIET)) {
1116 char *s = xmalloc_sockaddr2dotted(&lsa->u.sa);
1117 fprintf(stderr, "Connecting to %s (%s)\n", server.host, s);
1118 free(s);
1119 }
1120 establish_session:
1121
1122 G.got_clen = 0;
1123 G.chunked = 0;
1124 if (use_proxy || target.protocol[0] != 'f' ) {
1125
1126
1127
1128 char *str;
1129 int status;
1130
1131
1132#if ENABLE_FEATURE_WGET_OPENSSL
1133
1134 if (server.protocol == P_HTTPS) {
1135
1136
1137
1138 int fd = spawn_https_helper_openssl(server.host, server.port);
1139# if ENABLE_FEATURE_WGET_HTTPS
1140 if (fd < 0) {
1141 sfp = open_socket(lsa);
1142 spawn_ssl_client(server.host, fileno(sfp), 0);
1143 goto socket_opened;
1144 }
1145# else
1146
1147# endif
1148 sfp = fdopen(fd, "r+");
1149 if (!sfp)
1150 bb_die_memory_exhausted();
1151 goto socket_opened;
1152 }
1153 sfp = open_socket(lsa);
1154 socket_opened:
1155#elif ENABLE_FEATURE_WGET_HTTPS
1156
1157 sfp = open_socket(lsa);
1158 if (server.protocol == P_HTTPS)
1159 spawn_ssl_client(server.host, fileno(sfp), 0);
1160#else
1161
1162 sfp = open_socket(lsa);
1163#endif
1164
1165 if (use_proxy) {
1166 SENDFMT(sfp, "GET %s://%s/%s HTTP/1.1\r\n",
1167 target.protocol, target.host,
1168 target.path);
1169 } else {
1170 SENDFMT(sfp, "%s /%s HTTP/1.1\r\n",
1171 (option_mask32 & WGET_OPT_POST_DATA) ? "POST" : "GET",
1172 target.path);
1173 }
1174 if (!USR_HEADER_HOST)
1175 SENDFMT(sfp, "Host: %s\r\n", target.host);
1176 if (!USR_HEADER_USER_AGENT)
1177 SENDFMT(sfp, "User-Agent: %s\r\n", G.user_agent);
1178
1179
1180
1181
1182 SENDFMT(sfp, "Connection: close\r\n");
1183
1184#if ENABLE_FEATURE_WGET_AUTHENTICATION
1185 if (target.user && !USR_HEADER_AUTH) {
1186 SENDFMT(sfp, "Proxy-Authorization: Basic %s\r\n"+6,
1187 base64enc(target.user));
1188 }
1189 if (use_proxy && server.user && !USR_HEADER_PROXY_AUTH) {
1190 SENDFMT(sfp, "Proxy-Authorization: Basic %s\r\n",
1191 base64enc(server.user));
1192 }
1193#endif
1194
1195 if (G.beg_range != 0 && !USR_HEADER_RANGE)
1196 SENDFMT(sfp, "Range: bytes=%"OFF_FMT"u-\r\n", G.beg_range);
1197
1198#if ENABLE_FEATURE_WGET_LONG_OPTIONS
1199 if (G.extra_headers) {
1200 log_io(G.extra_headers);
1201 fputs(G.extra_headers, sfp);
1202 }
1203
1204 if (option_mask32 & WGET_OPT_POST_DATA) {
1205 SENDFMT(sfp,
1206 "Content-Type: application/x-www-form-urlencoded\r\n"
1207 "Content-Length: %u\r\n"
1208 "\r\n"
1209 "%s",
1210 (int) strlen(G.post_data), G.post_data
1211 );
1212 } else
1213#endif
1214 {
1215 SENDFMT(sfp, "\r\n");
1216 }
1217
1218 fflush(sfp);
1219
1220
1221
1222
1223#if SSL_SUPPORTED
1224 if (target.protocol == P_HTTPS) {
1225
1226
1227
1228
1229
1230 shutdown(fileno(sfp), SHUT_WR);
1231 }
1232#endif
1233
1234
1235
1236
1237 read_response:
1238 fgets_trim_sanitize(sfp, " %s\n");
1239
1240 str = G.wget_buf;
1241 str = skip_non_whitespace(str);
1242 str = skip_whitespace(str);
1243
1244
1245 status = atoi(str);
1246 switch (status) {
1247 case 0:
1248 case 100:
1249 while (get_sanitized_hdr(sfp) != NULL)
1250 ;
1251 goto read_response;
1252
1253
1254 case 200:
1255
1256 case 201:
1257
1258
1259
1260 case 202:
1261
1262
1263 case 203:
1264
1265
1266 case 204:
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291 if (G.beg_range != 0) {
1292
1293
1294
1295 reset_beg_range_to_zero();
1296 }
1297 break;
1298
1299
1300 case 300:
1301 case 301:
1302 case 302:
1303 case 303:
1304 break;
1305
1306 case 206:
1307 if (G.beg_range != 0)
1308
1309 break;
1310
1311
1312 default:
1313 bb_error_msg_and_die("server returned error: %s", G.wget_buf);
1314 }
1315
1316
1317
1318
1319 while ((str = get_sanitized_hdr(sfp)) != NULL) {
1320 static const char keywords[] ALIGN1 =
1321 "content-length\0""transfer-encoding\0""location\0";
1322 enum {
1323 KEY_content_length = 1, KEY_transfer_encoding, KEY_location
1324 };
1325 smalluint key;
1326
1327
1328
1329
1330 char *s = strchrnul(str, '\0') - 1;
1331 while (s >= str && (*s == ' ' || *s == '\t')) {
1332 *s = '\0';
1333 s--;
1334 }
1335 key = index_in_strings(keywords, G.wget_buf) + 1;
1336 if (key == KEY_content_length) {
1337 G.content_len = BB_STRTOOFF(str, NULL, 10);
1338 if (G.content_len < 0 || errno) {
1339 bb_error_msg_and_die("content-length %s is garbage", str);
1340 }
1341 G.got_clen = 1;
1342 continue;
1343 }
1344 if (key == KEY_transfer_encoding) {
1345 if (strcmp(str_tolower(str), "chunked") != 0)
1346 bb_error_msg_and_die("transfer encoding '%s' is not supported", str);
1347 G.chunked = 1;
1348 }
1349 if (key == KEY_location && status >= 300) {
1350 if (--redir_limit == 0)
1351 bb_error_msg_and_die("too many redirections");
1352 fclose(sfp);
1353 if (str[0] == '/') {
1354 free(redirected_path);
1355 target.path = redirected_path = xstrdup(str + 1);
1356
1357 } else {
1358 parse_url(str, &target);
1359 if (!use_proxy) {
1360
1361 free(server.allocated);
1362 server.allocated = NULL;
1363 server.protocol = target.protocol;
1364 server.host = target.host;
1365
1366
1367 server.port = target.port;
1368 free(lsa);
1369 goto resolve_lsa;
1370 }
1371 }
1372 goto establish_session;
1373 }
1374 }
1375
1376
1377
1378
1379 dfp = sfp;
1380 } else {
1381
1382
1383
1384 sfp = prepare_ftp_session(&dfp, &target, lsa);
1385 }
1386
1387 free(lsa);
1388
1389 if (!(option_mask32 & WGET_OPT_SPIDER)) {
1390 if (G.output_fd < 0)
1391 G.output_fd = xopen(G.fname_out, G.o_flags);
1392 retrieve_file_data(dfp);
1393 if (!(option_mask32 & WGET_OPT_OUTNAME)) {
1394 xclose(G.output_fd);
1395 G.output_fd = -1;
1396 }
1397 } else {
1398 if (!(option_mask32 & WGET_OPT_QUIET))
1399 fprintf(stderr, "remote file exists\n");
1400 }
1401
1402 if (dfp != sfp) {
1403
1404 fclose(dfp);
1405 if (ftpcmd(NULL, NULL, sfp) != 226)
1406 bb_error_msg_and_die("ftp error: %s", G.wget_buf);
1407
1408 }
1409 fclose(sfp);
1410
1411 free(server.allocated);
1412 free(target.allocated);
1413 free(server.user);
1414 free(target.user);
1415 free(fname_out_alloc);
1416 free(redirected_path);
1417}
1418
1419int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
1420int wget_main(int argc UNUSED_PARAM, char **argv)
1421{
1422#if ENABLE_FEATURE_WGET_LONG_OPTIONS
1423 static const char wget_longopts[] ALIGN1 =
1424
1425 "continue\0" No_argument "c"
1426 "quiet\0" No_argument "q"
1427 "server-response\0" No_argument "S"
1428 "output-document\0" Required_argument "O"
1429 "output-file\0" Required_argument "o"
1430 "directory-prefix\0" Required_argument "P"
1431 "proxy\0" Required_argument "Y"
1432 "user-agent\0" Required_argument "U"
1433IF_FEATURE_WGET_TIMEOUT(
1434 "timeout\0" Required_argument "T")
1435
1436IF_DESKTOP( "tries\0" Required_argument "t")
1437 "header\0" Required_argument "\xff"
1438 "post-data\0" Required_argument "\xfe"
1439 "spider\0" No_argument "\xfd"
1440 "no-check-certificate\0" No_argument "\xfc"
1441
1442IF_DESKTOP( "passive-ftp\0" No_argument "\xf0")
1443
1444IF_DESKTOP( "no-cache\0" No_argument "\xf0")
1445IF_DESKTOP( "no-verbose\0" No_argument "\xf0")
1446IF_DESKTOP( "no-clobber\0" No_argument "\xf0")
1447IF_DESKTOP( "no-host-directories\0" No_argument "\xf0")
1448IF_DESKTOP( "no-parent\0" No_argument "\xf0")
1449 ;
1450# define GETOPT32 getopt32long
1451# define LONGOPTS ,wget_longopts
1452#else
1453# define GETOPT32 getopt32
1454# define LONGOPTS
1455#endif
1456
1457#if ENABLE_FEATURE_WGET_LONG_OPTIONS
1458 llist_t *headers_llist = NULL;
1459#endif
1460
1461 INIT_G();
1462
1463#if ENABLE_FEATURE_WGET_TIMEOUT
1464 G.timeout_seconds = 900;
1465 signal(SIGALRM, alarm_handler);
1466#endif
1467 G.proxy_flag = "on";
1468 G.user_agent = "Wget";
1469
1470 GETOPT32(argv, "^"
1471 "cqSO:o:P:Y:U:T:+"
1472 "t:"
1473 "n::"
1474
1475
1476
1477
1478
1479
1480
1481
1482 "\0"
1483 "-1"
1484 IF_FEATURE_WGET_LONG_OPTIONS(":\xff::")
1485 LONGOPTS
1486 , &G.fname_out, &G.fname_log, &G.dir_prefix,
1487 &G.proxy_flag, &G.user_agent,
1488 IF_FEATURE_WGET_TIMEOUT(&G.timeout_seconds) IF_NOT_FEATURE_WGET_TIMEOUT(NULL),
1489 NULL,
1490 NULL
1491 IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist)
1492 IF_FEATURE_WGET_LONG_OPTIONS(, &G.post_data)
1493 );
1494#if 0
1495 if (option_mask32 & WGET_OPT_RETRIES) bb_error_msg("-t NUM");
1496 if (option_mask32 & WGET_OPT_nsomething) bb_error_msg("-nsomething");
1497 if (option_mask32 & WGET_OPT_HEADER) bb_error_msg("--header");
1498 if (option_mask32 & WGET_OPT_POST_DATA) bb_error_msg("--post-data");
1499 if (option_mask32 & WGET_OPT_SPIDER) bb_error_msg("--spider");
1500 if (option_mask32 & WGET_OPT_NO_CHECK_CERT) bb_error_msg("--no-check-certificate");
1501 exit(0);
1502#endif
1503 argv += optind;
1504
1505#if ENABLE_FEATURE_WGET_LONG_OPTIONS
1506 if (headers_llist) {
1507 int size = 0;
1508 char *hdr;
1509 llist_t *ll = headers_llist;
1510 while (ll) {
1511 size += strlen(ll->data) + 2;
1512 ll = ll->link;
1513 }
1514 G.extra_headers = hdr = xmalloc(size + 1);
1515 while (headers_llist) {
1516 int bit;
1517 const char *words;
1518
1519 size = sprintf(hdr, "%s\r\n",
1520 (char*)llist_pop(&headers_llist));
1521
1522 bit = 1;
1523 words = wget_user_headers;
1524 while (*words) {
1525 if (strstr(hdr, words) == hdr) {
1526 G.user_headers |= bit;
1527 break;
1528 }
1529 bit <<= 1;
1530 words += strlen(words) + 1;
1531 }
1532 hdr += size;
1533 }
1534 }
1535#endif
1536
1537 G.output_fd = -1;
1538 G.o_flags = O_WRONLY | O_CREAT | O_TRUNC | O_EXCL;
1539 if (G.fname_out) {
1540 if (LONE_DASH(G.fname_out)) {
1541 G.output_fd = 1;
1542 option_mask32 &= ~WGET_OPT_CONTINUE;
1543 }
1544
1545 G.o_flags = O_WRONLY | O_CREAT | O_TRUNC;
1546 }
1547
1548 G.log_fd = -1;
1549 if (G.fname_log) {
1550 if (!LONE_DASH(G.fname_log)) {
1551
1552 G.log_fd = xopen(G.fname_log, O_WRONLY | O_CREAT | O_TRUNC);
1553
1554 xdup2(G.log_fd, STDERR_FILENO);
1555 }
1556 }
1557
1558 while (*argv)
1559 download_one_url(*argv++);
1560
1561 if (G.output_fd >= 0)
1562 xclose(G.output_fd);
1563
1564 if (G.log_fd >= 0)
1565 xclose(G.log_fd);
1566
1567#if ENABLE_FEATURE_CLEAN_UP && ENABLE_FEATURE_WGET_LONG_OPTIONS
1568 free(G.extra_headers);
1569#endif
1570 FINI_G();
1571
1572 return EXIT_SUCCESS;
1573}
1574