Diff of /trunk/mkinitrd-magellan/busybox/networking/wget.c

Parent Directory | Revision Log | Patch

	revision 983 by niro, Fri Apr 24 18:33:46 2009 UTC	revision 984 by niro, Sun May 30 11:32:42 2010 UTC
#	Line 4	Line 4
4	*	*
5	* Chip Rosenthal Covad Communications <chip@laserlink.net>	* Chip Rosenthal Covad Communications <chip@laserlink.net>
6	*	*
7		* Licensed under GPLv2, see file LICENSE in this tarball for details.
8	*/	*/
9	#include "libbb.h"	#include "libbb.h"
10
11	struct host_info {	struct host_info {
#	Line 24  struct globals {	Line 24  struct globals {
24	off_t content_len;        /* Content-length of the file */	off_t content_len;        /* Content-length of the file */
25	off_t beg_range;          /* Range at which continue begins */	off_t beg_range;          /* Range at which continue begins */
26	#if ENABLE_FEATURE_WGET_STATUSBAR	#if ENABLE_FEATURE_WGET_STATUSBAR
	off_t lastsize;
	off_t totalsize;
27	off_t transferred;        /* Number of bytes transferred so far */	off_t transferred;        /* Number of bytes transferred so far */
28	const char *curfile;      /* Name of current file being transferred */	const char *curfile;      /* Name of current file being transferred */
29	unsigned lastupdate_sec;	bb_progress_t pmt;
	unsigned start_sec;
30	#endif	#endif
31	smallint chunked;             /* chunked transfer encoding */	smallint chunked;         /* chunked transfer encoding */
32		smallint got_clen;        /* got content-length: from server  */
33	};	};
34	#define G (*(struct globals*)&bb_common_bufsiz1)	#define G (*(struct globals*)&bb_common_bufsiz1)
35	struct BUG_G_too_big {	struct BUG_G_too_big {
36	char BUG_G_too_big[sizeof(G) <= COMMON_BUFSIZE ? 1 : -1];	char BUG_G_too_big[sizeof(G) <= COMMON_BUFSIZE ? 1 : -1];
37	};	};
	#define content_len     (G.content_len    )
	#define beg_range       (G.beg_range      )
	#define lastsize        (G.lastsize       )
	#define totalsize       (G.totalsize      )
	#define transferred     (G.transferred    )
	#define curfile         (G.curfile        )
	#define lastupdate_sec  (G.lastupdate_sec )
	#define start_sec       (G.start_sec      )
	#define chunked         (G.chunked        )
38	#define INIT_G() do { } while (0)	#define INIT_G() do { } while (0)
39
40
41	#if ENABLE_FEATURE_WGET_STATUSBAR	#if ENABLE_FEATURE_WGET_STATUSBAR
	enum {
	STALLTIME = 5                   /* Seconds when xfer considered "stalled" */
	};
42
43	static unsigned int getttywidth(void)	static void progress_meter(int flag)
	{
	unsigned width;
	get_terminal_width_height(0, &width, NULL);
	return width;
	}
	static void progressmeter(int flag)
44	{	{
45	/* We can be called from signal handler */	/* We can be called from signal handler */
46	int save_errno = errno;	int save_errno = errno;
	off_t abbrevsize;
	unsigned since_last_update, elapsed;
	unsigned ratio;
	int barlength, i;
	if (flag == -1) { /* first call to progressmeter */
	start_sec = monotonic_sec();
	lastupdate_sec = start_sec;
	lastsize = 0;
	totalsize = content_len + beg_range; /* as content_len changes.. */
	}
	ratio = 100;
	if (totalsize != 0 && !chunked) {
	/* long long helps to have it working even if !LFS */
	ratio = (unsigned) (100ULL * (transferred+beg_range) / totalsize);
	if (ratio > 100) ratio = 100;
	}
	fprintf(stderr, "\r%-20.20s%4d%% ", curfile, ratio);
	barlength = getttywidth() - 49;
	if (barlength > 0) {
	/* god bless gcc for variable arrays :) */
	i = barlength * ratio / 100;
	{
	char buf[i+1];
	memset(buf, '*', i);
	buf[i] = '\0';
	fprintf(stderr, "|%s%*s|", buf, barlength - i, "");
	}
	}
	i = 0;
	abbrevsize = transferred + beg_range;
	while (abbrevsize >= 100000) {
	i++;
	abbrevsize >>= 10;
	}
	/* see http://en.wikipedia.org/wiki/Tera */
	fprintf(stderr, "%6d%c ", (int)abbrevsize, " kMGTPEZY"[i]);
	// Nuts! Ain't it easier to update progress meter ONLY when we transferred++?
	elapsed = monotonic_sec();
	since_last_update = elapsed - lastupdate_sec;
	if (transferred > lastsize) {
	lastupdate_sec = elapsed;
	lastsize = transferred;
	if (since_last_update >= STALLTIME) {
	/* We "cut off" these seconds from elapsed time
	* by adjusting start time */
	start_sec += since_last_update;
	}
	since_last_update = 0; /* we are un-stalled now */
	}
	elapsed -= start_sec; /* now it's "elapsed since start" */
47
48	if (since_last_update >= STALLTIME) {	if (flag == -1) { /* first call to progress_meter */
49	fprintf(stderr, " - stalled -");	bb_progress_init(&G.pmt);
	} else {
	off_t to_download = totalsize - beg_range;
	if (transferred <= 0 || (int)elapsed <= 0 || transferred > to_download || chunked) {
	fprintf(stderr, "--:--:-- ETA");
	} else {
	/* to_download / (transferred/elapsed) - elapsed: */
	int eta = (int) ((unsigned long long)to_download*elapsed/transferred - elapsed);
	/* (long long helps to have working ETA even if !LFS) */
	i = eta % 3600;
	fprintf(stderr, "%02d:%02d:%02d ETA", eta / 3600, i / 60, i % 60);
	}
50	}	}
51
52		bb_progress_update(&G.pmt, G.curfile, G.beg_range, G.transferred,
53		G.chunked ? 0 : G.content_len + G.beg_range);
54
55	if (flag == 0) {	if (flag == 0) {
56	/* last call to progressmeter */	/* last call to progress_meter */
57	alarm(0);	alarm(0);
	transferred = 0;
58	fputc('\n', stderr);	fputc('\n', stderr);
59		G.transferred = 0;
60	} else {	} else {
61	if (flag == -1) { /* first call to progressmeter */	if (flag == -1) { /* first call to progress_meter */
62	signal_SA_RESTART_empty_mask(SIGALRM, progressmeter);	signal_SA_RESTART_empty_mask(SIGALRM, progress_meter);
63	}	}
64	alarm(1);	alarm(1);
65	}	}
66
67	errno = save_errno;	errno = save_errno;
68	}	}
69	/* Original copyright notice which applies to the CONFIG_FEATURE_WGET_STATUSBAR stuff,
	* much of which was blatantly stolen from openssh.  */
	/*-
	* Copyright (c) 1992, 1993
	* The Regents of the University of California.  All rights reserved.
	*
	* Redistribution and use in source and binary forms, with or without
	* modification, are permitted provided that the following conditions
	* are met:
	* 1. Redistributions of source code must retain the above copyright
	*    notice, this list of conditions and the following disclaimer.
	* 2. Redistributions in binary form must reproduce the above copyright
	*    notice, this list of conditions and the following disclaimer in the
	*    documentation and/or other materials provided with the distribution.
	*
	* 3. <BSD Advertising Clause omitted per the July 22, 1999 licensing change
	* ftp://ftp.cs.berkeley.edu/pub/4bsd/README.Impt.License.Change>
	*
	* 4. Neither the name of the University nor the names of its contributors
	*    may be used to endorse or promote products derived from this software
	*    without specific prior written permission.
	*
	* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
	* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	* ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
	* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
	* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
	* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
	* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
	* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
	* SUCH DAMAGE.
	*
	*/
70	#else /* FEATURE_WGET_STATUSBAR */	#else /* FEATURE_WGET_STATUSBAR */
71
72	static ALWAYS_INLINE void progressmeter(int flag UNUSED_PARAM) { }	static ALWAYS_INLINE void progress_meter(int flag UNUSED_PARAM) { }
73
74	#endif	#endif
75
76
77		/* IPv6 knows scoped address types i.e. link and site local addresses. Link
78		* local addresses can have a scope identifier to specify the
79		* interface/link an address is valid on (e.g. fe80::1%eth0). This scope
80		* identifier is only valid on a single node.
81		*
82		* RFC 4007 says that the scope identifier MUST NOT be sent across the wire,
83		* unless all nodes agree on the semantic. Apache e.g. regards zone identifiers
84		* in the Host header as invalid requests, see
85		* https://issues.apache.org/bugzilla/show_bug.cgi?id=35122
86		*/
87		static void strip_ipv6_scope_id(char *host)
88		{
89		char *scope, *cp;
90
91		/* bbox wget actually handles IPv6 addresses without [], like
92		* wget "http://::1/xxx", but this is not standard.
93		* To save code, _here_ we do not support it. */
94
95		if (host[0] != '[')
96		return; /* not IPv6 */
97
98		scope = strchr(host, '%');
99		if (!scope)
100		return;
101
102		/* Remove the IPv6 zone identifier from the host address */
103		cp = strchr(host, ']');
104		if (!cp || (cp[1] != ':' && cp[1] != '\0')) {
105		/* malformed address (not "[xx]:nn" or "[xx]") */
106		return;
107		}
108
109		/* cp points to "]...", scope points to "%eth0]..." */
110		overlapping_strcpy(scope, cp);
111		}
112
113	/* Read NMEMB bytes into PTR from STREAM.  Returns the number of bytes read,	/* Read NMEMB bytes into PTR from STREAM.  Returns the number of bytes read,
114	* and a short count if an eof or non-interrupt error is encountered.  */	* and a short count if an eof or non-interrupt error is encountered.  */
115	static size_t safe_fread(void *ptr, size_t nmemb, FILE *stream)	static size_t safe_fread(void *ptr, size_t nmemb, FILE *stream)
#	Line 202  static size_t safe_fread(void *ptr, size	Line 119  static size_t safe_fread(void *ptr, size
119
120	do {	do {
121	clearerr(stream);	clearerr(stream);
122		errno = 0;
123	ret = fread(p, 1, nmemb, stream);	ret = fread(p, 1, nmemb, stream);
124	p += ret;	p += ret;
125	nmemb -= ret;	nmemb -= ret;
#	Line 218  static char *safe_fgets(char *s, int siz	Line 136  static char *safe_fgets(char *s, int siz
136
137	do {	do {
138	clearerr(stream);	clearerr(stream);
139		errno = 0;
140	ret = fgets(s, size, stream);	ret = fgets(s, size, stream);
141	} while (ret == NULL && ferror(stream) && errno == EINTR);	} while (ret == NULL && ferror(stream) && errno == EINTR);
142
#	Line 236  static char *base64enc_512(char buf[512]	Line 155  static char *base64enc_512(char buf[512]
155	}	}
156	#endif	#endif
157
158		static char* sanitize_string(char *s)
159		{
160		unsigned char *p = (void *) s;
161		while (*p >= ' ')
162		p++;
163		*p = '\0';
164		return s;
165		}
166
167	static FILE *open_socket(len_and_sockaddr *lsa)	static FILE *open_socket(len_and_sockaddr *lsa)
168	{	{
#	Line 250  static FILE *open_socket(len_and_sockadd	Line 177  static FILE *open_socket(len_and_sockadd
177	return fp;	return fp;
178	}	}
179
180	static int ftpcmd(const char *s1, const char *s2, FILE *fp, char *buf)	static int ftpcmd(const char *s1, const char *s2, FILE *fp, char *buf)
181	{	{
182	int result;	int result;
#	Line 278  static int ftpcmd(const char *s1, const	Line 204  static int ftpcmd(const char *s1, const
204	return result;	return result;
205	}	}
206
207	static void parse_url(char *src_url, struct host_info *h)	static void parse_url(char *src_url, struct host_info *h)
208	{	{
209	char *url, *p, *sp;	char *url, *p, *sp;
#	Line 294  static void parse_url(char *src_url, str	Line 219  static void parse_url(char *src_url, str
219	h->host = url + 6;	h->host = url + 6;
220	h->is_ftp = 1;	h->is_ftp = 1;
221	} else	} else
222	bb_error_msg_and_die("not an http or ftp url: %s", url);	bb_error_msg_and_die("not an http or ftp url: %s", sanitize_string(url));
223
224	// FYI:	// FYI:
225	// "Real" wget 'http://busybox.net?var=a/b' sends this request:	// "Real" wget 'http://busybox.net?var=a/b' sends this request:
#	Line 326  static void parse_url(char *src_url, str	Line 251  static void parse_url(char *src_url, str
251	h->path = sp;	h->path = sp;
252	}	}
253
254		// We used to set h->user to NULL here, but this interferes
255		// with handling of code 302 ("object was moved")
256
257	sp = strrchr(h->host, '@');	sp = strrchr(h->host, '@');
	h->user = NULL;
258	if (sp != NULL) {	if (sp != NULL) {
259	h->user = h->host;	h->user = h->host;
260	*sp = '\0';	*sp = '\0';
#	Line 337  static void parse_url(char *src_url, str	Line 264  static void parse_url(char *src_url, str
264	sp = h->host;	sp = h->host;
265	}	}
266
267	static char *gethdr(char *buf, size_t bufsiz, FILE *fp /*, int *istrunc*/)	static char *gethdr(char *buf, size_t bufsiz, FILE *fp /*, int *istrunc*/)
268	{	{
269	char *s, *hdrval;	char *s, *hdrval;
#	Line 361  static char *gethdr(char *buf, size_t bu	Line 287  static char *gethdr(char *buf, size_t bu
287
288	/* verify we are at the end of the header name */	/* verify we are at the end of the header name */
289	if (*s != ':')	if (*s != ':')
290	bb_error_msg_and_die("bad header line: %s", buf);	bb_error_msg_and_die("bad header line: %s", sanitize_string(buf));
291
292	/* locate the start of the header value */	/* locate the start of the header value */
293	*s++ = '\0';	*s++ = '\0';
#	Line 377  static char *gethdr(char *buf, size_t bu	Line 303  static char *gethdr(char *buf, size_t bu
303	return hdrval;	return hdrval;
304	}	}
305
306	/* Rats! The buffer isn't big enough to hold the entire header value. */	/* Rats! The buffer isn't big enough to hold the entire header value */
307	while (c = getc(fp), c != EOF && c != '\n')	while (c = getc(fp), c != EOF && c != '\n')
308	continue;	continue;
309	/* *istrunc = 1; */	/* *istrunc = 1; */
310	return hdrval;	return hdrval;
311	}	}
312
313		#if ENABLE_FEATURE_WGET_LONG_OPTIONS
314		static char *URL_escape(const char *str)
315		{
316		/* URL encode, see RFC 2396 */
317		char *dst;
318		char *res = dst = xmalloc(strlen(str) * 3 + 1);
319		unsigned char c;
320
321		while (1) {
322		c = *str++;
323		if (c == '\0'
324		/* || strchr("!&'()*-.=_~", c) - more code */
325		|| c == '!'
326		|| c == '&'
327		|| c == '\''
328		|| c == '('
329		|| c == ')'
330		|| c == '*'
331		|| c == '-'
332		|| c == '.'
333		|| c == '='
334		|| c == '_'
335		|| c == '~'
336		|| (c >= '0' && c <= '9')
337		|| ((c|0x20) >= 'a' && (c|0x20) <= 'z')
338		) {
339		*dst++ = c;
340		if (c == '\0')
341		return res;
342		} else {
343		*dst++ = '%';
344		*dst++ = bb_hexdigits_upcase[c >> 4];
345		*dst++ = bb_hexdigits_upcase[c & 0xf];
346		}
347		}
348		}
349		#endif
350
351		static FILE* prepare_ftp_session(FILE **dfpp, struct host_info *target, len_and_sockaddr *lsa)
352		{
353		char buf[512];
354		FILE *sfp;
355		char *str;
356		int port;
357
358		if (!target->user)
359		target->user = xstrdup("anonymous:busybox@");
360
361		sfp = open_socket(lsa);
362		if (ftpcmd(NULL, NULL, sfp, buf) != 220)
363		bb_error_msg_and_die("%s", sanitize_string(buf+4));
364
365		/*
366		* Splitting username:password pair,
367		* trying to log in
368		*/
369		str = strchr(target->user, ':');
370		if (str)
371		*str++ = '\0';
372		switch (ftpcmd("USER ", target->user, sfp, buf)) {
373		case 230:
374		break;
375		case 331:
376		if (ftpcmd("PASS ", str, sfp, buf) == 230)
377		break;
378		/* fall through (failed login) */
379		default:
380		bb_error_msg_and_die("ftp login: %s", sanitize_string(buf+4));
381		}
382
383		ftpcmd("TYPE I", NULL, sfp, buf);
384
385		/*
386		* Querying file size
387		*/
388		if (ftpcmd("SIZE ", target->path, sfp, buf) == 213) {
389		G.content_len = BB_STRTOOFF(buf+4, NULL, 10);
390		if (G.content_len < 0 || errno) {
391		bb_error_msg_and_die("SIZE value is garbage");
392		}
393		G.got_clen = 1;
394		}
395
396		/*
397		* Entering passive mode
398		*/
399		if (ftpcmd("PASV", NULL, sfp, buf) != 227) {
400		pasv_error:
401		bb_error_msg_and_die("bad response to %s: %s", "PASV", sanitize_string(buf));
402		}
403		// Response is "227 garbageN1,N2,N3,N4,P1,P2[)garbage]
404		// Server's IP is N1.N2.N3.N4 (we ignore it)
405		// Server's port for data connection is P1*256+P2
406		str = strrchr(buf, ')');
407		if (str) str[0] = '\0';
408		str = strrchr(buf, ',');
409		if (!str) goto pasv_error;
410		port = xatou_range(str+1, 0, 255);
411		*str = '\0';
412		str = strrchr(buf, ',');
413		if (!str) goto pasv_error;
414		port += xatou_range(str+1, 0, 255) * 256;
415		set_nport(lsa, htons(port));
416
417		*dfpp = open_socket(lsa);
418
419		if (G.beg_range) {
420		sprintf(buf, "REST %"OFF_FMT"u", G.beg_range);
421		if (ftpcmd(buf, NULL, sfp, buf) == 350)
422		G.content_len -= G.beg_range;
423		}
424
425		if (ftpcmd("RETR ", target->path, sfp, buf) > 150)
426		bb_error_msg_and_die("bad response to %s: %s", "RETR", sanitize_string(buf));
427
428		return sfp;
429		}
430
431		/* Must match option string! */
432		enum {
433		WGET_OPT_CONTINUE   = (1 << 0),
434		WGET_OPT_SPIDER    = (1 << 1),
435		WGET_OPT_QUIET      = (1 << 2),
436		WGET_OPT_OUTNAME    = (1 << 3),
437		WGET_OPT_PREFIX     = (1 << 4),
438		WGET_OPT_PROXY      = (1 << 5),
439		WGET_OPT_USER_AGENT = (1 << 6),
440		WGET_OPT_RETRIES    = (1 << 7),
441		WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 8),
442		WGET_OPT_PASSIVE    = (1 << 9),
443		WGET_OPT_HEADER     = (1 << 10) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
444		WGET_OPT_POST_DATA  = (1 << 11) * ENABLE_FEATURE_WGET_LONG_OPTIONS,
445		};
446
447		static void NOINLINE retrieve_file_data(FILE *dfp, int output_fd)
448		{
449		char buf[512];
450
451		if (!(option_mask32 & WGET_OPT_QUIET))
452		progress_meter(-1);
453
454		if (G.chunked)
455		goto get_clen;
456
457		/* Loops only if chunked */
458		while (1) {
459		while (1) {
460		int n;
461		unsigned rdsz;
462
463		rdsz = sizeof(buf);
464		if (G.got_clen) {
465		if (G.content_len < (off_t)sizeof(buf)) {
466		if ((int)G.content_len <= 0)
467		break;
468		rdsz = (unsigned)G.content_len;
469		}
470		}
471		n = safe_fread(buf, rdsz, dfp);
472		if (n <= 0) {
473		if (ferror(dfp)) {
474		/* perror will not work: ferror doesn't set errno */
475		bb_error_msg_and_die(bb_msg_read_error);
476		}
477		break;
478		}
479		xwrite(output_fd, buf, n);
480		#if ENABLE_FEATURE_WGET_STATUSBAR
481		G.transferred += n;
482		#endif
483		if (G.got_clen)
484		G.content_len -= n;
485		}
486
487		if (!G.chunked)
488		break;
489
490		safe_fgets(buf, sizeof(buf), dfp); /* This is a newline */
491		get_clen:
492		safe_fgets(buf, sizeof(buf), dfp);
493		G.content_len = STRTOOFF(buf, NULL, 16);
494		/* FIXME: error check? */
495		if (G.content_len == 0)
496		break; /* all done! */
497		G.got_clen = 1;
498		}
499
500		if (!(option_mask32 & WGET_OPT_QUIET))
501		progress_meter(0);
502		}
503
504	int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;	int wget_main(int argc, char **argv) MAIN_EXTERNALLY_VISIBLE;
505	int wget_main(int argc UNUSED_PARAM, char **argv)	int wget_main(int argc UNUSED_PARAM, char **argv)
#	Line 391  int wget_main(int argc UNUSED_PARAM, cha	Line 507  int wget_main(int argc UNUSED_PARAM, cha
507	char buf[512];	char buf[512];
508	struct host_info server, target;	struct host_info server, target;
509	len_and_sockaddr *lsa;	len_and_sockaddr *lsa;
	int status;
	int port;
	int try = 5;
510	unsigned opt;	unsigned opt;
511	char *str;	int redir_limit;
512	char *proxy = 0;	char *proxy = NULL;
513	char *dir_prefix = NULL;	char *dir_prefix = NULL;
514	#if ENABLE_FEATURE_WGET_LONG_OPTIONS	#if ENABLE_FEATURE_WGET_LONG_OPTIONS
515		char *post_data;
516	char *extra_headers = NULL;	char *extra_headers = NULL;
517	llist_t *headers_llist = NULL;	llist_t *headers_llist = NULL;
518	#endif	#endif
519	FILE *sfp = NULL;               /* socket to web/ftp server         */	FILE *sfp;                      /* socket to web/ftp server         */
520	FILE *dfp;                      /* socket to ftp server (data)      */	FILE *dfp;                      /* socket to ftp server (data)      */
521	char *fname_out;                /* where to direct output (-O)      */	char *fname_out;                /* where to direct output (-O)      */
	bool got_clen = 0;              /* got content-length: from server  */
522	int output_fd = -1;	int output_fd = -1;
523	bool use_proxy = 1;             /* Use proxies if env vars are set  */	bool use_proxy;                 /* Use proxies if env vars are set  */
524	const char *proxy_flag = "on";  /* Use proxies if env vars are set  */	const char *proxy_flag = "on";  /* Use proxies if env vars are set  */
525	const char *user_agent = "Wget";/* "User-Agent" header field        */	const char *user_agent = "Wget";/* "User-Agent" header field        */
526
#	Line 416  int wget_main(int argc UNUSED_PARAM, cha	Line 529  int wget_main(int argc UNUSED_PARAM, cha
529	enum {	enum {
530	KEY_content_length = 1, KEY_transfer_encoding, KEY_chunked, KEY_location	KEY_content_length = 1, KEY_transfer_encoding, KEY_chunked, KEY_location
531	};	};
	enum {
	WGET_OPT_CONTINUE   = (1 << 0),
	WGET_OPT_SPIDER    = (1 << 1),
	WGET_OPT_QUIET      = (1 << 2),
	WGET_OPT_OUTNAME    = (1 << 3),
	WGET_OPT_PREFIX     = (1 << 4),
	WGET_OPT_PROXY      = (1 << 5),
	WGET_OPT_USER_AGENT = (1 << 6),
	WGET_OPT_RETRIES    = (1 << 7),
	WGET_OPT_NETWORK_READ_TIMEOUT = (1 << 8),
	WGET_OPT_PASSIVE    = (1 << 9),
	WGET_OPT_HEADER     = (1 << 10),
	};
532	#if ENABLE_FEATURE_WGET_LONG_OPTIONS	#if ENABLE_FEATURE_WGET_LONG_OPTIONS
533	static const char wget_longopts[] ALIGN1 =	static const char wget_longopts[] ALIGN1 =
534	/* name, has_arg, val */	/* name, has_arg, val */
#	Line 445  int wget_main(int argc UNUSED_PARAM, cha	Line 545  int wget_main(int argc UNUSED_PARAM, cha
545	/* Ignored (we always use PASV): */	/* Ignored (we always use PASV): */
546	"passive-ftp\0"      No_argument       "\xff"	"passive-ftp\0"      No_argument       "\xff"
547	"header\0"           Required_argument "\xfe"	"header\0"           Required_argument "\xfe"
548		"post-data\0"        Required_argument "\xfd"
549		/* Ignored (we don't do ssl) */
550		"no-check-certificate\0" No_argument   "\xfc"
551	;	;
552	#endif	#endif
553
#	Line 454  int wget_main(int argc UNUSED_PARAM, cha	Line 557  int wget_main(int argc UNUSED_PARAM, cha
557	applet_long_options = wget_longopts;	applet_long_options = wget_longopts;
558	#endif	#endif
559	/* server.allocated = target.allocated = NULL; */	/* server.allocated = target.allocated = NULL; */
560	opt_complementary = "-1" USE_FEATURE_WGET_LONG_OPTIONS(":\xfe::");	opt_complementary = "-1" IF_FEATURE_WGET_LONG_OPTIONS(":\xfe::");
561	opt = getopt32(argv, "csqO:P:Y:U:" /*ignored:*/ "t:T:",	opt = getopt32(argv, "csqO:P:Y:U:" /*ignored:*/ "t:T:",
562	&fname_out, &dir_prefix,	&fname_out, &dir_prefix,
563	&proxy_flag, &user_agent,	&proxy_flag, &user_agent,
564	NULL, /* -t RETRIES */	NULL, /* -t RETRIES */
565	NULL /* -T NETWORK_READ_TIMEOUT */	NULL /* -T NETWORK_READ_TIMEOUT */
566	USE_FEATURE_WGET_LONG_OPTIONS(, &headers_llist)	IF_FEATURE_WGET_LONG_OPTIONS(, &headers_llist)
567		IF_FEATURE_WGET_LONG_OPTIONS(, &post_data)
568	);	);
	if (strcmp(proxy_flag, "off") == 0) {
	/* Use the proxy if necessary */
	use_proxy = 0;
	}
569	#if ENABLE_FEATURE_WGET_LONG_OPTIONS	#if ENABLE_FEATURE_WGET_LONG_OPTIONS
570	if (headers_llist) {	if (headers_llist) {
571	int size = 1;	int size = 1;
#	Line 482  int wget_main(int argc UNUSED_PARAM, cha	Line 582  int wget_main(int argc UNUSED_PARAM, cha
582	}	}
583	#endif	#endif
584
585		/* TODO: compat issue: should handle "wget URL1 URL2..." */
586
587		target.user = NULL;
588	parse_url(argv[optind], &target);	parse_url(argv[optind], &target);
	server.host = target.host;
	server.port = target.port;
589
590	/* Use the proxy if necessary */	/* Use the proxy if necessary */
591		use_proxy = (strcmp(proxy_flag, "off") != 0);
592	if (use_proxy) {	if (use_proxy) {
593	proxy = getenv(target.is_ftp ? "ftp_proxy" : "http_proxy");	proxy = getenv(target.is_ftp ? "ftp_proxy" : "http_proxy");
594	if (proxy && *proxy) {	if (proxy && proxy[0]) {
595		server.user = NULL;
596	parse_url(proxy, &server);	parse_url(proxy, &server);
597	} else {	} else {
598	use_proxy = 0;	use_proxy = 0;
599	}	}
600	}	}
601		if (!use_proxy) {
602		server.port = target.port;
603		if (ENABLE_FEATURE_IPV6) {
604		server.host = xstrdup(target.host);
605		} else {
606		server.host = target.host;
607		}
608		}
609
610		if (ENABLE_FEATURE_IPV6)
611		strip_ipv6_scope_id(target.host);
612
613	/* Guess an output filename, if there was no -O FILE */	/* Guess an output filename, if there was no -O FILE */
614	if (!(opt & WGET_OPT_OUTNAME)) {	if (!(opt & WGET_OPT_OUTNAME)) {
#	Line 513  int wget_main(int argc UNUSED_PARAM, cha	Line 627  int wget_main(int argc UNUSED_PARAM, cha
627	}	}
628	}	}
629	#if ENABLE_FEATURE_WGET_STATUSBAR	#if ENABLE_FEATURE_WGET_STATUSBAR
630	curfile = bb_get_last_path_component_nostrip(fname_out);	G.curfile = bb_get_last_path_component_nostrip(fname_out);
631	#endif	#endif
632
633	/* Impossible?	/* Impossible?
634	if ((opt & WGET_OPT_CONTINUE) && !fname_out)	if ((opt & WGET_OPT_CONTINUE) && !fname_out)
635	bb_error_msg_and_die("cannot specify continue (-c) without a filename (-O)"); */	bb_error_msg_and_die("can't specify continue (-c) without a filename (-O)");
636		*/
637
638	/* Determine where to start transfer */	/* Determine where to start transfer */
639	if (opt & WGET_OPT_CONTINUE) {	if (opt & WGET_OPT_CONTINUE) {
640	output_fd = open(fname_out, O_WRONLY);	output_fd = open(fname_out, O_WRONLY);
641	if (output_fd >= 0) {	if (output_fd >= 0) {
642	beg_range = xlseek(output_fd, 0, SEEK_END);	G.beg_range = xlseek(output_fd, 0, SEEK_END);
643	}	}
644	/* File doesn't exist. We do not create file here yet.	/* File doesn't exist. We do not create file here yet.
645	We are not sure it exists on remove side */	* We are not sure it exists on remove side */
646	}	}
647
648	/* We want to do exactly _one_ DNS lookup, since some	redir_limit = 5;
649	* sites (i.e. ftp.us.debian.org) use round-robin DNS	resolve_lsa:
	* and we want to connect to only one IP... */
650	lsa = xhost2sockaddr(server.host, server.port);	lsa = xhost2sockaddr(server.host, server.port);
651	if (!(opt & WGET_OPT_QUIET)) {	if (!(opt & WGET_OPT_QUIET)) {
652	fprintf(stderr, "Connecting to %s (%s)\n", server.host,	char *s = xmalloc_sockaddr2dotted(&lsa->u.sa);
653	xmalloc_sockaddr2dotted(&lsa->u.sa));	fprintf(stderr, "Connecting to %s (%s)\n", server.host, s);
654	/* We leak result of xmalloc_sockaddr2dotted */	free(s);
655	}	}
656		establish_session:
657	if (use_proxy || !target.is_ftp) {	if (use_proxy || !target.is_ftp) {
658	/*	/*
659	*  HTTP session	*  HTTP session
660	*/	*/
661	do {	char *str;
662	got_clen = 0;	int status;
663	chunked = 0;
664		/* Open socket to http server */
665	if (!--try)	sfp = open_socket(lsa);
666	bb_error_msg_and_die("too many redirections");
667		/* Send HTTP request */
668	/* Open socket to http server */	if (use_proxy) {
669	if (sfp) fclose(sfp);	fprintf(sfp, "GET %stp://%s/%s HTTP/1.1\r\n",
670	sfp = open_socket(lsa);	target.is_ftp ? "f" : "ht", target.host,
671		target.path);
672	/* Send HTTP request.  */	} else {
673	if (use_proxy) {	if (opt & WGET_OPT_POST_DATA)
674	fprintf(sfp, "GET %stp://%s/%s HTTP/1.1\r\n",	fprintf(sfp, "POST /%s HTTP/1.1\r\n", target.path);
675	target.is_ftp ? "f" : "ht", target.host,	else
	target.path);
	} else {
676	fprintf(sfp, "GET /%s HTTP/1.1\r\n", target.path);	fprintf(sfp, "GET /%s HTTP/1.1\r\n", target.path);
677	}	}
678
679	fprintf(sfp, "Host: %s\r\nUser-Agent: %s\r\n",	fprintf(sfp, "Host: %s\r\nUser-Agent: %s\r\n",
680	target.host, user_agent);	target.host, user_agent);
681
682	#if ENABLE_FEATURE_WGET_AUTHENTICATION	#if ENABLE_FEATURE_WGET_AUTHENTICATION
683	if (target.user) {	if (target.user) {
684	fprintf(sfp, "Proxy-Authorization: Basic %s\r\n"+6,	fprintf(sfp, "Proxy-Authorization: Basic %s\r\n"+6,
685	base64enc_512(buf, target.user));	base64enc_512(buf, target.user));
686	}	}
687	if (use_proxy && server.user) {	if (use_proxy && server.user) {
688	fprintf(sfp, "Proxy-Authorization: Basic %s\r\n",	fprintf(sfp, "Proxy-Authorization: Basic %s\r\n",
689	base64enc_512(buf, server.user));	base64enc_512(buf, server.user));
690	}	}
691	#endif	#endif
692
693	if (beg_range)	if (G.beg_range)
694	fprintf(sfp, "Range: bytes=%"OFF_FMT"d-\r\n", beg_range);	fprintf(sfp, "Range: bytes=%"OFF_FMT"u-\r\n", G.beg_range);
695	#if ENABLE_FEATURE_WGET_LONG_OPTIONS	#if ENABLE_FEATURE_WGET_LONG_OPTIONS
696	if (extra_headers)	if (extra_headers)
697	fputs(extra_headers, sfp);	fputs(extra_headers, sfp);
698
699		if (opt & WGET_OPT_POST_DATA) {
700		char *estr = URL_escape(post_data);
701		fprintf(sfp, "Content-Type: application/x-www-form-urlencoded\r\n");
702		fprintf(sfp, "Content-Length: %u\r\n" "\r\n" "%s",
703		(int) strlen(estr), estr);
704		/*fprintf(sfp, "Connection: Keep-Alive\r\n\r\n");*/
705		/*fprintf(sfp, "%s\r\n", estr);*/
706		free(estr);
707		} else
708	#endif	#endif
709	fprintf(sfp, "Connection: close\r\n\r\n");	{ /* If "Connection:" is needed, document why */
710		fprintf(sfp, /* "Connection: close\r\n" */ "\r\n");
711		}
712
713	/*	/*
714	* Retrieve HTTP response line and check for "200" status code.	* Retrieve HTTP response line and check for "200" status code.
715	*/	*/
716	read_response:	read_response:
717	if (fgets(buf, sizeof(buf), sfp) == NULL)	if (fgets(buf, sizeof(buf), sfp) == NULL)
718	bb_error_msg_and_die("no response from server");	bb_error_msg_and_die("no response from server");
719
720	str = buf;	str = buf;
721	str = skip_non_whitespace(str);	str = skip_non_whitespace(str);
722	str = skip_whitespace(str);	str = skip_whitespace(str);
723	// FIXME: no error check	// FIXME: no error check
724	// xatou wouldn't work: "200 OK"	// xatou wouldn't work: "200 OK"
725	status = atoi(str);	status = atoi(str);
726	switch (status) {	switch (status) {
727	case 0:	case 0:
728	case 100:	case 100:
729	while (gethdr(buf, sizeof(buf), sfp /*, &n*/) != NULL)	while (gethdr(buf, sizeof(buf), sfp /*, &n*/) != NULL)
730	/* eat all remaining headers */;	/* eat all remaining headers */;
731	goto read_response;	goto read_response;
732	case 200:	case 200:
733	/*	/*
734	Response 204 doesn't say "null file", it says "metadata	Response 204 doesn't say "null file", it says "metadata
735	has changed but data didn't":	has changed but data didn't":
#	Line 630  is always terminated by the first empty	Line 754  is always terminated by the first empty
754	However, in real world it was observed that some web servers	However, in real world it was observed that some web servers
755	(e.g. Boa/0.94.14rc21) simply use code 204 when file size is zero.	(e.g. Boa/0.94.14rc21) simply use code 204 when file size is zero.
756	*/	*/
757	case 204:	case 204:
758	break;	break;
759	case 300: /* redirection */	case 300: /* redirection */
760	case 301:	case 301:
761	case 302:	case 302:
762	case 303:	case 303:
763		break;
764		case 206:
765		if (G.beg_range)
766	break;	break;
767	case 206:	/* fall through */
768	if (beg_range)	default:
769	break;	bb_error_msg_and_die("server returned error: %s", sanitize_string(buf));
770	/* fall through */	}
	default:
	/* Show first line only and kill any ESC tricks */
	buf[strcspn(buf, "\n\r\x1b")] = '\0';
	bb_error_msg_and_die("server returned error: %s", buf);
	}
771
772	/*	/*
773	* Retrieve HTTP headers.	* Retrieve HTTP headers.
774	*/	*/
775	while ((str = gethdr(buf, sizeof(buf), sfp /*, &n*/)) != NULL) {	while ((str = gethdr(buf, sizeof(buf), sfp /*, &n*/)) != NULL) {
776	/* gethdr did already convert the "FOO:" string to lowercase */	/* gethdr converted "FOO:" string to lowercase */
777	smalluint key = index_in_strings(keywords, *&buf) + 1;	smalluint key;
778	if (key == KEY_content_length) {	/* strip trailing whitespace */
779	content_len = BB_STRTOOFF(str, NULL, 10);	char *s = strchrnul(str, '\0') - 1;
780	if (errno || content_len < 0) {	while (s >= str && (*s == ' ' || *s == '\t')) {
781	bb_error_msg_and_die("content-length %s is garbage", str);	*s = '\0';
782	}	s--;
783	got_clen = 1;	}
784	continue;	key = index_in_strings(keywords, buf) + 1;
785	}	if (key == KEY_content_length) {
786	if (key == KEY_transfer_encoding) {	G.content_len = BB_STRTOOFF(str, NULL, 10);
787	if (index_in_strings(keywords, str_tolower(str)) + 1 != KEY_chunked)	if (G.content_len < 0 || errno) {
788	bb_error_msg_and_die("transfer encoding '%s' is not supported", str);	bb_error_msg_and_die("content-length %s is garbage", sanitize_string(str));
	chunked = got_clen = 1;
789	}	}
790	if (key == KEY_location) {	G.got_clen = 1;
791	if (str[0] == '/')	continue;
792	/* free(target.allocated); */	}
793	target.path = /* target.allocated = */ xstrdup(str+1);	if (key == KEY_transfer_encoding) {
794	else {	if (index_in_strings(keywords, str_tolower(str)) + 1 != KEY_chunked)
795	parse_url(str, &target);	bb_error_msg_and_die("transfer encoding '%s' is not supported", sanitize_string(str));
796	if (use_proxy == 0) {	G.chunked = G.got_clen = 1;
797	server.host = target.host;	}
798	server.port = target.port;	if (key == KEY_location && status >= 300) {
799	}	if (--redir_limit == 0)
800		bb_error_msg_and_die("too many redirections");
801		fclose(sfp);
802		G.got_clen = 0;
803		G.chunked = 0;
804		if (str[0] == '/')
805		/* free(target.allocated); */
806		target.path = /* target.allocated = */ xstrdup(str+1);
807		/* lsa stays the same: it's on the same server */
808		else {
809		parse_url(str, &target);
810		if (!use_proxy) {
811		server.host = target.host;
812		/* strip_ipv6_scope_id(target.host); - no! */
813		/* we assume remote never gives us IPv6 addr with scope id */
814		server.port = target.port;
815	free(lsa);	free(lsa);
816	lsa = xhost2sockaddr(server.host, server.port);	goto resolve_lsa;
817	break;	} /* else: lsa stays the same: we use proxy */
	}
818	}	}
819		goto establish_session;
820	}	}
821	} while (status >= 300);	}
822		// if (status >= 300)
823		// bb_error_msg_and_die("bad redirection (no Location: header from server)");
824
825		/* For HTTP, data is pumped over the same connection */
826	dfp = sfp;	dfp = sfp;
827
828	} else {	} else {
829	/*	/*
830	*  FTP session	*  FTP session
831	*/	*/
832	if (!target.user)	sfp = prepare_ftp_session(&dfp, &target, lsa);
	target.user = xstrdup("anonymous:busybox@");
	sfp = open_socket(lsa);
	if (ftpcmd(NULL, NULL, sfp, buf) != 220)
	bb_error_msg_and_die("%s", buf+4);
	/*
	* Splitting username:password pair,
	* trying to log in
	*/
	str = strchr(target.user, ':');
	if (str)
	*(str++) = '\0';
	switch (ftpcmd("USER ", target.user, sfp, buf)) {
	case 230:
	break;
	case 331:
	if (ftpcmd("PASS ", str, sfp, buf) == 230)
	break;
	/* fall through (failed login) */
	default:
	bb_error_msg_and_die("ftp login: %s", buf+4);
	}
	ftpcmd("TYPE I", NULL, sfp, buf);
	/*
	* Querying file size
	*/
	if (ftpcmd("SIZE ", target.path, sfp, buf) == 213) {
	content_len = BB_STRTOOFF(buf+4, NULL, 10);
	if (errno || content_len < 0) {
	bb_error_msg_and_die("SIZE value is garbage");
	}
	got_clen = 1;
	}
	/*
	* Entering passive mode
	*/
	if (ftpcmd("PASV", NULL, sfp, buf) != 227) {
	pasv_error:
	bb_error_msg_and_die("bad response to %s: %s", "PASV", buf);
	}
	// Response is "227 garbageN1,N2,N3,N4,P1,P2[)garbage]
	// Server's IP is N1.N2.N3.N4 (we ignore it)
	// Server's port for data connection is P1*256+P2
	str = strrchr(buf, ')');
	if (str) str[0] = '\0';
	str = strrchr(buf, ',');
	if (!str) goto pasv_error;
	port = xatou_range(str+1, 0, 255);
	*str = '\0';
	str = strrchr(buf, ',');
	if (!str) goto pasv_error;
	port += xatou_range(str+1, 0, 255) * 256;
	set_nport(lsa, htons(port));
	dfp = open_socket(lsa);
	if (beg_range) {
	sprintf(buf, "REST %"OFF_FMT"d", beg_range);
	if (ftpcmd(buf, NULL, sfp, buf) == 350)
	content_len -= beg_range;
	}
	if (ftpcmd("RETR ", target.path, sfp, buf) > 150)
	bb_error_msg_and_die("bad response to %s: %s", "RETR", buf);
833	}	}
834
835	if (opt & WGET_OPT_SPIDER) {	if (opt & WGET_OPT_SPIDER) {
#	Line 767  However, in real world it was observed t	Line 838  However, in real world it was observed t
838	return EXIT_SUCCESS;	return EXIT_SUCCESS;
839	}	}
840
	/*
	* Retrieve file
	*/
	/* Do it before progressmeter (want to have nice error message) */
841	if (output_fd < 0) {	if (output_fd < 0) {
842	int o_flags = O_WRONLY | O_CREAT | O_TRUNC | O_EXCL;	int o_flags = O_WRONLY | O_CREAT | O_TRUNC | O_EXCL;
843	/* compat with wget: -O FILE can overwrite */	/* compat with wget: -O FILE can overwrite */
#	Line 780  However, in real world it was observed t	Line 846  However, in real world it was observed t
846	output_fd = xopen(fname_out, o_flags);	output_fd = xopen(fname_out, o_flags);
847	}	}
848
849	if (!(opt & WGET_OPT_QUIET))	retrieve_file_data(dfp, output_fd);
850	progressmeter(-1);	xclose(output_fd);
	if (chunked)
	goto get_clen;
	/* Loops only if chunked */
	while (1) {
	while (content_len > 0 || !got_clen) {
	int n;
	unsigned rdsz = sizeof(buf);
	if (content_len < sizeof(buf) && (chunked || got_clen))
	rdsz = (unsigned)content_len;
	n = safe_fread(buf, rdsz, dfp);
	if (n <= 0) {
	if (ferror(dfp)) {
	/* perror will not work: ferror doesn't set errno */
	bb_error_msg_and_die(bb_msg_read_error);
	}
	break;
	}
	xwrite(output_fd, buf, n);
	#if ENABLE_FEATURE_WGET_STATUSBAR
	transferred += n;
	#endif
	if (got_clen)
	content_len -= n;
	}
	if (!chunked)
	break;
	safe_fgets(buf, sizeof(buf), dfp); /* This is a newline */
	get_clen:
	safe_fgets(buf, sizeof(buf), dfp);
	content_len = STRTOOFF(buf, NULL, 16);
	/* FIXME: error check? */
	if (content_len == 0)
	break; /* all done! */
	}
	if (!(opt & WGET_OPT_QUIET))
	progressmeter(0);
851
852	if ((use_proxy == 0) && target.is_ftp) {	if (dfp != sfp) {
853		/* It's ftp. Close it properly */
854	fclose(dfp);	fclose(dfp);
855	if (ftpcmd(NULL, NULL, sfp, buf) != 226)	if (ftpcmd(NULL, NULL, sfp, buf) != 226)
856	bb_error_msg_and_die("ftp error: %s", buf+4);	bb_error_msg_and_die("ftp error: %s", sanitize_string(buf+4));
857	ftpcmd("QUIT", NULL, sfp, buf);	/* ftpcmd("QUIT", NULL, sfp, buf); - why bother? */
858	}	}
859
860	return EXIT_SUCCESS;	return EXIT_SUCCESS;

---

Colored Diff Long Colored Diff Full Colored Diff Unidiff Context Diff Side by Side

Legend: Removed from v.983   changed lines   Added in v.984