WvStreams
wvftpstream.cc
1/*
2 * Worldvisions Weaver Software:
3 * Copyright (C) 1997-2002 Net Integration Technologies, Inc.
4 *
5 * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever.
6 *
7 * See wvhttppool.h.
8 */
9
10#ifdef __GNUC__
11# define alloca __builtin_alloca
12#else
13# ifdef _MSC_VER
14# include <malloc.h>
15# define alloca _alloca
16# else
17# if HAVE_ALLOCA_H
18# include <alloca.h>
19# else
20# ifdef _AIX
21#pragma alloca
22# else
23# ifndef alloca /* predefined by HP cc +Olibcalls */
24char *alloca ();
25# endif
26# endif
27# endif
28# endif
29#endif
30
31#include <ctype.h>
32#include <time.h>
33#include "wvhttppool.h"
34#include "wvbufstream.h"
35#include "wvtcp.h"
36#include "wvsslstream.h"
37#include "strutils.h"
38#include <stdlib.h> // for alloca()... FIXME: which we shouldn't be using!
39
40WvFtpStream::WvFtpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
41 WvStringParm _password)
42 : WvUrlStream(_remaddr, _username, WvString("FTP %s", _remaddr)),
43 cont(wv::bind(&WvFtpStream::real_execute, this, _1))
44{
45 data = NULL;
46 logged_in = false;
47 password = _password;
48 last_request_time = time(0);
49 alarm(60000); // timeout if no connection, or something goes wrong
50}
51
52
53void WvFtpStream::doneurl()
54{
55 log("Done URL: %s\n", curl->url);
56
57 curl->done();
58 curl = NULL;
59 WVRELEASE(data);
60 urls.unlink_first();
61 last_request_time = time(0);
62 alarm(60000);
63 request_next();
64 // We just processed the last url in the queue,
65 // so go away.
66 if (urls.isempty() && waiting_urls.isempty())
67 close();
68}
69
70
71void WvFtpStream::request_next()
72{
73 // don't do a request if we've done too many already or we have none
74 // waiting.
75 if (request_count >= max_requests || waiting_urls.isempty())
76 return;
77
78 if (!urls.isempty())
79 return;
80
81 // okay then, we really do want to send a new request.
82 WvUrlRequest *url = waiting_urls.first();
83
84 waiting_urls.unlink_first();
85
86 request_count++;
87 log("Request #%s: %s\n", request_count, url->url);
88 urls.append(url, false, "request_url");
89 alarm(0);
90}
91
92
94{
95 if (isok())
96 log("Closing.\n");
98
99 if (geterr())
100 {
101 // if there was an error, count the first URL as done. This prevents
102 // retrying indefinitely.
103 if (!curl && !urls.isempty())
104 curl = urls.first();
105 if (!curl && !waiting_urls.isempty())
106 curl = waiting_urls.first();
107 if (curl)
108 log("URL '%s' is FAILED\n", curl->url);
109 if (curl)
110 curl->done();
111 }
112
113 if (curl)
114 curl->done();
115}
116
117
118char *WvFtpStream::get_important_line()
119{
120 char *line;
121 do
122 {
123 line = blocking_getline(-1);
124 if (!line)
125 return NULL;
126 }
127 while (line[3] == '-');
128 log(WvLog::Debug5, ">> %s\n", line);
129 return line;
130}
131
132
134{
135 SelectRequest oldwant = si.wants;
136
138
139 if (data)
140 data->pre_select(si);
141
142 if (curl && curl->putstream)
143 curl->putstream->pre_select(si);
144
145 si.wants = oldwant;
146}
147
148
150{
151 SelectRequest oldwant = si.wants;
152
154 return true;
155
156 if (data && data->post_select(si))
157 return true;
158
159 if (curl && curl->putstream && curl->putstream->post_select(si))
160 return true;
161
162 si.wants = oldwant;
163
164 return false;
165}
166
167
168void *WvFtpStream::real_execute(void*)
169{
170 WvString line;
172
173 if (alarm_was_ticking && ((last_request_time + 60) <= time(0)))
174 {
175 log(WvLog::Debug4, "urls count: %s\n", urls.count());
176 if (urls.isempty())
177 close(); // timed out, but not really an error
178
179 return 0;
180 }
181
182 if (!logged_in)
183 {
184 line = get_important_line();
185 if (!line)
186 {
187 seterr("Server not reachable: %s\n",strerror(errno));
188 return 0;
189 }
190
191 if (strncmp(line, "220", 3))
192 {
193 log("Server rejected connection: %s\n", line);
194 seterr("server rejected connection");
195 return 0;
196 }
197 print("USER %s\r\n", !target.username ? WvString("anonymous") :
198 target.username);
199 line = get_important_line();
200 if (!line)
201 return 0;
202
203 if (!strncmp(line, "230", 3))
204 {
205 log(WvLog::Info, "Server doesn't need password.\n");
206 logged_in = true; // No password needed;
207 }
208 else if (!strncmp(line, "33", 2))
209 {
210 print("PASS %s\r\n", !password ? DEFAULT_ANON_PW : password);
211
212 line = get_important_line();
213 if (!line)
214 return 0;
215
216 if (line[0] == '2')
217 {
218 log(WvLog::Info, "Authenticated.\n");
219 logged_in = true;
220 }
221 else
222 {
223 log("Strange response to PASS command: %s\n", line);
224 seterr("strange response to PASS command");
225 return 0;
226 }
227 }
228 else
229 {
230 log("Strange response to USER command: %s\n", line);
231 seterr("strange response to USER command");
232 return 0;
233 }
234
235 print("TYPE I\r\n");
236 log(WvLog::Debug5, "<< TYPE I\n");
237 line = get_important_line();
238 if (!line)
239 return 0;
240
241 if (strncmp(line, "200", 3))
242 {
243 log("Strange response to TYPE I command: %s\n", line);
244 seterr("strange response to TYPE I command");
245 return 0;
246 }
247 }
248
249 if (!curl && !urls.isempty())
250 {
251 curl = urls.first();
252
253 print("CWD %s\r\n", curl->url.getfile());
254 line = get_important_line();
255 if (!line)
256 return 0;
257
258 if (!strncmp(line, "250", 3))
259 {
260 log(WvLog::Debug5, "This is a directory.\n");
261 curl->is_dir = true;
262 }
263
264 print("PASV\r\n");
265 line = get_important_line();
266 if (!line)
267 return 0;
268 WvIPPortAddr *dataip = parse_pasv_response(line.edit());
269
270 if (!dataip)
271 return 0;
272
273 log(WvLog::Debug4, "Data port is %s.\n", *dataip);
274 // Open data connection.
275 data = new WvTCPConn(*dataip);
276 if (!data)
277 {
278 log("Can't open data connection.\n");
279 seterr("can't open data connection");
280 return 0;
281 }
282
283 if (curl->is_dir)
284 {
285 if (!curl->putstream)
286 {
287 print("LIST %s\r\n", curl->url.getfile());
288 if (curl->outstream)
289 {
290 WvString url_no_pw("ftp://%s%s%s%s", curl->url.getuser(),
291 !!curl->url.getuser() ? "@" : "",
292 curl->url.gethost(),
293 curl->url.getfile());
294 curl->outstream->print("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML "
295 "4.01//EN\">\n"
296 "<html>\n<head>\n<title>%s</title>\n"
297 "<meta http-equiv=\"Content-Type\" "
298 "content=\"text/html; "
299 "charset=ISO-8859-1\">\n"
300 "<base href=\"%s\"/>\n</head>\n"
301 "<style type=\"text/css\">\n"
302 "img { border: 0; padding: 0 2px; vertical-align: "
303 "text-bottom; }\n"
304 "td { font-family: monospace; padding: 2px 3px; "
305 "text-align: right; vertical-align: bottom; }\n"
306 "td:first-child { text-align: left; padding: "
307 "2px 10px 2px 3px; }\n"
308 "table { border: 0; }\n"
309 "a.symlink { font-style: italic; }\n"
310 "</style>\n<body>\n"
311 "<h1>Index of %s</h1>\n"
312 "<hr/><table>\n", url_no_pw, curl->url, url_no_pw
313 );
314 }
315 }
316 else
317 {
318 log("Target is a directory.\n");
319 seterr("target is a directory");
320 doneurl();
321 return 0;
322 }
323 }
324 else if (!curl->putstream)
325 print("RETR %s\r\n", curl->url.getfile());
326 else
327 {
328 if (curl->create_dirs)
329 {
330 print("CWD %s\r\n", getdirname(curl->url.getfile()));
331 line = get_important_line();
332 if (!line)
333 return 0;
334 if (strncmp(line, "250", 3))
335 {
336 log("Path doesn't exist; creating directories...\n");
337 // create missing directories.
338 WvString current_dir("");
339 WvStringList dirs;
340 dirs.split(getdirname(curl->url.getfile()), "/");
341 WvStringList::Iter i(dirs);
342 for (i.rewind(); i.next(); )
343 {
344 current_dir.append(WvString("/%s", i()));
345 print("MKD %s\r\n", current_dir);
346 line = get_important_line();
347 if (!line)
348 return 0;
349 }
350 }
351 }
352 print("STOR %s\r\n", curl->url.getfile());
353 }
354
355 log(WvLog::Debug5, "Waiting for response to %s\n", curl->putstream ? "STOR" :
356 curl->is_dir ? "LIST" : "RETR");
357 line = get_important_line();
358
359 if (!line)
360 doneurl();
361 else if (strncmp(line, "150", 3))
362 {
363 log("Strange response to %s command: %s\n",
364 curl->putstream ? "STOR" : "RETR", line);
365 seterr(WvString("strange response to %s command",
366 curl->putstream ? "STOR" : "RETR"));
367 doneurl();
368 }
369
370 }
371
372 if (curl)
373 {
374 if (curl->is_dir)
375 {
376 line = data->blocking_getline(-1);
377 if (line && curl->outstream)
378 {
379 WvString output_line(parse_for_links(line.edit()));
380 if (!!output_line)
381 curl->outstream->write(output_line);
382 else
383 curl->outstream->write("Unknown format of LIST "
384 "response\n");
385 }
386 }
387 else
388 {
389 char buf[1024];
390
391 if (curl->putstream)
392 {
393 while (curl->putstream->isreadable())
394 {
395 int len = curl->putstream->read(buf, sizeof(buf));
396 log(WvLog::Debug5, "Read %s bytes.\n%s\n", len, hexdump_buffer(buf, len));
397
398 if (len)
399 {
400 int wrote = data->write(buf, len);
401 log(WvLog::Debug5,"Wrote %s bytes\n", wrote);
402 data->flush(0);
403 }
404 }
405 curl->putstream->close();
406 }
407 else
408 {
409 while (data->isreadable() && curl->outstream->isok())
410 {
411 int len = data->read(buf, sizeof(buf));
412 log(WvLog::Debug5, "Read %s bytes from remote.\n", len);
413
414 if (len && curl->outstream)
415 {
416 int wrote = curl->outstream->write(buf, len);
417 log(WvLog::Debug5, "Wrote %s bytes to local.\n", wrote);
418 }
419 }
420 }
421 }
422
423 if (!data->isok() || (curl->putstream && !curl->putstream->isok()))
424 {
425 log("OK, we should have finished writing!\n");
426 if (curl->putstream && data->isok())
427 data->close();
428 line = get_important_line();
429 if (!line)
430 {
431 doneurl();
432 return 0;
433 }
434
435 if (strncmp(line, "226", 3))
436 log("Unexpected message: %s\n", line);
437
438 if (curl->is_dir)
439 {
440 if (curl->outstream)
441 curl->outstream->write("</table><hr/></body>\n"
442 "</html>\n");
443 write("CWD /\r\n");
444 log(WvLog::Debug5, "Waiting for response to CWD /\n");
445 line = get_important_line();
446 if (!line)
447 return 0;
448
449 if (strncmp(line, "250", 3))
450 log("Strange resonse to \"CWD /\": %s\n", line);
451 // Don't bother failing here.
452 }
453 doneurl();
454 }
455 else
456 {
457 log("Why are we here??\n");
458 }
459 }
460
461 return 0;
462}
463
464
466{
467 real_execute(0);
468}
469
470
471WvString WvFtpStream::parse_for_links(char *line)
472{
473 WvString output_line("");
474 trim_string(line);
475
476 if (curl->is_dir && curl->outstream)
477 {
478 struct ftpparse fp;
479 int res = ftpparse(&fp, line, strlen(line));
480 if (res)
481 {
482 char *linkname = (char *)alloca(fp.namelen+1);
483 int i;
484 for (i = 0; i < fp.namelen; i++)
485 {
486 if (fp.name[i] >= 32)
487 linkname[i] = fp.name[i];
488 else
489 {
490 linkname[i] = '?';
491 }
492 }
493 linkname[i] = 0;
494
495 WvString linkurl(curl->url);
496 if (linkurl.cstr()[linkurl.len()-1] != '/')
497 linkurl.append("/");
498 linkurl.append(linkname);
499 WvUrlLink *link = new WvUrlLink(linkname, linkurl);
500 curl->outstream->links.append(link, true);
501
502 output_line.append("<tr>\n");
503
504 output_line.append(WvString(" <td>%s%s</td>\n", linkname,
505 fp.flagtrycwd ? "/" : ""));
506
507 if (fp.flagtryretr)
508 {
509 if (!fp.sizetype)
510 output_line.append(" <td>? bytes</td>\n");
511 else
512 output_line.append(WvString(" <td>%s bytes</td>\n",
513 fp.size));
514 if (fp.mtimetype > 0)
515 output_line.append(WvString(" <td>%s</td>\n", (fp.mtime)));
516 else
517 output_line.append(" <td>?</td>\n");
518 }
519 else
520 output_line.append(" <td></td>\n");
521
522 output_line.append("</tr>\n");
523 }
524 }
525 return output_line;
526}
527
528
529WvIPPortAddr *WvFtpStream::parse_pasv_response(char *line)
530{
531 if (strncmp(line, "227 ", 4))
532 {
533 log("Strange response to PASV command: %s\n", line);
534 seterr("strange response to PASV command");
535 return NULL;
536 }
537
538 char *p = &line[3];
539 while (!isdigit(*p))
540 {
541 if (*p == '\0' || *p == '\r' || *p == '\n')
542 {
543 log("Couldn't parse PASV response: %s\n", line);
544 seterr("couldn't parse response to PASV command");
545 return NULL;
546 }
547 p++;
548 }
549 char *ipstart = p;
550
551 for (int i = 0; i < 4; i++)
552 {
553 p = strchr(p, ',');
554 if (!p)
555 {
556 log("Couldn't parse PASV IP: %s\n", line);
557 seterr("couldn't parse PASV IP");
558 return NULL;
559 }
560 *p = '.';
561 }
562 *p = '\0';
563 WvString pasvip(ipstart);
564 p++;
565 int pasvport;
566 pasvport = atoi(p)*256;
567 p = strchr(p, ',');
568 if (!p)
569 {
570 log("Couldn't parse PASV IP port: %s\n", line);
571 seterr("couldn't parse PASV IP port");
572 return NULL;
573 }
574 pasvport += atoi(++p);
575
576 WvIPPortAddr *res = new WvIPPortAddr(pasvip.cstr(), pasvport);
577
578 return res;
579}
virtual bool isok() const
return true if the stream is actually usable right now
static WvString strerror(int errnum)
A replacement for the operating system strerror() function that can map more kinds of error strings (...
Definition wverror.cc:91
A WvFastString acts exactly like a WvString, but can take (const char *) strings without needing to a...
Definition wvstring.h:94
virtual void close()
Closes the file descriptors.
virtual void close()
Close this stream.
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling select().
virtual bool post_select(SelectInfo &si)
post_select() is called after select(), and returns true if this object is now ready.
An IP+Port address also includes a port number, with the resulting form www.xxx.yyy....
Definition wvaddr.h:394
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling select().
virtual void close()
Close this stream.
virtual bool isok() const
return true if the stream is actually usable right now
virtual void execute()
The callback() function calls execute(), and then calls the user- specified callback if one is define...
virtual bool post_select(SelectInfo &si)
post_select() is called after select(), and returns true if this object is now ready.
virtual int geterr() const
If isok() is false, return the system error number corresponding to the error, -1 for a special error...
virtual bool post_select(SelectInfo &si)
post_select() is called after select(), and returns true if this object is now ready.
Definition wvstream.cc:875
virtual bool flush(time_t msec_timeout)
flush the output buffer, if we can do it without delaying more than msec_timeout milliseconds at a ti...
Definition wvstream.cc:707
char * blocking_getline(time_t wait_msec, int separator='\n', int readahead=1024)
This is a version of getline() that allows you to block for more data to arrive.
Definition wvstream.cc:602
virtual bool isreadable()
Returns true if the stream is readable.
Definition wvstream.cc:590
virtual bool isok() const
return true if the stream is actually usable right now
Definition wvstream.cc:445
void alarm(time_t msec_timeout)
set an alarm, ie.
Definition wvstream.cc:1048
virtual size_t write(const void *buf, size_t count)
Write data to the stream.
Definition wvstream.cc:532
virtual void pre_select(SelectInfo &si)
pre_select() sets up for eventually calling select().
Definition wvstream.cc:844
virtual size_t read(void *buf, size_t count)
read a data block on the stream.
Definition wvstream.cc:490
virtual void close()
Close the stream if it is open; isok() becomes false from now on.
Definition wvstream.cc:341
virtual void seterr(int _errnum)
Override seterr() from WvError so that it auto-closes the stream.
Definition wvstream.cc:451
bool alarm_was_ticking
This will be true during callback execution if the callback was triggered by the alarm going off.
Definition wvstream.h:54
This is a WvList of WvStrings, and is a really handy way to parse strings.
void split(WvStringParm s, const char *splitchars=" \t\r\n", int limit=0)
split s and form a list ignoring splitchars (except at beginning and end) ie.
WvString is an implementation of a simple and efficient printable-string class.
Definition wvstring.h:330
char * edit()
make the string editable, and return a non-const (char*)
Definition wvstring.h:397
WvTCPConn tries to make all outgoing connections asynchronously (in the background).
Definition wvtcp.h:40
virtual bool isok() const
Is this connection OK? Note: isok() will always be true if !resolved, even though fd==-1.
Definition wvtcp.cc:375
virtual bool post_select(SelectInfo &si)
override post_select() to set the 'connected' variable as soon as we are connected.
Definition wvtcp.cc:320
virtual void pre_select(SelectInfo &si)
override pre_select() to cause select() results when resolving names.
Definition wvtcp.cc:291
the data structure used by pre_select()/post_select() and internally by select().
Definition iwvstream.h:50
A SelectRequest is a convenient way to remember what we want to do to a particular stream: read from ...
Definition iwvstream.h:34
WvString hexdump_buffer(const void *buf, size_t len, bool charRep=true)
Produce a hexadecimal dump of the data buffer in 'buf' of length 'len'.
Definition strutils.cc:245
char * trim_string(char *string)
Trims whitespace from the beginning and end of the character string, including carriage return / line...
Definition strutils.cc:59