include/wvhttppool.h

   1 /* -*- Mode: C++ -*-
   2  * Worldvisions Weaver Software:
   3  *   Copyright (C) 1997-2002 Net Integration Technologies, Inc.
   4  *
   5  * A fast, easy-to-use, parallelizing, pipelining HTTP/1.1 file retriever.
   6  *
   7  * Just create a WvHttpPool object, add it to your list, and use pool.addurl()
   8  * to get a WvStream* that gives you the file you requested.
   9  */
  10 #ifndef __WVHTTPPOOL_H
  11 #define __WVHTTPPOOL_H
  12
  13 #include "ftpparse.h"
  14 #include "wvurl.h"
  15 #include "wvistreamlist.h"
  16 #include "wvstreamclone.h"
  17 #include "wvlog.h"
  18 #include "wvhashtable.h"
  19 #include "wvbufstream.h"
  20 #include "wvbuf.h"
  21 #include "wvcont.h"
  22 #include "wvtcp.h"
  23
  24 class WvBufUrlStream;
  25 class WvUrlStream;
  26 class WvHttpStream;
  27
  28 static const WvString DEFAULT_ANON_PW("weasels@");
  29
  30 struct WvHTTPHeader
  31 {
  32     WvString name, value;
  33
  34     WvHTTPHeader(WvStringParm _name, WvStringParm _value)
  35         : name(_name), value(_value)
  36                 {}
  37 };
  38
  39
  40 DeclareWvDict(WvHTTPHeader, WvString, name);
  41
  42
  43 class WvUrlRequest
  44 {
  45 public:
  46     WvUrl url;
  47     WvString headers;
  48     WvUrlStream *instream;
  49     WvBufUrlStream *outstream;
  50     WvStream *putstream;
  51
  52     bool pipeline_test;
  53     bool inuse;
  54     bool is_dir;
  55     bool create_dirs;
  56     WvString method;
  57
  58     WvUrlRequest(WvStringParm _url, WvStringParm _method, WvStringParm _headers,
  59                  WvStream *content_source, bool _create_dirs, bool _pipeline_test);
  60     ~WvUrlRequest();
  61
  62     void done();
  63 };
  64
  65 DeclareWvList(WvUrlRequest);
  66
  67
  68 struct WvUrlLink
  69 {
  70     WvString linkname;
  71     WvUrl url;
  72
  73     WvUrlLink(WvStringParm _linkname, WvStringParm _url)
  74         : linkname(_linkname), url(_url)
  75     {}
  76 };
  77 DeclareWvList(WvUrlLink);
  78
  79
  80 class WvBufUrlStream : public WvBufStream
  81 {
  82 public:
  83     WvString url;
  84     WvString proto;
  85     WvUrlLinkList links;  // HTML links or FTP directory listing
  86
  87     // HTTP stuff...
  88     WvString version;
  89     int status;
  90     WvHTTPHeaderDict headers;
  91
  92     WvBufUrlStream() : status(0), headers(10)
  93         {}
  94     virtual ~WvBufUrlStream()
  95         {}
  96
  97 public:
  98     const char *wstype() const { return "WvBufUrlStream"; }
  99 };
 100
 101 DeclareWvTable(WvIPPortAddr);
 102
 103
 104 class WvUrlStream : public WvStreamClone
 105 {
 106 public:
 107     class Target
 108     {
 109     public:
 110         WvIPPortAddr remaddr;
 111         WvString username;
 112
 113         Target(const WvIPPortAddr &_remaddr, WvStringParm _username)
 114             : remaddr(_remaddr), username(_username) {}
 115
 116         ~Target() {}
 117
 118         bool operator== (const Target &n2) const
 119         { return (username == n2.username && remaddr == n2.remaddr); }
 120     };
 121     Target target;
 122     static int max_requests;
 123
 124 protected:
 125     WvLog log;
 126     WvUrlRequestList urls, waiting_urls;
 127     int request_count;
 128     WvUrlRequest *curl; // current url
 129     virtual void doneurl() = 0;
 130     virtual void request_next() = 0;
 131
 132 public:
 133     WvUrlStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
 134         WvStringParm logname)
 135         : WvStreamClone(new WvTCPConn(_remaddr)), target(_remaddr, _username),
 136           log(logname, WvLog::Debug)
 137     {
 138         request_count = 0;
 139         curl = NULL;
 140     }
 141
 142     virtual ~WvUrlStream() {};
 143
 144     virtual void close() = 0;
 145     void addurl(WvUrlRequest *url);
 146     void delurl(WvUrlRequest *url);
 147     // only implemented in WvHttpStream
 148     virtual size_t remaining()
 149     { return 0; }
 150
 151     virtual void execute() = 0;
 152
 153 public:
 154     const char *wstype() const { return "WvUrlStream"; }
 155 };
 156
 157 unsigned WvHash(const WvUrlStream::Target &n);
 158
 159 DeclareWvDict(WvUrlStream, WvUrlStream::Target, target);
 160
 161
 162 class WvHttpStream : public WvUrlStream
 163 {
 164 public:
 165     static bool global_enable_pipelining;
 166     bool enable_pipelining;
 167
 168 private:
 169     int pipeline_test_count;
 170     bool ssl;
 171     bool sent_url_request;      // Have we sent a request to the server yet?
 172     WvIPPortAddrTable &pipeline_incompatible;
 173     WvString http_response, pipeline_test_response;
 174     WvDynBuf putstream_data;
 175
 176     enum { Unknown, Chunked, ContentLength, Infinity,
 177            PostHeadInfinity, PostHeadChunked, PostHeadStream,
 178            ChuckInfinity, ChuckChunked, ChuckStream } encoding;
 179     size_t bytes_remaining;
 180     bool in_chunk_trailer, last_was_pipeline_test, in_doneurl;
 181
 182     virtual void doneurl();
 183     virtual void request_next();
 184     void start_pipeline_test(WvUrl *url);
 185     WvString request_str(WvUrlRequest *url, bool keep_alive);
 186     void send_request(WvUrlRequest *url);
 187     void pipelining_is_broken(int why);
 188
 189 public:
 190     WvHttpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
 191          bool ssl, WvIPPortAddrTable &_pipeline_incompatible);
 192     virtual ~WvHttpStream();
 193
 194     virtual void close();
 195     virtual void pre_select(SelectInfo &si);
 196     virtual bool post_select(SelectInfo &si);
 197     virtual void execute();
 198     virtual size_t remaining()
 199     { return bytes_remaining; }
 200
 201 public:
 202     const char *wstype() const { return "WvHttpStream"; }
 203 };
 204
 205
 206 class WvFtpStream : public WvUrlStream
 207 {
 208     bool logged_in, pasv_acked;
 209     WvString password;
 210     WvTCPConn *data;
 211     time_t last_request_time;
 212     bool sure;
 213
 214     virtual void doneurl();
 215     virtual void request_next();
 216
 217     // Disregard all lines that are of the form "xxx-", meaning that another
 218     // line follows.  Only the last line is important for us.
 219     char *get_important_line();
 220
 221     // Parse response to "PASV" command and returns a pointer to the address
 222     // of the data port (or NULL if it can't parse the response)..
 223     // This mucks about with line.
 224     WvIPPortAddr *parse_pasv_response(char *line);
 225
 226     WvString parse_for_links(char *line);
 227
 228     WvCont cont;
 229     void* real_execute(void*);
 230
 231 public:
 232     WvFtpStream(const WvIPPortAddr &_remaddr, WvStringParm _username,
 233                 WvStringParm _password);
 234
 235     virtual void pre_select(SelectInfo &si);
 236     virtual bool post_select(SelectInfo &si);
 237     virtual void close();
 238     virtual void execute();
 239
 240 public:
 241     const char *wstype() const { return "WvFtpStream"; }
 242 };
 243
 244
 245 // FIXME: Rename this to WvUrlPool someday.
 246 class WvHttpPool : public WvIStreamList
 247 {
 248     WvLog log;
 249     WvResolver dns;
 250     WvUrlStreamDict conns;
 251     WvUrlRequestList urls;
 252     int num_streams_created;
 253     bool sure;
 254
 255     WvIPPortAddrTable pipeline_incompatible;
 256
 257 public:
 258     WvHttpPool();
 259     virtual ~WvHttpPool();
 260
 261     virtual void pre_select(SelectInfo &si);
 262     virtual bool post_select(SelectInfo &si);
 263     virtual void execute();
 264
 265     WvBufUrlStream *addurl(WvStringParm _url, WvStringParm _method = "GET",
 266                             WvStringParm _headers = "",
 267                             WvStream *content_source = NULL,
 268                             bool create_dirs = false);
 269
 270     // For URL uploads.  create_dirs should be true if you want all
 271     // non-existent directories in _url to be created.
 272 //    WvBufUrlStream *addputurl(WvStringParm _url, WvStringParm _headers,
 273 //                            WvStream *s, bool create_dirs = false);
 274 private:
 275     void unconnect(WvUrlStream *s);
 276
 277 public:
 278     bool idle() const
 279         { return !urls.count(); }
 280
 281 public:
 282     const char *wstype() const { return "WvHttpPool"; }
 283 };
 284
 285
 286 #endif // __WVHTTPPOOL_H