src/hooks/HTTPClient.cc

   1 /*
   2 *
   3 * centericq HTTP protocol handling class
   4 * $Id: HTTPClient.cc,v 1.19 2005/01/18 23:20:17 konst Exp $
   5 *
   6 * Copyright (C) 2003-2004 by Konstantin Klyagin <konst@konst.org.ua>
   7 *
   8 * This program is free software; you can redistribute it and/or modify
   9 * it under the terms of the GNU General Public License as published by
  10 * the Free Software Foundation; either version 2 of the License, or (at
  11 * your option) any later version.
  12 *
  13 * This program is distributed in the hope that it will be useful, but
  14 * WITHOUT ANY WARRANTY; without even the implied warranty of
  15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16 * General Public License for more details.
  17 *
  18 * You should have received a copy of the GNU General Public License
  19 * along with this program; if not, write to the Free Software
  20 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
  21 * USA
  22 *
  23 */
  24
  25 #include "HTTPClient.h"
  26
  27 #include <md5.h>
  28 #include <connwrap.h>
  29
  30 #ifdef BUILD_RSS
  31
  32 HTTPClient::HTTPClient() : m_state(NOT_CONNECTED), m_recv(),
  33 m_proxy_port(8080), m_timeout(50)  {
  34     m_socket = new TCPSocket();
  35     Init();
  36 }
  37
  38 HTTPClient::~HTTPClient() {
  39     if(m_socket->getSocketHandle() > -1) SignalRemoveSocket(m_socket->getSocketHandle());
  40     delete m_socket;
  41 }
  42
  43 void HTTPClient::Init() {
  44 }
  45
  46 void HTTPClient::Connect() {
  47     if(m_queue.empty())
  48         return;
  49
  50     int pos;
  51     HTTPRequestEvent *ev = m_queue.front();
  52     m_hostname = !m_redirect.empty() ? m_redirect : ev->getURL();
  53
  54     m_port = 80;
  55     m_resource = "/";
  56     m_content = m_redirect = "";
  57     m_recv.clear();
  58
  59     if(up(m_hostname.substr(0, 7)) == "HTTP://") {
  60         m_hostname.erase(0, 7);
  61     }
  62
  63     if((pos = m_hostname.find("@")) != -1) {
  64         ev->setAuth(HTTPRequestEvent::Basic, m_hostname.substr(0, pos), "");
  65         m_hostname.erase(0, pos+1);
  66
  67         if((pos = ev->m_user.find(":")) != -1) {
  68             ev->m_pass = ev->m_user.substr(pos+1);
  69             ev->m_user.erase(pos);
  70         }
  71     }
  72
  73     if((pos = m_hostname.find("/")) != -1) {
  74         m_resource = m_hostname.substr(pos);
  75         m_hostname.erase(pos);
  76     }
  77
  78     if((pos = m_hostname.find(":")) != -1) {
  79         m_port = strtoul(m_hostname.substr(pos+1).c_str(), 0, 0);
  80         m_hostname.erase(pos);
  81     }
  82
  83     try {
  84         if(ev->connectTries++ > 10) {
  85             ev->setHTTPResp("too many connect attempts");
  86             throw DisconnectedException("Too many connect attempts");
  87         }
  88
  89         if(m_proxy_hostname.empty()) {
  90             m_socket->setRemoteHost(m_hostname.c_str());
  91             m_socket->setRemotePort(m_port);
  92         } else {
  93             m_socket->setRemoteHost(m_proxy_hostname.c_str());
  94             m_socket->setRemotePort(m_proxy_port);
  95         }
  96
  97         m_socket->setBindHost(m_bindhost.c_str());
  98         m_socket->setBlocking(false);
  99         m_socket->Connect();
 100         SignalAddSocket(m_socket->getSocketHandle(), SocketEvent::WRITE);
 101
 102         time(&m_last_operation);
 103         m_state = WAITING_FOR_CONNECT;
 104
 105     } catch(DisconnectedException e) {
 106         SignalLog(LogEvent::WARN, e.what());
 107         Disconnect();
 108
 109     } catch(SocketException e) {
 110         SignalLog(LogEvent::WARN, e.what());
 111         Disconnect();
 112
 113     } catch(...) {
 114         SignalLog(LogEvent::WARN, "Uncaught HTTP connect exception");
 115         Disconnect();
 116
 117     }
 118
 119     if(m_state == NOT_CONNECTED) {
 120         MessageEvent *ev = m_queue.front();
 121
 122         ev->setDelivered(false);
 123         ev->setFinished(true);
 124         ev->setDeliveryFailureReason(MessageEvent::Failed);
 125         messageack.emit(ev);
 126
 127         delete ev;
 128         m_queue.pop_front();
 129     }
 130 }
 131
 132 void HTTPClient::Parse() {
 133     int npos;
 134     string response, head, val;
 135     HTTPRequestEvent *ev = m_queue.front();
 136
 137     npos = m_recv.size();
 138
 139     while(1) {
 140         response = "";
 141         m_recv.UnpackCRLFString(response);
 142         if(response.empty()) break;
 143
 144         while(!response.empty()) {
 145             if(response.substr(response.size()-1).find_first_of("\r\n") != -1)
 146                 response.erase(response.size()-1); else break;
 147         }
 148
 149         time(&m_last_operation);
 150
 151         string log = "Received HTTP response from " +
 152             IPtoString(m_socket->getRemoteIP()) + ":" +
 153             i2str(m_socket->getRemotePort()) + "\n" + response;
 154
 155         SignalLog(LogEvent::DIRECTPACKET, log);
 156
 157         if(m_state == WAITING_FOR_HEADER) {
 158             m_code = 0;
 159
 160             if((npos = response.find(" ")) != -1) {
 161                 m_code = strtoul(response.substr(npos+1).c_str(), 0, 0);
 162             }
 163
 164             switch(m_code) {
 165                 case 200:
 166                 case 301:
 167                 case 302:
 168                 case 401:
 169                     break;
 170                 default:
 171                     ev->setHTTPResp(response.substr(npos+1));
 172                     throw HTTPException("Didn't receive HTTP OK");
 173             }
 174
 175             m_state = RECEIVING_HEADER;
 176
 177         } else if(m_state == RECEIVING_HEADER) {
 178             if((npos = response.find(" ")) != -1)
 179                 head = up(response.substr(0, npos));
 180
 181             if(head == "CONTENT-LENGTH:") {
 182                 m_length = strtoul(response.substr(npos+1).c_str(), 0, 0);
 183             }
 184
 185             switch(m_code) {
 186                 case 301:
 187                 case 302:
 188                     if(head == "LOCATION:") {
 189                         m_redirect = response.substr(npos+1);
 190                         m_socket->Disconnect();
 191                         SignalRemoveSocket(m_socket->getSocketHandle());
 192                         m_state = NOT_CONNECTED;
 193                         Connect();
 194                     }
 195                     break;
 196
 197                 case 401:
 198                     if(head == "WWW-AUTHENTICATE:") {
 199                         if(ev->authTries++) {
 200                             ev->setHTTPResp("HTTP auth failed");
 201                             throw HTTPException("Authentification failed");
 202                         }
 203
 204                         response.erase(0, npos+1);
 205
 206                         if((npos = response.find(" ")) != -1) {
 207                             head = up(response.substr(0, npos));
 208                             response.erase(0, npos+1);
 209
 210                             if(head == "DIGEST")
 211                                 ev->authmethod = HTTPRequestEvent::Digest;
 212
 213                             while(!response.empty()) {
 214                                 if((npos = response.find("=")) != -1) {
 215                                     head = response.substr(0, npos);
 216                                     response.erase(0, npos+1);
 217
 218                                     if(response.substr(0, 1) == "\"") {
 219                                         response.erase(0, 1);
 220                                         if((npos = response.find("\"")) != -1)
 221                                             val = response.substr(0, npos);
 222                                     } else {
 223                                         if((npos = response.find(",")) != -1)
 224                                             val = response.substr(0, npos);
 225                                     }
 226
 227                                     ev->authparams[head] = val;
 228                                     response.erase(0, npos+1);
 229
 230                                     while(response.substr(0, 1).find_first_of(", \t") != -1)
 231                                         response.erase(0, 1);
 232
 233                                 } else break;
 234                             }
 235
 236                             m_socket->Disconnect();
 237                             SignalRemoveSocket(m_socket->getSocketHandle());
 238                             m_state = NOT_CONNECTED;
 239                             Connect();
 240                         }
 241                     }
 242                     break;
 243             }
 244
 245             if(response.empty()) m_state = RECEIVING_CONTENT;
 246
 247         } else if(m_state == RECEIVING_CONTENT) {
 248             m_content += response + "\n";
 249
 250         }
 251     }
 252 }
 253
 254 void HTTPClient::Send(Buffer &b) {
 255     string log;
 256
 257     try {
 258         log = "Sending HTTP command to " + IPtoString(m_socket->getRemoteIP()) + ":" +
 259             i2str(m_socket->getRemotePort());
 260
 261         SignalLog(LogEvent::DIRECTPACKET, log);
 262         m_socket->Send(b);
 263
 264     } catch(SocketException e) {
 265         log = (string) "Failed to send: " + e.what();
 266         m_state = NOT_CONNECTED;
 267         throw DisconnectedException(log);
 268     }
 269 }
 270
 271 string HTTPClient::strMethod(HTTPRequestEvent::RequestMethod m) {
 272     string r;
 273
 274     switch(m) {
 275         case HTTPRequestEvent::GET: r = "GET"; break;
 276         case HTTPRequestEvent::POST: r = "POST"; break;
 277     }
 278
 279     return r;
 280 }
 281
 282 static string getMD5(const string &s) {
 283     md5_state_t state;
 284     md5_byte_t digest[16];
 285     char hexdigest[3];
 286     string r;
 287     int a;
 288
 289     md5_init(&state);
 290     md5_append(&state, (md5_byte_t *) s.c_str(), s.size());
 291     md5_finish(&state, digest);
 292
 293     for(a = 0; a < 16; a++) {
 294         sprintf(hexdigest, "%02x", digest[a]);
 295         r += hexdigest;
 296     }
 297
 298     return r;
 299 }
 300
 301 void HTTPClient::SendRequest() {
 302     Buffer b;
 303     HTTPRequestEvent *ev = m_queue.front();
 304     string rq, cnonce;
 305     int i;
 306
 307     rq = strMethod(ev->getMethod());
 308
 309     if(m_proxy_hostname.empty()) {
 310         b.Pack(rq + " " + m_resource + " HTTP/1.0\r\n");
 311         b.Pack((string) "Host: " + m_hostname);
 312         if(m_port != 80) b.Pack((string) ":" + i2str(m_port));
 313         b.Pack("\r\n");
 314     } else {
 315         b.Pack(rq + " http://" + m_hostname + m_resource + " HTTP/1.0\r\n");
 316     }
 317
 318     if(!m_proxy_user.empty() && !m_proxy_hostname.empty()) {
 319         auto_ptr<char> ba(cw_base64_encode((m_proxy_user + ":" + m_proxy_passwd).c_str()));
 320         b.Pack((string) "Proxy-Authorization: Basic " + ba.get() + "\r\n");
 321     }
 322
 323 //    b.Pack((string) "Connection: keep-alive\r\n");
 324     b.Pack((string) "User-Agent: " + PACKAGE + "/" + VERSION + "\r\n");
 325
 326     if(!ev->m_user.empty()) {
 327         rq = "";
 328
 329         switch(ev->authmethod) {
 330             case HTTPRequestEvent::Basic: {
 331                     auto_ptr<char> ba(cw_base64_encode((ev->m_user + ":" + ev->m_pass).c_str()));
 332                     rq = (string) "Basic " + ba.get();
 333                 }
 334                 break;
 335
 336             case HTTPRequestEvent::User:
 337                 rq = (string) "User " + ev->m_user + ":" + ev->m_pass;
 338                 break;
 339
 340             case HTTPRequestEvent::Digest:
 341                 if(!ev->authparams.empty()) {
 342                     rq = "Digest ";
 343
 344                     for(i = 0; i < 8; i++) {
 345                         char c = randlimit(0, 15);
 346                         if(c < 10) c += 48; else c += 87;
 347                         cnonce += c;
 348                     }
 349
 350                     ev->authparams["username"] = ev->m_user;
 351                     ev->authparams["uri"] = ev->m_url;
 352                     ev->authparams["nc"] = "00000001";
 353                     ev->authparams["cnonce"] = cnonce;
 354
 355                     string a1 = ev->m_user + ":" + ev->authparams["realm"] + ":" + ev->m_pass;
 356                     string a2 = strMethod(ev->getMethod()) + ":" + ev->m_url;
 357
 358                     string mid = (string) ":" + ev->authparams["nonce"] + ":" +
 359                         ev->authparams["nc"] + ":" + cnonce + ":" +
 360                         ev->authparams["qop"] + ":";
 361
 362                     ev->authparams["response"] = getMD5(getMD5(a1) + mid + getMD5(a2));
 363
 364                     map<string, string>::const_iterator ia = ev->authparams.begin();
 365                     while(ia != ev->authparams.end()) {
 366                         if(ia != ev->authparams.begin()) rq += ", ";
 367                         rq += ia->first + "=\"" + ia->second + "\"";
 368                         ++ia;
 369                     }
 370                 }
 371
 372                 break;
 373         }
 374
 375         if(!rq.empty()) {
 376             b.Pack((string) "Authorization: " + rq + "\r\n");
 377         }
 378     }
 379
 380     if(ev->getMethod() == HTTPRequestEvent::POST) {
 381         int len = 0;
 382         vector<pair<string, string> >::const_iterator ip = ev->pbegin();
 383
 384         b.Pack("Content-type: application/x-www-form-urlencoded\r\n");
 385
 386         while(ip != ev->pend()) {
 387             len += ip->first.size() + ip->second.size() + 2;
 388             ++ip;
 389         }
 390
 391         len--;
 392         b.Pack((string) "Content-length: " + i2str(len) + "\r\n\r\n");
 393
 394         for(ip = ev->pbegin(); ip != ev->pend(); ++ip) {
 395             if(ip != ev->pbegin()) b.Pack("&");
 396             b.Pack(ip->first);
 397             b.Pack("=");
 398             b.Pack(ip->second);
 399         }
 400
 401     }
 402
 403     b.Pack("\r\n");
 404
 405     Send(b);
 406     m_state = WAITING_FOR_HEADER;
 407 }
 408
 409 void HTTPClient::Disconnect() {
 410     SignalLog(LogEvent::WARN, "HTTP disconnected");
 411     m_socket->Disconnect();
 412     m_state = NOT_CONNECTED;
 413     if(m_socket->getSocketHandle() > -1) SignalRemoveSocket(m_socket->getSocketHandle());
 414 }
 415
 416 void HTTPClient::FinishNonBlockingConnect() {
 417     SendRequest();
 418 }
 419
 420 void HTTPClient::Recv() {
 421     m_length = 0;
 422
 423     try {
 424         while(m_socket->connected()) {
 425             if(!m_socket->Recv(m_recv)) break;
 426             Parse();
 427             if(m_length && m_recv.size() >= m_length) break;
 428         }
 429
 430     } catch(SocketException e) {
 431         Disconnect();
 432         throw DisconnectedException((string) "Failed on recv: " + e.what());
 433
 434     } catch(ParseException e) {
 435         Disconnect();
 436         throw DisconnectedException((string) "Failed parsing: " + e.what());
 437
 438     } catch(HTTPException e) {
 439         MessageEvent *ev = *m_queue.begin();
 440         ev->setDelivered(false);
 441         ev->setFinished(true);
 442         ev->setDeliveryFailureReason(MessageEvent::Failed);
 443         messageack.emit(ev);
 444
 445         delete ev;
 446         m_queue.pop_front();
 447
 448         Disconnect();
 449         SignalLog(LogEvent::ERROR, e.what());
 450
 451     }
 452
 453     if(m_length && m_recv.size() >= m_length) {
 454         Disconnect();
 455         throw DisconnectedException("Content-Length reached");
 456     }
 457 }
 458
 459 void HTTPClient::check_timeout() {
 460     time_t now = time(0);
 461
 462     if(now-m_last_operation > m_timeout) {
 463         MessageEvent *ev = *m_queue.begin();
 464         ev->setDelivered(false);
 465         ev->setFinished(true);
 466         dynamic_cast<HTTPRequestEvent*>(ev)->setHTTPResp("Timed out");
 467         ev->setDeliveryFailureReason(MessageEvent::Failed);
 468         messageack.emit(ev);
 469
 470         delete ev;
 471         m_queue.pop_front();
 472         Disconnect();
 473     }
 474 }
 475
 476 void HTTPClient::setTimeout(time_t t) { m_timeout = t; }
 477 time_t HTTPClient::getTimeout() const { return m_timeout; }
 478
 479 void HTTPClient::clearoutMessagesPoll() {
 480     if(!m_queue.empty()) {
 481         if(m_state == NOT_CONNECTED) {
 482             Connect();
 483         } else {
 484             check_timeout();
 485         }
 486     }
 487 }
 488
 489 void HTTPClient::SendEvent(MessageEvent* ev) {
 490     HTTPRequestEvent *rev = dynamic_cast<HTTPRequestEvent*>(ev);
 491     if(rev) {
 492         list<HTTPRequestEvent *>::const_iterator ir = m_queue.begin();
 493         while(ir != m_queue.end()) {
 494             if(**ir == *rev) {
 495                 delete ev;
 496                 return;
 497             }
 498             ++ir;
 499         }
 500
 501         m_queue.push_back(rev);
 502
 503         if(m_state == NOT_CONNECTED) {
 504             Connect();
 505         }
 506     }
 507 }
 508
 509 // ----------------------------------------------------------------------------
 510
 511 void HTTPClient::socket_cb(int fd, SocketEvent::Mode m) {
 512     TCPSocket *sock = getSocket();
 513
 514     if(sock->getState() == TCPSocket::NONBLOCKING_CONNECT && (m & SocketEvent::WRITE)) {
 515         try {
 516             sock->FinishNonBlockingConnect();
 517
 518         } catch(SocketException e) {
 519             SignalLog(LogEvent::ERROR, (string) "Failed on non-blocking connect for direct connection: " + e.what());
 520             SignalRemoveSocket(fd);
 521             return;
 522
 523         }
 524
 525         SignalRemoveSocket(fd);
 526         SignalAddSocket(fd, SocketEvent::READ);
 527
 528         try {
 529             FinishNonBlockingConnect();
 530
 531         } catch(DisconnectedException e) {
 532             SignalLog(LogEvent::WARN, e.what());
 533             SignalRemoveSocket(fd);
 534         }
 535
 536     } else if(sock->getState() == TCPSocket::CONNECTED && (m & SocketEvent::READ)) {
 537         try {
 538             Recv();
 539
 540         } catch(DisconnectedException e) {
 541             SignalLog(LogEvent::WARN, e.what());
 542             SignalRemoveSocket(fd);
 543
 544             if(!m_recv.empty()) {
 545                 string remains;
 546                 m_recv.Unpack(remains, m_recv.size());
 547                 m_content += remains;
 548             }
 549
 550             HTTPRequestEvent *ev = *m_queue.begin();
 551             ev->setDelivered(true);
 552             ev->setFinished(true);
 553             ev->setContent(m_content);
 554             messageack.emit(ev);
 555
 556             delete ev;
 557             m_queue.pop_front();
 558
 559         }
 560
 561     } else {
 562         SignalLog(LogEvent::ERROR, "Direct Connection socket in inconsistent state!");
 563         SignalRemoveSocket(fd);
 564
 565     }
 566 }
 567
 568 #endif