hphp/runtime/base/libevent-http-client.cpp

   1 /*
   2    +----------------------------------------------------------------------+
   3    | HipHop for PHP                                                       |
   4    +----------------------------------------------------------------------+
   5    | Copyright (c) 2010-present Facebook, Inc. (http://www.facebook.com)  |
   6    +----------------------------------------------------------------------+
   7    | This source file is subject to version 3.01 of the PHP license,      |
   8    | that is bundled with this package in the file LICENSE, and is        |
   9    | available through the world-wide-web at the following url:           |
  10    | http://www.php.net/license/3_01.txt                                  |
  11    | If you did not receive a copy of the PHP license and are unable to   |
  12    | obtain it through the world-wide-web, please send a note to          |
  13    | license@php.net so we can mail you a copy immediately.               |
  14    +----------------------------------------------------------------------+
  15 */
  16 #include "hphp/runtime/base/libevent-http-client.h"
  17 #include <map>
  18 #include <vector>
  19
  20 #include <folly/Conv.h>
  21
  22 #include "hphp/runtime/server/server-stats.h"
  23 #include "hphp/runtime/base/runtime-option.h"
  24 #include "hphp/util/compression.h"
  25 #include "hphp/util/logger.h"
  26 #include "hphp/util/timer.h"
  27
  28 // libevent is not exposing this data structure, but we need it.
  29 struct evkeyvalq_ {
  30   struct evkeyval *tqh_first;
  31 };
  32
  33 ///////////////////////////////////////////////////////////////////////////////
  34 // static handlers delegating work to instance ones
  35
  36 static void on_request_completed(struct evhttp_request *req, void *obj) {
  37   assert(obj);
  38   ((HPHP::LibEventHttpClient*)obj)->onRequestCompleted(req);
  39 }
  40
  41 static void on_connection_closed(struct evhttp_connection *conn, void *obj) {
  42   assert(obj);
  43   ((HPHP::LibEventHttpClient*)obj)->onConnectionClosed();
  44 }
  45
  46 static void timer_callback(int fd, short events, void *context) {
  47   event_base_loopbreak((struct event_base *)context);
  48 }
  49
  50 namespace HPHP {
  51 ///////////////////////////////////////////////////////////////////////////////
  52 // connection pooling
  53
  54 static std::string get_hash(const std::string &address, int port) {
  55   std::string hash = address;
  56   if (port != 80) {
  57     hash += ':';
  58     hash += folly::to<std::string>(port);
  59   }
  60   return hash;
  61 }
  62
  63 ReadWriteMutex LibEventHttpClient::ConnectionPoolMutex;
  64 std::map<std::string,int> LibEventHttpClient::ConnectionPoolConfig;
  65 std::map<std::string,std::vector<LibEventHttpClientPtr>>
  66   LibEventHttpClient::ConnectionPool;
  67
  68 void LibEventHttpClient::SetCache(const std::string &address, int port,
  69                                   int maxConnection) {
  70   auto const hash = get_hash(address, port);
  71
  72   WriteLock lock(ConnectionPoolMutex);
  73   if (maxConnection > 0) {
  74     ConnectionPoolConfig[hash] = maxConnection;
  75   } else {
  76     ConnectionPoolConfig.erase(hash);
  77   }
  78 }
  79
  80 LibEventHttpClientPtr LibEventHttpClient::Get(const std::string &address,
  81                                               int port) {
  82   auto const hash = get_hash(address, port);
  83
  84   int maxConnection = 0;
  85   {
  86     ReadLock lock(ConnectionPoolMutex);
  87     auto iter = ConnectionPoolConfig.find(hash);
  88     if (iter == ConnectionPoolConfig.end()) {
  89       // not configured to cache
  90       ServerStats::Log("evhttp.skip", 1);
  91       ServerStats::Log("evhttp.skip." + hash, 1);
  92       return LibEventHttpClientPtr(new LibEventHttpClient(address, port));
  93     }
  94     maxConnection = iter->second;
  95   }
  96
  97   WriteLock lock(ConnectionPoolMutex);
  98   auto& pool = ConnectionPool[hash];
  99   for (unsigned int i = 0; i < pool.size(); i++) {
 100     auto client = pool[i];
 101     if (!client->m_busy) {
 102       client->m_busy = true;
 103       ServerStats::Log("evhttp.hit", 1);
 104       ServerStats::Log("evhttp.hit." + hash, 1);
 105       return client;
 106     }
 107   }
 108
 109   LibEventHttpClientPtr ret(new LibEventHttpClient(address, port));
 110   if ((int)pool.size() < maxConnection) {
 111     if (pool.empty()) {
 112       pool.reserve(maxConnection);
 113     }
 114     pool.push_back(ret);
 115   }
 116   ServerStats::Log("evhttp.miss", 1);
 117   ServerStats::Log("evhttp.miss." + hash, 1);
 118   return ret;
 119 }
 120
 121 ///////////////////////////////////////////////////////////////////////////////
 122 // constructor and destructor
 123
 124 LibEventHttpClient::LibEventHttpClient(const std::string &address, int port)
 125   : m_busy(true), m_address(address), m_port(port), m_requests(0),
 126     m_conn(nullptr), m_thread(nullptr), m_code(0), m_response(nullptr), m_len(0) {
 127   m_eventBase = event_base_new();
 128 }
 129
 130 LibEventHttpClient::~LibEventHttpClient() {
 131   clear();
 132
 133   // reset all per-connection data structures
 134   if (m_conn) {
 135     evhttp_connection_free(m_conn);
 136   }
 137   if (m_eventBase) {
 138     event_base_free(m_eventBase);
 139   }
 140 }
 141
 142 void LibEventHttpClient::clear() {
 143   // reset all per-request data structures
 144   if (m_thread) {
 145     m_thread->waitForEnd();
 146     delete m_thread;
 147     m_thread = nullptr;
 148   }
 149   m_url.clear();
 150   m_code = 0;
 151   m_codeLine.clear();
 152   m_len = 0;
 153   if (m_response) {
 154     free(m_response);
 155     m_response = nullptr;
 156   }
 157   m_responseHeaders.clear();
 158 }
 159
 160 void LibEventHttpClient::release() {
 161   clear();
 162   m_busy = false;
 163 }
 164
 165 ///////////////////////////////////////////////////////////////////////////////
 166
 167 bool LibEventHttpClient::send(const std::string &url,
 168                               const std::vector<std::string> &headers,
 169                               int timeoutSeconds, bool async,
 170                               const void *data /* = NULL */,
 171                               int size /* = 0 */) {
 172   clear();
 173   m_url = url;
 174
 175   evhttp_request* request = evhttp_request_new(on_request_completed, this);
 176
 177   // request headers
 178   bool keepalive = true;
 179   bool addHost = true;
 180   for (unsigned int i = 0; i < headers.size(); i++) {
 181     const std::string &header = headers[i];
 182     size_t pos = header.find(':');
 183     if (pos != std::string::npos && header[pos + 1] == ' ') {
 184       std::string name = header.substr(0, pos);
 185       if (strcasecmp(name.c_str(), "Connection") == 0) {
 186         keepalive = false;
 187       } else if (strcasecmp(name.c_str(), "Host") == 0) {
 188         addHost = false;
 189       }
 190       int ret = evhttp_add_header(request->output_headers,
 191                                   name.c_str(), header.c_str() + pos + 2);
 192       if (ret >= 0) {
 193         continue;
 194       }
 195     }
 196     Logger::Error("invalid request header: [%s]", header.c_str());
 197   }
 198   if (keepalive) {
 199     evhttp_add_header(request->output_headers, "Connection", "keep-alive");
 200   }
 201   if (addHost) {
 202     // REVIEW: libevent never sends a Host header (nor does it properly send
 203     // HTTP 400 for HTTP/1.1 requests without such a header), in blatant
 204     // violation of RFC2616; this should perhaps be fixed in the library
 205     // proper.  For now, add it if it wasn't set by the caller.
 206     if (m_port == 80) {
 207       evhttp_add_header(request->output_headers, "Host", m_address.c_str());
 208     } else {
 209       std::ostringstream ss;
 210       ss << m_address << ":" << m_port;
 211       evhttp_add_header(request->output_headers, "Host", ss.str().c_str());
 212     }
 213   }
 214
 215   // post data
 216   if (data && size) {
 217     evbuffer_add(request->output_buffer, data, size);
 218   }
 219
 220   // url
 221   evhttp_cmd_type cmd = data ? EVHTTP_REQ_POST : EVHTTP_REQ_GET;
 222
 223   // if we have a cached connection, we need to pump the event loop to clear
 224   // any "connection closed" events that may be sitting there patiently.
 225   if (m_conn) {
 226     event_base_loop(m_eventBase, EVLOOP_NONBLOCK);
 227   }
 228
 229   // even if we had an m_conn immediately above, it may have been cleared out
 230   // by onConnectionClosed().
 231   if (m_conn == nullptr) {
 232     m_conn = evhttp_connection_new(m_address.c_str(), m_port);
 233     evhttp_connection_set_closecb(m_conn, on_connection_closed, this);
 234     evhttp_connection_set_base(m_conn, m_eventBase);
 235   }
 236
 237   int ret = evhttp_make_request(m_conn, request, cmd, url.c_str());
 238   if (ret != 0) {
 239     Logger::Error("evhttp_make_request failed");
 240     return false;
 241   }
 242
 243   if (timeoutSeconds > 0) {
 244     struct timeval timeout;
 245     timeout.tv_sec = timeoutSeconds;
 246     timeout.tv_usec = 0;
 247
 248     event_set(&m_eventTimeout, -1, 0, timer_callback, m_eventBase);
 249     event_base_set(m_eventBase, &m_eventTimeout);
 250     event_add(&m_eventTimeout, &timeout);
 251   }
 252
 253   if (async) {
 254     m_thread = new AsyncFunc<LibEventHttpClient>
 255       (this, &LibEventHttpClient::sendImpl);
 256     m_thread->start();
 257   } else {
 258     IOStatusHelper io("libevent_http", m_address.c_str(), m_port);
 259     sendImpl();
 260   }
 261   return true;
 262 }
 263
 264 void LibEventHttpClient::sendImpl() {
 265   SlowTimer timer(RuntimeOption::HttpSlowQueryThreshold, "evhttp",
 266                   m_url.c_str());
 267   event_base_dispatch(m_eventBase);
 268   event_del(&m_eventTimeout);
 269 }
 270
 271 void LibEventHttpClient::onRequestCompleted(evhttp_request* request) {
 272   if (!request) {
 273     // eek -- this is just a clean-up notification because the connection's
 274     // been closed, but we already dealt with it in onConnectionClosed
 275     return;
 276   }
 277
 278   // response code line
 279   m_code = request->response_code;
 280   if (request->response_code_line) {
 281     m_codeLine = request->response_code_line;
 282   }
 283
 284   bool gzip = false;
 285   // response headers
 286   for (evkeyval *p = ((evkeyvalq_*)request->input_headers)->tqh_first; p;
 287        p = p->next.tqe_next) {
 288     if (p->key && p->value) {
 289       if (strcasecmp(p->key, "Content-Encoding") == 0 &&
 290           strncmp(p->value, "gzip", 4) == 0 &&
 291           (!p->value[4] || isspace(p->value[4]))) {
 292         // in the (illegal) case of multiple Content-Encoding headers, any one
 293         // with the value 'gzip' means we treat it as gzip.
 294         gzip = true;
 295       }
 296       m_responseHeaders.push_back(std::string(p->key) + ": " + p->value);
 297     }
 298   }
 299
 300   // response body
 301   m_len = EVBUFFER_LENGTH(request->input_buffer);
 302   if (gzip) {
 303     m_response =
 304       gzdecode((const char*)EVBUFFER_DATA(request->input_buffer), m_len);
 305   } else {
 306     m_response = (char*)malloc(m_len + 1);
 307     strncpy(m_response, (char*)EVBUFFER_DATA(request->input_buffer), m_len);
 308     m_response[m_len] = '\0';
 309   }
 310
 311   ++m_requests;
 312   event_base_loopbreak(m_eventBase);
 313 }
 314
 315 void LibEventHttpClient::onConnectionClosed() {
 316   m_conn = nullptr;
 317   m_requests = 0;
 318 }
 319
 320 char *LibEventHttpClient::recv(int &len) {
 321   if (m_thread) {
 322     m_thread->waitForEnd();
 323     delete m_thread;
 324     m_thread = nullptr;
 325   }
 326
 327   char *ret = m_response;
 328   len = m_len;
 329   m_response = nullptr;
 330   m_len = 0;
 331   return ret;
 332 }
 333
 334 ///////////////////////////////////////////////////////////////////////////////
 335 }