src/library/httpreq.cpp

   1 #define CURL_STATICLIB
   2 #include <cstdio>
   3 #include "httpreq.hpp"
   4 #include "httpauth.hpp"
   5 #include "string.hpp"
   6 #include "minmax.hpp"
   7 #include "threadtypes.hpp"
   8 #include "streamcompress.hpp"
   9 #include <curl/curl.h>
  10 #include <cstring>
  11 #include <fstream>
  12 #include <boost/iostreams/categories.hpp>
  13 #include <boost/iostreams/copy.hpp>
  14 #include <boost/iostreams/stream.hpp>
  15 #include <boost/iostreams/stream_buffer.hpp>
  16 #include <boost/iostreams/filter/symmetric.hpp>
  17 #include <boost/iostreams/filter/zlib.hpp>
  18 #include <boost/iostreams/filtering_stream.hpp>
  19 #include <boost/iostreams/device/back_inserter.hpp>
  20
  21 http_request::input_handler::~input_handler()
  22 {
  23 }
  24
  25 size_t http_request::input_handler::read_fn(char* ptr, size_t size, size_t nmemb, void* userdata)
  26 {
  27         if(reinterpret_cast<http_request::input_handler*>(userdata)->canceled) return CURL_READFUNC_ABORT;
  28         try {
  29                 return reinterpret_cast<http_request::input_handler*>(userdata)->read(ptr, size * nmemb);
  30         } catch(...) {
  31                 return CURL_READFUNC_ABORT;
  32         }
  33 }
  34
  35 http_request::null_input_handler::~null_input_handler()
  36 {
  37 }
  38
  39 uint64_t http_request::null_input_handler::get_length()
  40 {
  41         return 0;
  42 }
  43
  44 size_t http_request::null_input_handler::read(char* target, size_t maxread)
  45 {
  46         return 0;
  47 }
  48
  49 http_request::output_handler::~output_handler()
  50 {
  51 }
  52
  53 size_t http_request::output_handler::write_fn(char* ptr, size_t size, size_t nmemb, void* userdata)
  54 {
  55         if(reinterpret_cast<http_request::output_handler*>(userdata)->canceled) return 0;
  56         try {
  57                 reinterpret_cast<http_request::output_handler*>(userdata)->write(ptr, size * nmemb);
  58                 return size * nmemb;
  59         } catch(...) {
  60                 return 0;
  61         }
  62 }
  63
  64 size_t http_request::output_handler::header_fn(void* _ptr, size_t size, size_t nmemb, void* userdata)
  65 {
  66         size_t hsize = size * nmemb;
  67         char* ptr = (char*)_ptr;
  68         while(hsize > 0 && (ptr[hsize - 1] == '\r' || ptr[hsize - 1] == '\n')) hsize--;
  69         char* split = strchr((char*)ptr, ':');
  70         char* firstns = split;
  71         if(firstns) {
  72                 firstns++;
  73                 while(*firstns && (*firstns == '\t' || *firstns == ' ')) firstns++;
  74         }
  75         char* end = (char*)ptr + hsize;
  76         if(split == NULL)
  77                 reinterpret_cast<http_request::output_handler*>(userdata)->header("", std::string((char*)ptr, hsize));
  78         else
  79                 reinterpret_cast<http_request::output_handler*>(userdata)->header(std::string((char*)ptr,
  80                         split - (char*)ptr), std::string(firstns, end - firstns));
  81         return size * nmemb;
  82 }
  83
  84 http_request::www_authenticate_extractor::www_authenticate_extractor(
  85         std::function<void(const std::string& value)> _callback)
  86 {
  87         callback = _callback;
  88 }
  89
  90 http_request::www_authenticate_extractor::~www_authenticate_extractor()
  91 {
  92 }
  93
  94 std::string http_strlower(const std::string& name)
  95 {
  96         std::string name2 = name;
  97         for(size_t i = 0; i < name2.length(); i++)
  98                 if(name2[i] >= 65 && name2[i] <= 90) name2[i] = name2[i] + 32;
  99         return name2;
 100 }
 101
 102 void http_request::www_authenticate_extractor::header(const std::string& name, const std::string& content)
 103 {
 104         if(http_strlower(name) == "www-authenticate") callback(content);
 105 }
 106
 107 void http_request::www_authenticate_extractor::write(const char* source, size_t srcsize)
 108 {
 109         //Do nothing.
 110 }
 111
 112
 113 http_request::~http_request()
 114 {
 115         if(handle)
 116                 curl_easy_cleanup((CURL*)handle);
 117 }
 118
 119 http_request::http_request(const std::string& verb, const std::string& url)
 120 {
 121         dlnow = dltotal = ulnow = ultotal = 0;
 122         handle = curl_easy_init();
 123         if(!handle)
 124                 throw std::runtime_error("Can't initialize HTTP transfer");
 125         has_body = false;
 126         if(verb == "GET") {
 127         } else if(verb == "HEAD") {
 128                 auto err = curl_easy_setopt((CURL*)handle, CURLOPT_NOBODY, 1);
 129                 if(err) throw std::runtime_error(curl_easy_strerror(err));
 130         } else if(verb == "POST") {
 131                 auto err = curl_easy_setopt((CURL*)handle, CURLOPT_POST, 1);
 132                 if(err) throw std::runtime_error(curl_easy_strerror(err));
 133                 has_body = true;
 134         } else if(verb == "PUT") {
 135                 auto err = curl_easy_setopt((CURL*)handle, CURLOPT_PUT, 1);
 136                 if(err) throw std::runtime_error(curl_easy_strerror(err));
 137                 has_body = true;
 138         } else
 139                 throw std::runtime_error("Unknown HTTP verb");
 140                 auto err = curl_easy_setopt((CURL*)handle, CURLOPT_URL, url.c_str());
 141                 if(err) throw std::runtime_error(curl_easy_strerror(err));
 142 }
 143
 144 void http_request::do_transfer(input_handler* inhandler, output_handler* outhandler)
 145 {
 146         auto err = curl_easy_setopt((CURL*)handle, CURLOPT_NOPROGRESS, 0);
 147         if(err) throw std::runtime_error(curl_easy_strerror(err));
 148         err = curl_easy_setopt((CURL*)handle, CURLOPT_WRITEFUNCTION, http_request::output_handler::write_fn);
 149         if(err) throw std::runtime_error(curl_easy_strerror(err));
 150         err = curl_easy_setopt((CURL*)handle, CURLOPT_WRITEDATA, (void*)outhandler);
 151         if(err) throw std::runtime_error(curl_easy_strerror(err));
 152         if(has_body) {
 153                 err = curl_easy_setopt((CURL*)handle, CURLOPT_READFUNCTION,
 154                         http_request::input_handler::read_fn);
 155                 if(err) throw std::runtime_error(curl_easy_strerror(err));
 156                 err = curl_easy_setopt((CURL*)handle, CURLOPT_READDATA, (void*)inhandler);
 157                 if(err) throw std::runtime_error(curl_easy_strerror(err));
 158                 err = curl_easy_setopt((CURL*)handle, CURLOPT_INFILESIZE_LARGE,
 159                         (curl_off_t)inhandler->get_length());
 160                 if(err) throw std::runtime_error(curl_easy_strerror(err));
 161         }
 162         err = curl_easy_setopt((CURL*)handle, CURLOPT_PROGRESSFUNCTION, &http_request::progress);
 163         if(err) throw std::runtime_error(curl_easy_strerror(err));
 164         err = curl_easy_setopt((CURL*)handle, CURLOPT_PROGRESSDATA, (void*)this);
 165         if(err) throw std::runtime_error(curl_easy_strerror(err));
 166         err = curl_easy_setopt((CURL*)handle, CURLOPT_HEADERFUNCTION, &http_request::output_handler::header_fn);
 167         if(err) throw std::runtime_error(curl_easy_strerror(err));
 168         err = curl_easy_setopt((CURL*)handle, CURLOPT_HEADERDATA, (void*)outhandler);
 169         if(err) throw std::runtime_error(curl_easy_strerror(err));
 170
 171         struct curl_slist* list = NULL;
 172         if(authorization != "") {
 173                 std::string foo = "Authorization: " + authorization;
 174                 list = curl_slist_append(list, foo.c_str());
 175         }
 176         if(list) {
 177                 curl_easy_setopt((CURL*)handle, CURLOPT_HTTPHEADER, list);
 178         }
 179
 180         err = curl_easy_perform((CURL*)handle);
 181         if(err) throw std::runtime_error(curl_easy_strerror(err));
 182
 183         if(list)
 184                 curl_slist_free_all(list);
 185 }
 186
 187 void http_request::global_init()
 188 {
 189         curl_global_init(CURL_GLOBAL_ALL);
 190 }
 191
 192 int http_request::progress(void* userdata, double dltotal, double dlnow, double ultotal, double ulnow)
 193 {
 194         return reinterpret_cast<http_request*>(userdata)->_progress(dltotal, dlnow, ultotal, ulnow);
 195 }
 196
 197 int http_request::_progress(double _dltotal, double _dlnow, double _ultotal, double _ulnow)
 198 {
 199         dltotal = _dltotal;
 200         dlnow = _dlnow;
 201         ultotal = _ultotal;
 202         ulnow = _ulnow;
 203         return 0;
 204 }
 205
 206 void http_request::get_xfer_status(int64_t& dnow, int64_t& dtotal, int64_t& unow, int64_t& utotal)
 207 {
 208         dnow = dlnow;
 209         dtotal = dltotal;
 210         unow = ulnow;
 211         utotal = ultotal;
 212 }
 213
 214 long http_request::get_http_code()
 215 {
 216         long ret = 0;
 217         curl_easy_getinfo((CURL*)handle, CURLINFO_RESPONSE_CODE, &ret);
 218         return ret;
 219 }
 220
 221 http_async_request::http_async_request()
 222 {
 223         ihandler = NULL;
 224         ohandler = NULL;
 225         final_dl = 0;
 226         final_ul = 0;
 227         finished = false;
 228         req = NULL;
 229 }
 230
 231 void http_async_request::get_xfer_status(int64_t& dnow, int64_t& dtotal, int64_t& unow, int64_t& utotal)
 232 {
 233         umutex_class h(m);
 234         if(req) {
 235                 req->get_xfer_status(dnow, dtotal, unow, utotal);
 236         } else {
 237                 dnow = dtotal = final_dl;
 238                 unow = utotal = final_ul;
 239         }
 240 }
 241
 242 namespace
 243 {
 244         void async_http_trampoline(http_async_request* r)
 245         {
 246                 try {
 247                         r->req->do_transfer(r->ihandler, r->ohandler);
 248                 } catch(std::exception& e) {
 249                         umutex_class h(r->m);
 250                         r->finished_cond.notify_all();
 251                         r->finished = true;
 252                         delete r->req;
 253                         r->req = NULL;
 254                         r->errormsg = e.what();
 255                         return;
 256                 }
 257                 int64_t tmp1, tmp2;
 258                 umutex_class h(r->m);
 259                 r->http_code = r->req->get_http_code();
 260                 r->req->get_xfer_status(r->final_dl, tmp1, r->final_ul, tmp2);
 261                 r->finished_cond.notify_all();
 262                 r->finished = true;
 263                 delete r->req;
 264                 r->req = NULL;
 265         }
 266 }
 267
 268 void http_async_request::lauch_async()
 269 {
 270         try {
 271                 {
 272                         umutex_class h(m);
 273                         req = new http_request(verb, url);
 274                         if(authorization != "") req->set_authorization(authorization);
 275                 }
 276                 (new thread_class(async_http_trampoline, this))->detach();
 277         } catch(std::exception& e) {
 278                 umutex_class h(m);
 279                 finished_cond.notify_all();
 280                 finished = true;
 281                 delete req;
 282                 req = NULL;
 283                 errormsg = e.what();
 284         }
 285 }
 286
 287 void http_async_request::cancel()
 288 {
 289         if(ihandler) ihandler->cancel();
 290         if(ohandler) ohandler->cancel();
 291 }
 292
 293 property_upload_request::property_upload_request()
 294 {
 295         state = 0;
 296         sent = 0;
 297 }
 298
 299 property_upload_request::~property_upload_request()
 300 {
 301 }
 302
 303 uint64_t property_upload_request::get_length()
 304 {
 305         uint64_t tmp = 0;
 306         for(auto j : data) {
 307                 std::string X = (stringfmt() << "," << j.second.length() << ":").str();
 308                 tmp = tmp + X.length() + j.first.length() + j.second.length();
 309         }
 310         return tmp;
 311 }
 312
 313 void property_upload_request::rewind()
 314 {
 315         state = 0;
 316         sent = 0;
 317 }
 318
 319 void property_upload_request::str_helper(const std::string& str, char*& target, size_t& maxread, size_t& x,
 320         unsigned next)
 321 {
 322         size_t y = min((uint64_t)maxread, (uint64_t)(str.length() - sent));
 323         if(y == 0) {
 324                 state = next;
 325                 sent = 0;
 326                 return;
 327         }
 328         std::copy(str.begin() + sent, str.begin() + sent + y, target);
 329         target += y;
 330         maxread -= y;
 331         x += y;
 332         sent += y;
 333 }
 334
 335 void property_upload_request::chr_helper(char ch, char*& target, size_t& maxread, size_t& x, unsigned next)
 336 {
 337         *(target++) = ch;
 338         maxread--;
 339         x++;
 340         state = next;
 341 }
 342
 343 void property_upload_request::len_helper(size_t len, char*& target, size_t& maxread, size_t& x, unsigned next)
 344 {
 345         std::string tmp = (stringfmt() << len).str();
 346         str_helper(tmp, target, maxread, x, next);
 347 }
 348
 349 size_t property_upload_request::read(char* target, size_t maxread)
 350 {
 351         size_t x = 0;
 352         size_t y;
 353         while(maxread > 0) {
 354                 switch(state) {
 355                 case 0:
 356                         state = 1;
 357                         itr = data.begin();
 358                         break;
 359                 case 1:
 360                         if(itr == data.end()) {
 361                                 state = 7;
 362                         } else
 363                                 str_helper(itr->first, target, maxread, x, 2);
 364                         break;
 365                 case 2:
 366                         chr_helper('=', target, maxread, x, 3);
 367                         break;
 368                 case 3:         //Length of value.
 369                         len_helper(itr->second.length(), target, maxread, x, 4);
 370                         break;
 371                 case 4:         //The separator of value.
 372                         chr_helper(':', target, maxread, x, 5);
 373                         break;
 374                 case 5:         //Value.
 375                         str_helper(itr->second, target, maxread, x, 6);
 376                         break;
 377                 case 6:         //End of entry.
 378                         itr++;
 379                         state = 1;
 380                         break;
 381                 case 7:
 382                         return x;
 383                 }
 384         }
 385         return x;
 386 }