release/src/router/transmission/libtransmission/web.c

   1 /*
   2  * This file Copyright (C) Mnemosyne LLC
   3  *
   4  * This file is licensed by the GPL version 2. Works owned by the
   5  * Transmission project are granted a special exemption to clause 2 (b)
   6  * so that the bulk of its code can remain under the MIT license.
   7  * This exemption does not extend to derived works not owned by
   8  * the Transmission project.
   9  *
  10  * $Id: web.c 14140 2013-07-24 00:00:03Z jordan $
  11  */
  12
  13 #include <assert.h>
  14 #include <string.h> /* strlen (), strstr () */
  15 #include <stdlib.h> /* getenv () */
  16
  17 #ifdef WIN32
  18   #include <ws2tcpip.h>
  19 #else
  20   #include <sys/select.h>
  21 #endif
  22
  23 #include <curl/curl.h>
  24
  25 #include <event2/buffer.h>
  26
  27 #include "transmission.h"
  28 #include "list.h"
  29 #include "log.h"
  30 #include "net.h" /* tr_address */
  31 #include "torrent.h"
  32 #include "platform.h" /* mutex */
  33 #include "session.h"
  34 #include "trevent.h" /* tr_runInEventThread () */
  35 #include "utils.h"
  36 #include "version.h" /* User-Agent */
  37 #include "web.h"
  38
  39 #if LIBCURL_VERSION_NUM >= 0x070F06 /* CURLOPT_SOCKOPT* was added in 7.15.6 */
  40  #define USE_LIBCURL_SOCKOPT
  41 #endif
  42
  43 enum
  44 {
  45   THREADFUNC_MAX_SLEEP_MSEC = 200,
  46 };
  47
  48 #if 0
  49 #define dbgmsg(...) \
  50   do { \
  51     fprintf (stderr, __VA_ARGS__); \
  52     fprintf (stderr, "\n"); \
  53   } while (0)
  54 #else
  55 #define dbgmsg(...) \
  56   do { \
  57     if (tr_logGetDeepEnabled ()) \
  58       tr_logAddDeep (__FILE__, __LINE__, "web", __VA_ARGS__); \
  59   } while (0)
  60 #endif
  61
  62 /***
  63 ****
  64 ***/
  65
  66 struct tr_web_task
  67 {
  68   int torrentId;
  69   long code;
  70   long timeout_secs;
  71   bool did_connect;
  72   bool did_timeout;
  73   struct evbuffer * response;
  74   struct evbuffer * freebuf;
  75   char * url;
  76   char * range;
  77   char * cookies;
  78   tr_session * session;
  79   tr_web_done_func * done_func;
  80   void * done_func_user_data;
  81   CURL * curl_easy;
  82   struct tr_web_task * next;
  83 };
  84
  85 static void
  86 task_free (struct tr_web_task * task)
  87 {
  88   if (task->freebuf)
  89     evbuffer_free (task->freebuf);
  90   tr_free (task->cookies);
  91   tr_free (task->range);
  92   tr_free (task->url);
  93   tr_free (task);
  94 }
  95
  96 /***
  97 ****
  98 ***/
  99
 100 static tr_list  * paused_easy_handles = NULL;
 101
 102 struct tr_web
 103 {
 104   bool curl_verbose;
 105   bool curl_ssl_verify;
 106   const char * curl_ca_bundle;
 107   int close_mode;
 108   struct tr_web_task * tasks;
 109   tr_lock * taskLock;
 110   char * cookie_filename;
 111 };
 112
 113 /***
 114 ****
 115 ***/
 116
 117 static size_t
 118 writeFunc (void * ptr, size_t size, size_t nmemb, void * vtask)
 119 {
 120   const size_t byteCount = size * nmemb;
 121   struct tr_web_task * task = vtask;
 122
 123   /* webseed downloads should be speed limited */
 124   if (task->torrentId != -1)
 125     {
 126       tr_torrent * tor = tr_torrentFindFromId (task->session, task->torrentId);
 127
 128       if (tor && !tr_bandwidthClamp (&tor->bandwidth, TR_DOWN, nmemb))
 129         {
 130           tr_list_append (&paused_easy_handles, task->curl_easy);
 131           return CURL_WRITEFUNC_PAUSE;
 132         }
 133     }
 134
 135   evbuffer_add (task->response, ptr, byteCount);
 136   dbgmsg ("wrote %zu bytes to task %p's buffer", byteCount, task);
 137   return byteCount;
 138 }
 139
 140 #ifdef USE_LIBCURL_SOCKOPT
 141 static int
 142 sockoptfunction (void * vtask, curl_socket_t fd, curlsocktype purpose UNUSED)
 143 {
 144   struct tr_web_task * task = vtask;
 145   const bool isScrape = strstr (task->url, "scrape") != NULL;
 146   const bool isAnnounce = strstr (task->url, "announce") != NULL;
 147
 148   /* announce and scrape requests have tiny payloads. */
 149   if (isScrape || isAnnounce)
 150     {
 151       const int sndbuf = isScrape ? 4096 : 1024;
 152       const int rcvbuf = isScrape ? 4096 : 3072;
 153       setsockopt (fd, SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof (sndbuf));
 154       setsockopt (fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof (rcvbuf));
 155     }
 156
 157   /* return nonzero if this function encountered an error */
 158   return 0;
 159 }
 160 #endif
 161
 162 static long
 163 getTimeoutFromURL (const struct tr_web_task * task)
 164 {
 165   long timeout;
 166   const tr_session * session = task->session;
 167
 168   if (!session || session->isClosed) timeout = 20L;
 169   else if (strstr (task->url, "scrape") != NULL) timeout = 30L;
 170   else if (strstr (task->url, "announce") != NULL) timeout = 90L;
 171   else timeout = 240L;
 172
 173   return timeout;
 174 }
 175
 176 static CURL *
 177 createEasy (tr_session * s, struct tr_web * web, struct tr_web_task * task)
 178 {
 179   bool is_default_value;
 180   const tr_address * addr;
 181   CURL * e = task->curl_easy = curl_easy_init ();
 182
 183   task->timeout_secs = getTimeoutFromURL (task);
 184
 185   curl_easy_setopt (e, CURLOPT_AUTOREFERER, 1L);
 186   curl_easy_setopt (e, CURLOPT_ENCODING, "gzip;q=1.0, deflate, identity");
 187   curl_easy_setopt (e, CURLOPT_FOLLOWLOCATION, 1L);
 188   curl_easy_setopt (e, CURLOPT_MAXREDIRS, -1L);
 189   curl_easy_setopt (e, CURLOPT_NOSIGNAL, 1L);
 190   curl_easy_setopt (e, CURLOPT_PRIVATE, task);
 191 #ifdef USE_LIBCURL_SOCKOPT
 192   curl_easy_setopt (e, CURLOPT_SOCKOPTFUNCTION, sockoptfunction);
 193   curl_easy_setopt (e, CURLOPT_SOCKOPTDATA, task);
 194 #endif
 195   if (web->curl_ssl_verify)
 196     {
 197       curl_easy_setopt (e, CURLOPT_CAINFO, web->curl_ca_bundle);
 198     }
 199   else
 200     {
 201       curl_easy_setopt (e, CURLOPT_SSL_VERIFYHOST, 0L);
 202       curl_easy_setopt (e, CURLOPT_SSL_VERIFYPEER, 0L);
 203     }
 204   curl_easy_setopt (e, CURLOPT_TIMEOUT, task->timeout_secs);
 205   curl_easy_setopt (e, CURLOPT_URL, task->url);
 206   curl_easy_setopt (e, CURLOPT_USERAGENT, TR_NAME "/" SHORT_VERSION_STRING);
 207   curl_easy_setopt (e, CURLOPT_VERBOSE, (long)(web->curl_verbose?1:0));
 208   curl_easy_setopt (e, CURLOPT_WRITEDATA, task);
 209   curl_easy_setopt (e, CURLOPT_WRITEFUNCTION, writeFunc);
 210
 211   if (((addr = tr_sessionGetPublicAddress (s, TR_AF_INET, &is_default_value))) && !is_default_value)
 212     curl_easy_setopt (e, CURLOPT_INTERFACE, tr_address_to_string (addr));
 213   else if (((addr = tr_sessionGetPublicAddress (s, TR_AF_INET6, &is_default_value))) && !is_default_value)
 214     curl_easy_setopt (e, CURLOPT_INTERFACE, tr_address_to_string (addr));
 215
 216   if (task->cookies != NULL)
 217     curl_easy_setopt (e, CURLOPT_COOKIE, task->cookies);
 218
 219   if (web->cookie_filename != NULL)
 220     curl_easy_setopt (e, CURLOPT_COOKIEFILE, web->cookie_filename);
 221
 222   if (task->range != NULL)
 223     {
 224       curl_easy_setopt (e, CURLOPT_RANGE, task->range);
 225       /* don't bother asking the server to compress webseed fragments */
 226       curl_easy_setopt (e, CURLOPT_ENCODING, "identity");
 227     }
 228
 229   return e;
 230 }
 231
 232 /***
 233 ****
 234 ***/
 235
 236 static void
 237 task_finish_func (void * vtask)
 238 {
 239   struct tr_web_task * task = vtask;
 240   dbgmsg ("finished web task %p; got %ld", task, task->code);
 241
 242   if (task->done_func != NULL)
 243     task->done_func (task->session,
 244                      task->did_connect,
 245                      task->did_timeout,
 246                      task->code,
 247                      evbuffer_pullup (task->response, -1),
 248                      evbuffer_get_length (task->response),
 249                      task->done_func_user_data);
 250
 251   task_free (task);
 252 }
 253
 254 /****
 255 *****
 256 ****/
 257
 258 static void tr_webThreadFunc (void * vsession);
 259
 260 static struct tr_web_task *
 261 tr_webRunImpl (tr_session         * session,
 262                int                  torrentId,
 263                const char         * url,
 264                const char         * range,
 265                const char         * cookies,
 266                tr_web_done_func     done_func,
 267                void               * done_func_user_data,
 268                struct evbuffer    * buffer)
 269 {
 270   struct tr_web_task * task = NULL;
 271
 272   if (!session->isClosing)
 273     {
 274       if (session->web == NULL)
 275         {
 276           tr_threadNew (tr_webThreadFunc, session);
 277
 278           while (session->web == NULL)
 279             tr_wait_msec (20);
 280         }
 281
 282       task = tr_new0 (struct tr_web_task, 1);
 283       task->session = session;
 284       task->torrentId = torrentId;
 285       task->url = tr_strdup (url);
 286       task->range = tr_strdup (range);
 287       task->cookies = tr_strdup (cookies);
 288       task->done_func = done_func;
 289       task->done_func_user_data = done_func_user_data;
 290       task->response = buffer ? buffer : evbuffer_new ();
 291       task->freebuf = buffer ? NULL : task->response;
 292
 293       tr_lockLock (session->web->taskLock);
 294       task->next = session->web->tasks;
 295       session->web->tasks = task;
 296       tr_lockUnlock (session->web->taskLock);
 297     }
 298
 299   return task;
 300 }
 301
 302 struct tr_web_task *
 303 tr_webRunWithCookies (tr_session        * session,
 304                       const char        * url,
 305                       const char        * cookies,
 306                       tr_web_done_func    done_func,
 307                       void              * done_func_user_data)
 308 {
 309   return tr_webRunImpl (session, -1, url,
 310                         NULL, cookies,
 311                         done_func, done_func_user_data,
 312                         NULL);
 313 }
 314
 315 struct tr_web_task *
 316 tr_webRun (tr_session         * session,
 317            const char         * url,
 318            tr_web_done_func     done_func,
 319            void               * done_func_user_data)
 320 {
 321   return tr_webRunWithCookies (session, url, NULL,
 322                                done_func, done_func_user_data);
 323 }
 324
 325
 326 struct tr_web_task *
 327 tr_webRunWebseed (tr_torrent        * tor,
 328                   const char        * url,
 329                   const char        * range,
 330                   tr_web_done_func    done_func,
 331                   void              * done_func_user_data,
 332                   struct evbuffer   * buffer)
 333 {
 334   return tr_webRunImpl (tor->session, tr_torrentId (tor), url,
 335                         range, NULL,
 336                         done_func, done_func_user_data,
 337                         buffer);
 338 }
 339
 340 /**
 341  * Portability wrapper for select ().
 342  *
 343  * http://msdn.microsoft.com/en-us/library/ms740141%28VS.85%29.aspx
 344  * On win32, any two of the parameters, readfds, writefds, or exceptfds,
 345  * can be given as null. At least one must be non-null, and any non-null
 346  * descriptor set must contain at least one handle to a socket.
 347  */
 348 static void
 349 tr_select (int nfds,
 350            fd_set * r_fd_set, fd_set * w_fd_set, fd_set * c_fd_set,
 351            struct timeval  * t)
 352 {
 353 #ifdef WIN32
 354   if (!r_fd_set->fd_count && !w_fd_set->fd_count && !c_fd_set->fd_count)
 355     {
 356       const long int msec = t->tv_sec*1000 + t->tv_usec/1000;
 357       tr_wait_msec (msec);
 358     }
 359   else if (select (0, r_fd_set->fd_count ? r_fd_set : NULL,
 360                       w_fd_set->fd_count ? w_fd_set : NULL,
 361                       c_fd_set->fd_count ? c_fd_set : NULL, t) < 0)
 362     {
 363       char errstr[512];
 364       const int e = EVUTIL_SOCKET_ERROR ();
 365       tr_net_strerror (errstr, sizeof (errstr), e);
 366       dbgmsg ("Error: select (%d) %s", e, errstr);
 367     }
 368 #else
 369   select (nfds, r_fd_set, w_fd_set, c_fd_set, t);
 370 #endif
 371 }
 372
 373 static void
 374 tr_webThreadFunc (void * vsession)
 375 {
 376   char * str;
 377   CURLM * multi;
 378   struct tr_web * web;
 379   int taskCount = 0;
 380   struct tr_web_task * task;
 381   tr_session * session = vsession;
 382
 383   /* try to enable ssl for https support; but if that fails,
 384    * try a plain vanilla init */
 385   if (curl_global_init (CURL_GLOBAL_SSL))
 386     curl_global_init (0);
 387
 388   web = tr_new0 (struct tr_web, 1);
 389   web->close_mode = ~0;
 390   web->taskLock = tr_lockNew ();
 391   web->tasks = NULL;
 392   web->curl_verbose = getenv ("TR_CURL_VERBOSE") != NULL;
 393   web->curl_ssl_verify = getenv ("TR_CURL_SSL_VERIFY") != NULL;
 394   web->curl_ca_bundle = getenv ("CURL_CA_BUNDLE");
 395   if (web->curl_ssl_verify)
 396     {
 397       tr_logAddNamedInfo ("web", "will verify tracker certs using envvar CURL_CA_BUNDLE: %s",
 398                web->curl_ca_bundle == NULL ? "none" : web->curl_ca_bundle);
 399       tr_logAddNamedInfo ("web", "NB: this only works if you built against libcurl with openssl or gnutls, NOT nss");
 400       tr_logAddNamedInfo ("web", "NB: invalid certs will show up as 'Could not connect to tracker' like many other errors");
 401     }
 402
 403   str = tr_buildPath (session->configDir, "cookies.txt", NULL);
 404   if (tr_fileExists (str, NULL))
 405     web->cookie_filename = tr_strdup (str);
 406   tr_free (str);
 407
 408   multi = curl_multi_init ();
 409   session->web = web;
 410
 411   for (;;)
 412     {
 413       long msec;
 414       int unused;
 415       CURLMsg * msg;
 416       CURLMcode mcode;
 417
 418       if (web->close_mode == TR_WEB_CLOSE_NOW)
 419         break;
 420       if ((web->close_mode == TR_WEB_CLOSE_WHEN_IDLE) && (web->tasks == NULL))
 421         break;
 422
 423       /* add tasks from the queue */
 424       tr_lockLock (web->taskLock);
 425       while (web->tasks != NULL)
 426         {
 427           /* pop the task */
 428           task = web->tasks;
 429           web->tasks = task->next;
 430           task->next = NULL;
 431
 432           dbgmsg ("adding task to curl: [%s]", task->url);
 433           curl_multi_add_handle (multi, createEasy (session, web, task));
 434           /*fprintf (stderr, "adding a task.. taskCount is now %d\n", taskCount);*/
 435           ++taskCount;
 436         }
 437       tr_lockUnlock (web->taskLock);
 438
 439       /* unpause any paused curl handles */
 440       if (paused_easy_handles != NULL)
 441         {
 442           CURL * handle;
 443           tr_list * tmp;
 444
 445           /* swap paused_easy_handles to prevent oscillation
 446              between writeFunc this while loop */
 447           tmp = paused_easy_handles;
 448           paused_easy_handles = NULL;
 449
 450           while ((handle = tr_list_pop_front (&tmp)))
 451             curl_easy_pause (handle, CURLPAUSE_CONT);
 452         }
 453
 454       /* maybe wait a little while before calling curl_multi_perform () */
 455       msec = 0;
 456       curl_multi_timeout (multi, &msec);
 457       if (msec < 0)
 458         msec = THREADFUNC_MAX_SLEEP_MSEC;
 459       if (session->isClosed)
 460         msec = 100; /* on shutdown, call perform () more frequently */
 461       if (msec > 0)
 462         {
 463           int usec;
 464           int max_fd;
 465           struct timeval t;
 466           fd_set r_fd_set, w_fd_set, c_fd_set;
 467
 468           max_fd = 0;
 469           FD_ZERO (&r_fd_set);
 470           FD_ZERO (&w_fd_set);
 471           FD_ZERO (&c_fd_set);
 472           curl_multi_fdset (multi, &r_fd_set, &w_fd_set, &c_fd_set, &max_fd);
 473
 474           if (msec > THREADFUNC_MAX_SLEEP_MSEC)
 475             msec = THREADFUNC_MAX_SLEEP_MSEC;
 476
 477           usec = msec * 1000;
 478           t.tv_sec =  usec / 1000000;
 479           t.tv_usec = usec % 1000000;
 480           tr_select (max_fd+1, &r_fd_set, &w_fd_set, &c_fd_set, &t);
 481         }
 482
 483       /* call curl_multi_perform () */
 484       do
 485         mcode = curl_multi_perform (multi, &unused);
 486       while (mcode == CURLM_CALL_MULTI_PERFORM);
 487
 488       /* pump completed tasks from the multi */
 489       while ((msg = curl_multi_info_read (multi, &unused)))
 490         {
 491           if ((msg->msg == CURLMSG_DONE) && (msg->easy_handle != NULL))
 492             {
 493               double total_time;
 494               struct tr_web_task * task;
 495               long req_bytes_sent;
 496               CURL * e = msg->easy_handle;
 497               curl_easy_getinfo (e, CURLINFO_PRIVATE, (void*)&task);
 498               assert (e == task->curl_easy);
 499               curl_easy_getinfo (e, CURLINFO_RESPONSE_CODE, &task->code);
 500               curl_easy_getinfo (e, CURLINFO_REQUEST_SIZE, &req_bytes_sent);
 501               curl_easy_getinfo (e, CURLINFO_TOTAL_TIME, &total_time);
 502               task->did_connect = task->code>0 || req_bytes_sent>0;
 503               task->did_timeout = !task->code && (total_time >= task->timeout_secs);
 504               curl_multi_remove_handle (multi, e);
 505               tr_list_remove_data (&paused_easy_handles, e);
 506               curl_easy_cleanup (e);
 507               tr_runInEventThread (task->session, task_finish_func, task);
 508               --taskCount;
 509             }
 510         }
 511     }
 512
 513   /* Discard any remaining tasks.
 514    * This is rare, but can happen on shutdown with unresponsive trackers. */
 515   while (web->tasks != NULL)
 516     {
 517       task = web->tasks;
 518       web->tasks = task->next;
 519       dbgmsg ("Discarding task \"%s\"", task->url);
 520       task_free (task);
 521     }
 522
 523   /* cleanup */
 524   tr_list_free (&paused_easy_handles, NULL);
 525   curl_multi_cleanup (multi);
 526   tr_lockFree (web->taskLock);
 527   tr_free (web->cookie_filename);
 528   tr_free (web);
 529   session->web = NULL;
 530 }
 531
 532
 533 void
 534 tr_webClose (tr_session * session, tr_web_close_mode close_mode)
 535 {
 536   if (session->web != NULL)
 537     {
 538       session->web->close_mode = close_mode;
 539
 540       if (close_mode == TR_WEB_CLOSE_NOW)
 541         while (session->web != NULL)
 542           tr_wait_msec (100);
 543     }
 544 }
 545
 546 void
 547 tr_webGetTaskInfo (struct tr_web_task * task, tr_web_task_info info, void * dst)
 548 {
 549   curl_easy_getinfo (task->curl_easy, (CURLINFO) info, dst);
 550 }
 551
 552 /*****
 553 ******
 554 ******
 555 *****/
 556
 557 const char *
 558 tr_webGetResponseStr (long code)
 559 {
 560   switch (code)
 561     {
 562       case   0: return "No Response";
 563       case 101: return "Switching Protocols";
 564       case 200: return "OK";
 565       case 201: return "Created";
 566       case 202: return "Accepted";
 567       case 203: return "Non-Authoritative Information";
 568       case 204: return "No Content";
 569       case 205: return "Reset Content";
 570       case 206: return "Partial Content";
 571       case 300: return "Multiple Choices";
 572       case 301: return "Moved Permanently";
 573       case 302: return "Found";
 574       case 303: return "See Other";
 575       case 304: return "Not Modified";
 576       case 305: return "Use Proxy";
 577       case 306: return " (Unused)";
 578       case 307: return "Temporary Redirect";
 579       case 400: return "Bad Request";
 580       case 401: return "Unauthorized";
 581       case 402: return "Payment Required";
 582       case 403: return "Forbidden";
 583       case 404: return "Not Found";
 584       case 405: return "Method Not Allowed";
 585       case 406: return "Not Acceptable";
 586       case 407: return "Proxy Authentication Required";
 587       case 408: return "Request Timeout";
 588       case 409: return "Conflict";
 589       case 410: return "Gone";
 590       case 411: return "Length Required";
 591       case 412: return "Precondition Failed";
 592       case 413: return "Request Entity Too Large";
 593       case 414: return "Request-URI Too Long";
 594       case 415: return "Unsupported Media Type";
 595       case 416: return "Requested Range Not Satisfiable";
 596       case 417: return "Expectation Failed";
 597       case 500: return "Internal Server Error";
 598       case 501: return "Not Implemented";
 599       case 502: return "Bad Gateway";
 600       case 503: return "Service Unavailable";
 601       case 504: return "Gateway Timeout";
 602       case 505: return "HTTP Version Not Supported";
 603       default:  return "Unknown Error";
 604     }
 605 }
 606
 607 void
 608 tr_http_escape (struct evbuffer  * out,
 609                 const char       * str,
 610                 int                len,
 611                 bool               escape_slashes)
 612 {
 613   const char * end;
 614
 615   if ((len < 0) && (str != NULL))
 616     len = strlen (str);
 617
 618   for (end=str+len; str && str!=end; ++str)
 619     {
 620       if ((*str == ',') || (*str == '-')
 621                         || (*str == '.')
 622                         || (('0' <= *str) && (*str <= '9'))
 623                         || (('A' <= *str) && (*str <= 'Z'))
 624                         || (('a' <= *str) && (*str <= 'z'))
 625                         || ((*str == '/') && (!escape_slashes)))
 626         evbuffer_add_printf (out, "%c", *str);
 627       else
 628         evbuffer_add_printf (out, "%%%02X", (unsigned)(*str&0xFF));
 629     }
 630 }
 631
 632 char *
 633 tr_http_unescape (const char * str, int len)
 634 {
 635   char * tmp = curl_unescape (str, len);
 636   char * ret = tr_strdup (tmp);
 637   curl_free (tmp);
 638   return ret;
 639 }
 640
 641 static int
 642 is_rfc2396_alnum (uint8_t ch)
 643 {
 644   return ('0' <= ch && ch <= '9')
 645       || ('A' <= ch && ch <= 'Z')
 646       || ('a' <= ch && ch <= 'z')
 647       || ch == '.'
 648       || ch == '-'
 649       || ch == '_'
 650       || ch == '~';
 651 }
 652
 653 void
 654 tr_http_escape_sha1 (char * out, const uint8_t * sha1_digest)
 655 {
 656   const uint8_t * in = sha1_digest;
 657   const uint8_t * end = in + SHA_DIGEST_LENGTH;
 658
 659   while (in != end)
 660     if (is_rfc2396_alnum (*in))
 661       *out++ = (char) *in++;
 662     else
 663       out += tr_snprintf (out, 4, "%%%02x", (unsigned int)*in++);
 664
 665   *out = '\0';
 666 }