release/src/router/transmission/libtransmission/web.c

   1 /*
   2  * This file Copyright (C) Mnemosyne LLC
   3  *
   4  * This file is licensed by the GPL version 2. Works owned by the
   5  * Transmission project are granted a special exemption to clause 2(b)
   6  * so that the bulk of its code can remain under the MIT license.
   7  * This exemption does not extend to derived works not owned by
   8  * the Transmission project.
   9  *
  10  * $Id: web.c 13245 2012-03-04 13:21:42Z jordan $
  11  */
  12
  13 #include <string.h> /* strlen(), strstr() */
  14 #include <stdlib.h> /* getenv() */
  15
  16 #ifdef WIN32
  17   #include <ws2tcpip.h>
  18 #else
  19   #include <sys/select.h>
  20 #endif
  21
  22 #include <curl/curl.h>
  23
  24 #include <event2/buffer.h>
  25
  26 #include "transmission.h"
  27 #include "net.h" /* tr_address */
  28 #include "platform.h" /* mutex */
  29 #include "session.h"
  30 #include "trevent.h" /* tr_runInEventThread() */
  31 #include "utils.h"
  32 #include "version.h" /* User-Agent */
  33 #include "web.h"
  34
  35 #if LIBCURL_VERSION_NUM >= 0x070F06 /* CURLOPT_SOCKOPT* was added in 7.15.6 */
  36  #define USE_LIBCURL_SOCKOPT
  37 #endif
  38
  39 enum
  40 {
  41     THREADFUNC_MAX_SLEEP_MSEC = 1000,
  42 };
  43
  44 #if 0
  45 #define dbgmsg(...) \
  46     do { \
  47         fprintf( stderr, __VA_ARGS__ ); \
  48         fprintf( stderr, "\n" ); \
  49     } while( 0 )
  50 #else
  51 #define dbgmsg( ... ) \
  52     do { \
  53         if( tr_deepLoggingIsActive( ) ) \
  54             tr_deepLog( __FILE__, __LINE__, "web", __VA_ARGS__ ); \
  55     } while( 0 )
  56 #endif
  57
  58 /***
  59 ****
  60 ***/
  61
  62 struct tr_web_task
  63 {
  64     long code;
  65     long timeout_secs;
  66     bool did_connect;
  67     bool did_timeout;
  68     struct evbuffer * response;
  69     struct evbuffer * freebuf;
  70     char * url;
  71     char * range;
  72     char * cookies;
  73     tr_session * session;
  74     tr_web_done_func * done_func;
  75     void * done_func_user_data;
  76     CURL * curl_easy;
  77     struct tr_web_task * next;
  78 };
  79
  80 static void
  81 task_free( struct tr_web_task * task )
  82 {
  83     if( task->freebuf )
  84         evbuffer_free( task->freebuf );
  85     tr_free( task->cookies );
  86     tr_free( task->range );
  87     tr_free( task->url );
  88     tr_free( task );
  89 }
  90
  91 /***
  92 ****
  93 ***/
  94
  95 struct tr_web
  96 {
  97     bool curl_verbose;
  98     bool curl_ssl_verify;
  99     const char * curl_ca_bundle;
 100     int close_mode;
 101     struct tr_web_task * tasks;
 102     tr_lock * taskLock;
 103     char * cookie_filename;
 104 };
 105
 106 /***
 107 ****
 108 ***/
 109
 110 static size_t
 111 writeFunc( void * ptr, size_t size, size_t nmemb, void * vtask )
 112 {
 113     const size_t byteCount = size * nmemb;
 114     struct tr_web_task * task = vtask;
 115     evbuffer_add( task->response, ptr, byteCount );
 116     dbgmsg( "wrote %zu bytes to task %p's buffer", byteCount, task );
 117     return byteCount;
 118 }
 119
 120 #ifdef USE_LIBCURL_SOCKOPT
 121 static int
 122 sockoptfunction( void * vtask, curl_socket_t fd, curlsocktype purpose UNUSED )
 123 {
 124     struct tr_web_task * task = vtask;
 125     const bool isScrape = strstr( task->url, "scrape" ) != NULL;
 126     const bool isAnnounce = strstr( task->url, "announce" ) != NULL;
 127
 128     /* announce and scrape requests have tiny payloads. */
 129     if( isScrape || isAnnounce )
 130     {
 131         const int sndbuf = 1024;
 132         const int rcvbuf = isScrape ? 2048 : 3072;
 133         setsockopt( fd, SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf) );
 134         setsockopt( fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf) );
 135     }
 136
 137     /* return nonzero if this function encountered an error */
 138     return 0;
 139 }
 140 #endif
 141
 142 static long
 143 getTimeoutFromURL( const struct tr_web_task * task )
 144 {
 145     long timeout;
 146     const tr_session * session = task->session;
 147
 148     if( !session || session->isClosed ) timeout = 20L;
 149     else if( strstr( task->url, "scrape" ) != NULL ) timeout = 30L;
 150     else if( strstr( task->url, "announce" ) != NULL ) timeout = 90L;
 151     else timeout = 240L;
 152
 153     return timeout;
 154 }
 155
 156 static CURL *
 157 createEasy( tr_session * s, struct tr_web * web, struct tr_web_task * task )
 158 {
 159     bool is_default_value;
 160     const tr_address * addr;
 161     CURL * e = task->curl_easy = curl_easy_init( );
 162
 163     task->timeout_secs = getTimeoutFromURL( task );
 164
 165     curl_easy_setopt( e, CURLOPT_AUTOREFERER, 1L );
 166     curl_easy_setopt( e, CURLOPT_COOKIEFILE, web->cookie_filename );
 167     curl_easy_setopt( e, CURLOPT_ENCODING, "gzip;q=1.0, deflate, identity" );
 168     curl_easy_setopt( e, CURLOPT_FOLLOWLOCATION, 1L );
 169     curl_easy_setopt( e, CURLOPT_MAXREDIRS, -1L );
 170     curl_easy_setopt( e, CURLOPT_NOSIGNAL, 1L );
 171     curl_easy_setopt( e, CURLOPT_PRIVATE, task );
 172 #ifdef USE_LIBCURL_SOCKOPT
 173     curl_easy_setopt( e, CURLOPT_SOCKOPTFUNCTION, sockoptfunction );
 174     curl_easy_setopt( e, CURLOPT_SOCKOPTDATA, task );
 175 #endif
 176     if( web->curl_ssl_verify )
 177         curl_easy_setopt( e, CURLOPT_CAINFO, web->curl_ca_bundle );
 178     else {
 179         curl_easy_setopt( e, CURLOPT_SSL_VERIFYHOST, 0L );
 180         curl_easy_setopt( e, CURLOPT_SSL_VERIFYPEER, 0L );
 181     }
 182     curl_easy_setopt( e, CURLOPT_TIMEOUT, task->timeout_secs );
 183     curl_easy_setopt( e, CURLOPT_URL, task->url );
 184     curl_easy_setopt( e, CURLOPT_USERAGENT, TR_NAME "/" SHORT_VERSION_STRING );
 185     curl_easy_setopt( e, CURLOPT_VERBOSE, (long)(web->curl_verbose?1:0) );
 186     curl_easy_setopt( e, CURLOPT_WRITEDATA, task );
 187     curl_easy_setopt( e, CURLOPT_WRITEFUNCTION, writeFunc );
 188
 189     if((( addr = tr_sessionGetPublicAddress( s, TR_AF_INET, &is_default_value ))) && !is_default_value )
 190         curl_easy_setopt( e, CURLOPT_INTERFACE, tr_address_to_string( addr ) );
 191     else if ((( addr = tr_sessionGetPublicAddress( s, TR_AF_INET6, &is_default_value ))) && !is_default_value )
 192         curl_easy_setopt( e, CURLOPT_INTERFACE, tr_address_to_string( addr ) );
 193
 194     if( task->cookies != NULL )
 195         curl_easy_setopt( e, CURLOPT_COOKIE, task->cookies );
 196
 197     if( task->range != NULL ) {
 198         curl_easy_setopt( e, CURLOPT_RANGE, task->range );
 199         /* don't bother asking the server to compress webseed fragments */
 200         curl_easy_setopt( e, CURLOPT_ENCODING, "identity" );
 201     }
 202
 203     return e;
 204 }
 205
 206 /***
 207 ****
 208 ***/
 209
 210 static void
 211 task_finish_func( void * vtask )
 212 {
 213     struct tr_web_task * task = vtask;
 214     dbgmsg( "finished web task %p; got %ld", task, task->code );
 215
 216     if( task->done_func != NULL )
 217         task->done_func( task->session,
 218                          task->did_connect,
 219                          task->did_timeout,
 220                          task->code,
 221                          evbuffer_pullup( task->response, -1 ),
 222                          evbuffer_get_length( task->response ),
 223                          task->done_func_user_data );
 224
 225     task_free( task );
 226 }
 227
 228 /****
 229 *****
 230 ****/
 231
 232 struct tr_web_task *
 233 tr_webRun( tr_session         * session,
 234            const char         * url,
 235            const char         * range,
 236            const char         * cookies,
 237            tr_web_done_func     done_func,
 238            void               * done_func_user_data )
 239 {
 240     return tr_webRunWithBuffer( session, url, range, cookies,
 241                                 done_func, done_func_user_data,
 242                                 NULL );
 243 }
 244
 245 struct tr_web_task *
 246 tr_webRunWithBuffer( tr_session         * session,
 247                      const char         * url,
 248                      const char         * range,
 249                      const char         * cookies,
 250                      tr_web_done_func     done_func,
 251                      void               * done_func_user_data,
 252                      struct evbuffer    * buffer )
 253 {
 254     struct tr_web * web = session->web;
 255
 256     if( web != NULL )
 257     {
 258         struct tr_web_task * task = tr_new0( struct tr_web_task, 1 );
 259
 260         task->session = session;
 261         task->url = tr_strdup( url );
 262         task->range = tr_strdup( range );
 263         task->cookies = tr_strdup( cookies);
 264         task->done_func = done_func;
 265         task->done_func_user_data = done_func_user_data;
 266         task->response = buffer ? buffer : evbuffer_new( );
 267         task->freebuf = buffer ? NULL : task->response;
 268
 269         tr_lockLock( web->taskLock );
 270         task->next = web->tasks;
 271         web->tasks = task;
 272         tr_lockUnlock( web->taskLock );
 273         return task;
 274     }
 275     return NULL;
 276 }
 277
 278 /**
 279  * Portability wrapper for select().
 280  *
 281  * http://msdn.microsoft.com/en-us/library/ms740141%28VS.85%29.aspx
 282  * On win32, any two of the parameters, readfds, writefds, or exceptfds,
 283  * can be given as null. At least one must be non-null, and any non-null
 284  * descriptor set must contain at least one handle to a socket.
 285  */
 286 static void
 287 tr_select( int nfds,
 288            fd_set * r_fd_set, fd_set * w_fd_set, fd_set * c_fd_set,
 289            struct timeval  * t )
 290 {
 291 #ifdef WIN32
 292     if( !r_fd_set->fd_count && !w_fd_set->fd_count && !c_fd_set->fd_count )
 293     {
 294         const long int msec = t->tv_sec*1000 + t->tv_usec/1000;
 295         tr_wait_msec( msec );
 296     }
 297     else if( select( 0, r_fd_set->fd_count ? r_fd_set : NULL,
 298                         w_fd_set->fd_count ? w_fd_set : NULL,
 299                         c_fd_set->fd_count ? c_fd_set : NULL, t ) < 0 )
 300     {
 301         char errstr[512];
 302         const int e = EVUTIL_SOCKET_ERROR( );
 303         tr_net_strerror( errstr, sizeof( errstr ), e );
 304         dbgmsg( "Error: select (%d) %s", e, errstr );
 305     }
 306 #else
 307     select( nfds, r_fd_set, w_fd_set, c_fd_set, t );
 308 #endif
 309 }
 310
 311 static void
 312 tr_webThreadFunc( void * vsession )
 313 {
 314     CURLM * multi;
 315     struct tr_web * web;
 316     int taskCount = 0;
 317     struct tr_web_task * task;
 318     tr_session * session = vsession;
 319
 320     /* try to enable ssl for https support; but if that fails,
 321      * try a plain vanilla init */
 322     if( curl_global_init( CURL_GLOBAL_SSL ) )
 323         curl_global_init( 0 );
 324
 325     web = tr_new0( struct tr_web, 1 );
 326     web->close_mode = ~0;
 327     web->taskLock = tr_lockNew( );
 328     web->tasks = NULL;
 329     web->curl_verbose = getenv( "TR_CURL_VERBOSE" ) != NULL;
 330     web->curl_ssl_verify = getenv( "TR_CURL_SSL_VERIFY" ) != NULL;
 331     web->curl_ca_bundle = getenv( "CURL_CA_BUNDLE" );
 332     if( web->curl_ssl_verify ) {
 333         tr_ninf( "web", "will verify tracker certs using envvar CURL_CA_BUNDLE: %s",
 334                   web->curl_ca_bundle == NULL ? "none" : web->curl_ca_bundle );
 335         tr_ninf( "web", "NB: this only works if you built against libcurl with openssl or gnutls, NOT nss" );
 336         tr_ninf( "web", "NB: invalid certs will show up as 'Could not connect to tracker' like many other errors" );
 337     }
 338     web->cookie_filename = tr_buildPath( session->configDir, "cookies.txt", NULL );
 339
 340     multi = curl_multi_init( );
 341     session->web = web;
 342
 343     for( ;; )
 344     {
 345         long msec;
 346         int unused;
 347         CURLMsg * msg;
 348         CURLMcode mcode;
 349
 350         if( web->close_mode == TR_WEB_CLOSE_NOW )
 351             break;
 352         if( ( web->close_mode == TR_WEB_CLOSE_WHEN_IDLE ) && ( web->tasks == NULL ) )
 353             break;
 354
 355         /* add tasks from the queue */
 356         tr_lockLock( web->taskLock );
 357         while( web->tasks != NULL )
 358         {
 359             /* pop the task */
 360             task = web->tasks;
 361             web->tasks = task->next;
 362             task->next = NULL;
 363
 364             dbgmsg( "adding task to curl: [%s]", task->url );
 365             curl_multi_add_handle( multi, createEasy( session, web, task ));
 366             /*fprintf( stderr, "adding a task.. taskCount is now %d\n", taskCount );*/
 367             ++taskCount;
 368         }
 369         tr_lockUnlock( web->taskLock );
 370
 371         /* maybe wait a little while before calling curl_multi_perform() */
 372         msec = 0;
 373         curl_multi_timeout( multi, &msec );
 374         if( msec < 0 )
 375             msec = THREADFUNC_MAX_SLEEP_MSEC;
 376         if( session->isClosed )
 377             msec = 100; /* on shutdown, call perform() more frequently */
 378         if( msec > 0 )
 379         {
 380             int usec;
 381             int max_fd;
 382             struct timeval t;
 383             fd_set r_fd_set, w_fd_set, c_fd_set;
 384
 385             max_fd = 0;
 386             FD_ZERO( &r_fd_set );
 387             FD_ZERO( &w_fd_set );
 388             FD_ZERO( &c_fd_set );
 389             curl_multi_fdset( multi, &r_fd_set, &w_fd_set, &c_fd_set, &max_fd );
 390
 391             if( msec > THREADFUNC_MAX_SLEEP_MSEC )
 392                 msec = THREADFUNC_MAX_SLEEP_MSEC;
 393
 394             usec = msec * 1000;
 395             t.tv_sec =  usec / 1000000;
 396             t.tv_usec = usec % 1000000;
 397             tr_select( max_fd+1, &r_fd_set, &w_fd_set, &c_fd_set, &t );
 398         }
 399
 400         /* call curl_multi_perform() */
 401         do {
 402             mcode = curl_multi_perform( multi, &unused );
 403         } while( mcode == CURLM_CALL_MULTI_PERFORM );
 404
 405         /* pump completed tasks from the multi */
 406         while(( msg = curl_multi_info_read( multi, &unused )))
 407         {
 408             if(( msg->msg == CURLMSG_DONE ) && ( msg->easy_handle != NULL ))
 409             {
 410                 double total_time;
 411                 struct tr_web_task * task;
 412                 long req_bytes_sent;
 413                 CURL * e = msg->easy_handle;
 414                 curl_easy_getinfo( e, CURLINFO_PRIVATE, (void*)&task );
 415                 curl_easy_getinfo( e, CURLINFO_RESPONSE_CODE, &task->code );
 416                 curl_easy_getinfo( e, CURLINFO_REQUEST_SIZE, &req_bytes_sent );
 417                 curl_easy_getinfo( e, CURLINFO_TOTAL_TIME, &total_time );
 418                 task->did_connect = task->code>0 || req_bytes_sent>0;
 419                 task->did_timeout = !task->code && ( total_time >= task->timeout_secs );
 420                 curl_multi_remove_handle( multi, e );
 421                 curl_easy_cleanup( e );
 422 /*fprintf( stderr, "removing a completed task.. taskCount is now %d (response code: %d, response len: %d)\n", taskCount, (int)task->code, (int)evbuffer_get_length(task->response) );*/
 423                 tr_runInEventThread( task->session, task_finish_func, task );
 424                 --taskCount;
 425             }
 426         }
 427     }
 428
 429     /* Discard any remaining tasks.
 430      * This is rare, but can happen on shutdown with unresponsive trackers. */
 431     while( web->tasks != NULL ) {
 432         task = web->tasks;
 433         web->tasks = task->next;
 434         dbgmsg( "Discarding task \"%s\"", task->url );
 435         task_free( task );
 436     }
 437
 438     /* cleanup */
 439     curl_multi_cleanup( multi );
 440     tr_lockFree( web->taskLock );
 441     tr_free( web->cookie_filename );
 442     tr_free( web );
 443     session->web = NULL;
 444 }
 445
 446 void
 447 tr_webInit( tr_session * session )
 448 {
 449     tr_threadNew( tr_webThreadFunc, session );
 450 }
 451
 452 void
 453 tr_webClose( tr_session * session, tr_web_close_mode close_mode )
 454 {
 455     if( session->web != NULL )
 456     {
 457         session->web->close_mode = close_mode;
 458
 459         if( close_mode == TR_WEB_CLOSE_NOW )
 460             while( session->web != NULL )
 461                 tr_wait_msec( 100 );
 462     }
 463 }
 464
 465 void
 466 tr_webGetTaskInfo( struct tr_web_task * task, tr_web_task_info info, void * dst )
 467 {
 468     curl_easy_getinfo( task->curl_easy, (CURLINFO) info, dst );
 469 }
 470
 471 /*****
 472 ******
 473 ******
 474 *****/
 475
 476 const char *
 477 tr_webGetResponseStr( long code )
 478 {
 479     switch( code )
 480     {
 481         case   0: return "No Response";
 482         case 101: return "Switching Protocols";
 483         case 200: return "OK";
 484         case 201: return "Created";
 485         case 202: return "Accepted";
 486         case 203: return "Non-Authoritative Information";
 487         case 204: return "No Content";
 488         case 205: return "Reset Content";
 489         case 206: return "Partial Content";
 490         case 300: return "Multiple Choices";
 491         case 301: return "Moved Permanently";
 492         case 302: return "Found";
 493         case 303: return "See Other";
 494         case 304: return "Not Modified";
 495         case 305: return "Use Proxy";
 496         case 306: return "(Unused)";
 497         case 307: return "Temporary Redirect";
 498         case 400: return "Bad Request";
 499         case 401: return "Unauthorized";
 500         case 402: return "Payment Required";
 501         case 403: return "Forbidden";
 502         case 404: return "Not Found";
 503         case 405: return "Method Not Allowed";
 504         case 406: return "Not Acceptable";
 505         case 407: return "Proxy Authentication Required";
 506         case 408: return "Request Timeout";
 507         case 409: return "Conflict";
 508         case 410: return "Gone";
 509         case 411: return "Length Required";
 510         case 412: return "Precondition Failed";
 511         case 413: return "Request Entity Too Large";
 512         case 414: return "Request-URI Too Long";
 513         case 415: return "Unsupported Media Type";
 514         case 416: return "Requested Range Not Satisfiable";
 515         case 417: return "Expectation Failed";
 516         case 500: return "Internal Server Error";
 517         case 501: return "Not Implemented";
 518         case 502: return "Bad Gateway";
 519         case 503: return "Service Unavailable";
 520         case 504: return "Gateway Timeout";
 521         case 505: return "HTTP Version Not Supported";
 522         default:  return "Unknown Error";
 523     }
 524 }
 525
 526 void
 527 tr_http_escape( struct evbuffer  * out,
 528                 const char * str, int len, bool escape_slashes )
 529 {
 530     const char * end;
 531
 532     if( ( len < 0 ) && ( str != NULL ) )
 533         len = strlen( str );
 534
 535     for( end=str+len; str && str!=end; ++str ) {
 536         if(    ( *str == ',' )
 537             || ( *str == '-' )
 538             || ( *str == '.' )
 539             || ( ( '0' <= *str ) && ( *str <= '9' ) )
 540             || ( ( 'A' <= *str ) && ( *str <= 'Z' ) )
 541             || ( ( 'a' <= *str ) && ( *str <= 'z' ) )
 542             || ( ( *str == '/' ) && ( !escape_slashes ) ) )
 543             evbuffer_add_printf( out, "%c", *str );
 544         else
 545             evbuffer_add_printf( out, "%%%02X", (unsigned)(*str&0xFF) );
 546     }
 547 }
 548
 549 char *
 550 tr_http_unescape( const char * str, int len )
 551 {
 552     char * tmp = curl_unescape( str, len );
 553     char * ret = tr_strdup( tmp );
 554     curl_free( tmp );
 555     return ret;
 556 }
 557
 558 static int
 559 is_rfc2396_alnum( uint8_t ch )
 560 {
 561     return ( '0' <= ch && ch <= '9' )
 562         || ( 'A' <= ch && ch <= 'Z' )
 563         || ( 'a' <= ch && ch <= 'z' )
 564         || ch == '.'
 565         || ch == '-'
 566         || ch == '_'
 567         || ch == '~';
 568 }
 569
 570 void
 571 tr_http_escape_sha1( char * out, const uint8_t * sha1_digest )
 572 {
 573     const uint8_t * in = sha1_digest;
 574     const uint8_t * end = in + SHA_DIGEST_LENGTH;
 575
 576     while( in != end )
 577         if( is_rfc2396_alnum( *in ) )
 578             *out++ = (char) *in++;
 579         else
 580             out += tr_snprintf( out, 4, "%%%02x", (unsigned int)*in++ );
 581
 582     *out = '\0';
 583 }