release/src/router/transmission/libtransmission/web.c

   1 /*
   2  * This file Copyright (C) Mnemosyne LLC
   3  *
   4  * This file is licensed by the GPL version 2. Works owned by the
   5  * Transmission project are granted a special exemption to clause 2(b)
   6  * so that the bulk of its code can remain under the MIT license.
   7  * This exemption does not extend to derived works not owned by
   8  * the Transmission project.
   9  *
  10  * $Id: web.c 12539 2011-07-10 15:24:51Z jordan $
  11  */
  12
  13 #include <string.h> /* strlen(), strstr() */
  14 #include <stdlib.h> /* getenv() */
  15
  16 #ifdef WIN32
  17   #include <ws2tcpip.h>
  18 #else
  19   #include <sys/select.h>
  20 #endif
  21
  22 #include <curl/curl.h>
  23
  24 #include <event2/buffer.h>
  25
  26 #include "transmission.h"
  27 #include "net.h" /* tr_address */
  28 #include "platform.h" /* mutex */
  29 #include "session.h"
  30 #include "trevent.h" /* tr_runInEventThread() */
  31 #include "utils.h"
  32 #include "version.h" /* User-Agent */
  33 #include "web.h"
  34
  35 #if LIBCURL_VERSION_NUM >= 0x070F06 /* CURLOPT_SOCKOPT* was added in 7.15.6 */
  36  #define USE_LIBCURL_SOCKOPT
  37 #endif
  38
  39 enum
  40 {
  41     THREADFUNC_MAX_SLEEP_MSEC = 1000,
  42 };
  43
  44 #if 0
  45 #define dbgmsg(...) \
  46     do { \
  47         fprintf( stderr, __VA_ARGS__ ); \
  48         fprintf( stderr, "\n" ); \
  49     } while( 0 )
  50 #else
  51 #define dbgmsg( ... ) \
  52     do { \
  53         if( tr_deepLoggingIsActive( ) ) \
  54             tr_deepLog( __FILE__, __LINE__, "web", __VA_ARGS__ ); \
  55     } while( 0 )
  56 #endif
  57
  58 /***
  59 ****
  60 ***/
  61
  62 struct tr_web_task
  63 {
  64     long code;
  65     long timeout_secs;
  66     bool did_connect;
  67     bool did_timeout;
  68     struct evbuffer * response;
  69     struct evbuffer * freebuf;
  70     char * url;
  71     char * range;
  72     char * cookies;
  73     tr_session * session;
  74     tr_web_done_func * done_func;
  75     void * done_func_user_data;
  76     CURL * curl_easy;
  77     struct tr_web_task * next;
  78 };
  79
  80 static void
  81 task_free( struct tr_web_task * task )
  82 {
  83     if( task->freebuf )
  84         evbuffer_free( task->freebuf );
  85     tr_free( task->cookies );
  86     tr_free( task->range );
  87     tr_free( task->url );
  88     tr_free( task );
  89 }
  90
  91 /***
  92 ****
  93 ***/
  94
  95 struct tr_web
  96 {
  97     bool curl_verbose;
  98     int close_mode;
  99     struct tr_web_task * tasks;
 100     tr_lock * taskLock;
 101     char * cookie_filename;
 102 };
 103
 104 /***
 105 ****
 106 ***/
 107
 108 static size_t
 109 writeFunc( void * ptr, size_t size, size_t nmemb, void * vtask )
 110 {
 111     const size_t byteCount = size * nmemb;
 112     struct tr_web_task * task = vtask;
 113     evbuffer_add( task->response, ptr, byteCount );
 114     dbgmsg( "wrote %zu bytes to task %p's buffer", byteCount, task );
 115     return byteCount;
 116 }
 117
 118 #ifdef USE_LIBCURL_SOCKOPT
 119 static int
 120 sockoptfunction( void * vtask, curl_socket_t fd, curlsocktype purpose UNUSED )
 121 {
 122     struct tr_web_task * task = vtask;
 123     const bool isScrape = strstr( task->url, "scrape" ) != NULL;
 124     const bool isAnnounce = strstr( task->url, "announce" ) != NULL;
 125
 126     /* announce and scrape requests have tiny payloads. */
 127     if( isScrape || isAnnounce )
 128     {
 129         const int sndbuf = 1024;
 130         const int rcvbuf = isScrape ? 2048 : 3072;
 131         setsockopt( fd, SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(sndbuf) );
 132         setsockopt( fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(rcvbuf) );
 133     }
 134
 135     /* return nonzero if this function encountered an error */
 136     return 0;
 137 }
 138 #endif
 139
 140 static long
 141 getTimeoutFromURL( const struct tr_web_task * task )
 142 {
 143     long timeout;
 144     const tr_session * session = task->session;
 145
 146     if( !session || session->isClosed ) timeout = 20L;
 147     else if( strstr( task->url, "scrape" ) != NULL ) timeout = 30L;
 148     else if( strstr( task->url, "announce" ) != NULL ) timeout = 90L;
 149     else timeout = 240L;
 150
 151     return timeout;
 152 }
 153
 154 static CURL *
 155 createEasy( tr_session * s, struct tr_web * web, struct tr_web_task * task )
 156 {
 157     bool is_default_value;
 158     const tr_address * addr;
 159     CURL * e = task->curl_easy = curl_easy_init( );
 160
 161     task->timeout_secs = getTimeoutFromURL( task );
 162
 163     curl_easy_setopt( e, CURLOPT_AUTOREFERER, 1L );
 164     curl_easy_setopt( e, CURLOPT_COOKIEFILE, web->cookie_filename );
 165     curl_easy_setopt( e, CURLOPT_ENCODING, "gzip;q=1.0, deflate, identity" );
 166     curl_easy_setopt( e, CURLOPT_FOLLOWLOCATION, 1L );
 167     curl_easy_setopt( e, CURLOPT_MAXREDIRS, -1L );
 168     curl_easy_setopt( e, CURLOPT_NOSIGNAL, 1L );
 169     curl_easy_setopt( e, CURLOPT_PRIVATE, task );
 170 #ifdef USE_LIBCURL_SOCKOPT
 171     curl_easy_setopt( e, CURLOPT_SOCKOPTFUNCTION, sockoptfunction );
 172     curl_easy_setopt( e, CURLOPT_SOCKOPTDATA, task );
 173 #endif
 174     curl_easy_setopt( e, CURLOPT_SSL_VERIFYHOST, 0L );
 175     curl_easy_setopt( e, CURLOPT_SSL_VERIFYPEER, 0L );
 176     curl_easy_setopt( e, CURLOPT_TIMEOUT, task->timeout_secs );
 177     curl_easy_setopt( e, CURLOPT_URL, task->url );
 178     curl_easy_setopt( e, CURLOPT_USERAGENT, TR_NAME "/" SHORT_VERSION_STRING );
 179     curl_easy_setopt( e, CURLOPT_VERBOSE, (long)(web->curl_verbose?1:0) );
 180     curl_easy_setopt( e, CURLOPT_WRITEDATA, task );
 181     curl_easy_setopt( e, CURLOPT_WRITEFUNCTION, writeFunc );
 182
 183     if((( addr = tr_sessionGetPublicAddress( s, TR_AF_INET, &is_default_value ))) && !is_default_value )
 184         curl_easy_setopt( e, CURLOPT_INTERFACE, tr_address_to_string( addr ) );
 185     else if ((( addr = tr_sessionGetPublicAddress( s, TR_AF_INET6, &is_default_value ))) && !is_default_value )
 186         curl_easy_setopt( e, CURLOPT_INTERFACE, tr_address_to_string( addr ) );
 187
 188     if( task->cookies != NULL )
 189         curl_easy_setopt( e, CURLOPT_COOKIE, task->cookies );
 190
 191     if( task->range )
 192         curl_easy_setopt( e, CURLOPT_RANGE, task->range );
 193
 194     return e;
 195 }
 196
 197 /***
 198 ****
 199 ***/
 200
 201 static void
 202 task_finish_func( void * vtask )
 203 {
 204     struct tr_web_task * task = vtask;
 205     dbgmsg( "finished web task %p; got %ld", task, task->code );
 206
 207     if( task->done_func != NULL )
 208         task->done_func( task->session,
 209                          task->did_connect,
 210                          task->did_timeout,
 211                          task->code,
 212                          evbuffer_pullup( task->response, -1 ),
 213                          evbuffer_get_length( task->response ),
 214                          task->done_func_user_data );
 215
 216     task_free( task );
 217 }
 218
 219 /****
 220 *****
 221 ****/
 222
 223 struct tr_web_task *
 224 tr_webRun( tr_session         * session,
 225            const char         * url,
 226            const char         * range,
 227            const char         * cookies,
 228            tr_web_done_func     done_func,
 229            void               * done_func_user_data )
 230 {
 231     return tr_webRunWithBuffer( session, url, range, cookies,
 232                                 done_func, done_func_user_data,
 233                                 NULL );
 234 }
 235
 236 struct tr_web_task *
 237 tr_webRunWithBuffer( tr_session         * session,
 238                      const char         * url,
 239                      const char         * range,
 240                      const char         * cookies,
 241                      tr_web_done_func     done_func,
 242                      void               * done_func_user_data,
 243                      struct evbuffer    * buffer )
 244 {
 245     struct tr_web * web = session->web;
 246
 247     if( web != NULL )
 248     {
 249         struct tr_web_task * task = tr_new0( struct tr_web_task, 1 );
 250
 251         task->session = session;
 252         task->url = tr_strdup( url );
 253         task->range = tr_strdup( range );
 254         task->cookies = tr_strdup( cookies);
 255         task->done_func = done_func;
 256         task->done_func_user_data = done_func_user_data;
 257         task->response = buffer ? buffer : evbuffer_new( );
 258         task->freebuf = buffer ? NULL : task->response;
 259
 260         tr_lockLock( web->taskLock );
 261         task->next = web->tasks;
 262         web->tasks = task;
 263         tr_lockUnlock( web->taskLock );
 264         return task;
 265     }
 266     return NULL;
 267 }
 268
 269 /**
 270  * Portability wrapper for select().
 271  *
 272  * http://msdn.microsoft.com/en-us/library/ms740141%28VS.85%29.aspx
 273  * On win32, any two of the parameters, readfds, writefds, or exceptfds,
 274  * can be given as null. At least one must be non-null, and any non-null
 275  * descriptor set must contain at least one handle to a socket.
 276  */
 277 static void
 278 tr_select( int nfds,
 279            fd_set * r_fd_set, fd_set * w_fd_set, fd_set * c_fd_set,
 280            struct timeval  * t )
 281 {
 282 #ifdef WIN32
 283     if( !r_fd_set->fd_count && !w_fd_set->fd_count && !c_fd_set->fd_count )
 284     {
 285         const long int msec = t->tv_sec*1000 + t->tv_usec/1000;
 286         tr_wait_msec( msec );
 287     }
 288     else if( select( 0, r_fd_set->fd_count ? r_fd_set : NULL,
 289                         w_fd_set->fd_count ? w_fd_set : NULL,
 290                         c_fd_set->fd_count ? c_fd_set : NULL, t ) < 0 )
 291     {
 292         char errstr[512];
 293         const int e = EVUTIL_SOCKET_ERROR( );
 294         tr_net_strerror( errstr, sizeof( errstr ), e );
 295         dbgmsg( "Error: select (%d) %s", e, errstr );
 296     }
 297 #else
 298     select( nfds, r_fd_set, w_fd_set, c_fd_set, t );
 299 #endif
 300 }
 301
 302 static void
 303 tr_webThreadFunc( void * vsession )
 304 {
 305     CURLM * multi;
 306     struct tr_web * web;
 307     int taskCount = 0;
 308     struct tr_web_task * task;
 309     tr_session * session = vsession;
 310
 311     /* try to enable ssl for https support; but if that fails,
 312      * try a plain vanilla init */
 313     if( curl_global_init( CURL_GLOBAL_SSL ) )
 314         curl_global_init( 0 );
 315
 316     web = tr_new0( struct tr_web, 1 );
 317     web->close_mode = ~0;
 318     web->taskLock = tr_lockNew( );
 319     web->tasks = NULL;
 320     web->curl_verbose = getenv( "TR_CURL_VERBOSE" ) != NULL;
 321     web->cookie_filename = tr_buildPath( session->configDir, "cookies.txt", NULL );
 322
 323     multi = curl_multi_init( );
 324     session->web = web;
 325
 326     for( ;; )
 327     {
 328         long msec;
 329         int unused;
 330         CURLMsg * msg;
 331         CURLMcode mcode;
 332
 333         if( web->close_mode == TR_WEB_CLOSE_NOW )
 334             break;
 335         if( ( web->close_mode == TR_WEB_CLOSE_WHEN_IDLE ) && ( web->tasks == NULL ) )
 336             break;
 337
 338         /* add tasks from the queue */
 339         tr_lockLock( web->taskLock );
 340         while( web->tasks != NULL )
 341         {
 342             /* pop the task */
 343             task = web->tasks;
 344             web->tasks = task->next;
 345             task->next = NULL;
 346
 347             dbgmsg( "adding task to curl: [%s]", task->url );
 348             curl_multi_add_handle( multi, createEasy( session, web, task ));
 349             /*fprintf( stderr, "adding a task.. taskCount is now %d\n", taskCount );*/
 350             ++taskCount;
 351         }
 352         tr_lockUnlock( web->taskLock );
 353
 354         /* maybe wait a little while before calling curl_multi_perform() */
 355         msec = 0;
 356         curl_multi_timeout( multi, &msec );
 357         if( msec < 0 )
 358             msec = THREADFUNC_MAX_SLEEP_MSEC;
 359         if( session->isClosed )
 360             msec = 100; /* on shutdown, call perform() more frequently */
 361         if( msec > 0 )
 362         {
 363             int usec;
 364             int max_fd;
 365             struct timeval t;
 366             fd_set r_fd_set, w_fd_set, c_fd_set;
 367
 368             max_fd = 0;
 369             FD_ZERO( &r_fd_set );
 370             FD_ZERO( &w_fd_set );
 371             FD_ZERO( &c_fd_set );
 372             curl_multi_fdset( multi, &r_fd_set, &w_fd_set, &c_fd_set, &max_fd );
 373
 374             if( msec > THREADFUNC_MAX_SLEEP_MSEC )
 375                 msec = THREADFUNC_MAX_SLEEP_MSEC;
 376
 377             usec = msec * 1000;
 378             t.tv_sec =  usec / 1000000;
 379             t.tv_usec = usec % 1000000;
 380             tr_select( max_fd+1, &r_fd_set, &w_fd_set, &c_fd_set, &t );
 381         }
 382
 383         /* call curl_multi_perform() */
 384         do {
 385             mcode = curl_multi_perform( multi, &unused );
 386         } while( mcode == CURLM_CALL_MULTI_PERFORM );
 387
 388         /* pump completed tasks from the multi */
 389         while(( msg = curl_multi_info_read( multi, &unused )))
 390         {
 391             if(( msg->msg == CURLMSG_DONE ) && ( msg->easy_handle != NULL ))
 392             {
 393                 double total_time;
 394                 struct tr_web_task * task;
 395                 long req_bytes_sent;
 396                 CURL * e = msg->easy_handle;
 397                 curl_easy_getinfo( e, CURLINFO_PRIVATE, (void*)&task );
 398                 curl_easy_getinfo( e, CURLINFO_RESPONSE_CODE, &task->code );
 399                 curl_easy_getinfo( e, CURLINFO_REQUEST_SIZE, &req_bytes_sent );
 400                 curl_easy_getinfo( e, CURLINFO_TOTAL_TIME, &total_time );
 401                 task->did_connect = task->code>0 || req_bytes_sent>0;
 402                 task->did_timeout = !task->code && ( total_time >= task->timeout_secs );
 403                 curl_multi_remove_handle( multi, e );
 404                 curl_easy_cleanup( e );
 405 /*fprintf( stderr, "removing a completed task.. taskCount is now %d (response code: %d, response len: %d)\n", taskCount, (int)task->code, (int)evbuffer_get_length(task->response) );*/
 406                 tr_runInEventThread( task->session, task_finish_func, task );
 407                 --taskCount;
 408             }
 409         }
 410     }
 411
 412     /* Discard any remaining tasks.
 413      * This is rare, but can happen on shutdown with unresponsive trackers. */
 414     while( web->tasks != NULL ) {
 415         task = web->tasks;
 416         web->tasks = task->next;
 417         dbgmsg( "Discarding task \"%s\"", task->url );
 418         task_free( task );
 419     }
 420
 421     /* cleanup */
 422     curl_multi_cleanup( multi );
 423     tr_lockFree( web->taskLock );
 424     tr_free( web->cookie_filename );
 425     tr_free( web );
 426     session->web = NULL;
 427 }
 428
 429 void
 430 tr_webInit( tr_session * session )
 431 {
 432     tr_threadNew( tr_webThreadFunc, session );
 433 }
 434
 435 void
 436 tr_webClose( tr_session * session, tr_web_close_mode close_mode )
 437 {
 438     if( session->web != NULL )
 439     {
 440         session->web->close_mode = close_mode;
 441
 442         if( close_mode == TR_WEB_CLOSE_NOW )
 443             while( session->web != NULL )
 444                 tr_wait_msec( 100 );
 445     }
 446 }
 447
 448 void
 449 tr_webGetTaskInfo( struct tr_web_task * task, tr_web_task_info info, void * dst )
 450 {
 451     curl_easy_getinfo( task->curl_easy, (CURLINFO) info, dst );
 452 }
 453
 454 /*****
 455 ******
 456 ******
 457 *****/
 458
 459 const char *
 460 tr_webGetResponseStr( long code )
 461 {
 462     switch( code )
 463     {
 464         case   0: return "No Response";
 465         case 101: return "Switching Protocols";
 466         case 200: return "OK";
 467         case 201: return "Created";
 468         case 202: return "Accepted";
 469         case 203: return "Non-Authoritative Information";
 470         case 204: return "No Content";
 471         case 205: return "Reset Content";
 472         case 206: return "Partial Content";
 473         case 300: return "Multiple Choices";
 474         case 301: return "Moved Permanently";
 475         case 302: return "Found";
 476         case 303: return "See Other";
 477         case 304: return "Not Modified";
 478         case 305: return "Use Proxy";
 479         case 306: return "(Unused)";
 480         case 307: return "Temporary Redirect";
 481         case 400: return "Bad Request";
 482         case 401: return "Unauthorized";
 483         case 402: return "Payment Required";
 484         case 403: return "Forbidden";
 485         case 404: return "Not Found";
 486         case 405: return "Method Not Allowed";
 487         case 406: return "Not Acceptable";
 488         case 407: return "Proxy Authentication Required";
 489         case 408: return "Request Timeout";
 490         case 409: return "Conflict";
 491         case 410: return "Gone";
 492         case 411: return "Length Required";
 493         case 412: return "Precondition Failed";
 494         case 413: return "Request Entity Too Large";
 495         case 414: return "Request-URI Too Long";
 496         case 415: return "Unsupported Media Type";
 497         case 416: return "Requested Range Not Satisfiable";
 498         case 417: return "Expectation Failed";
 499         case 500: return "Internal Server Error";
 500         case 501: return "Not Implemented";
 501         case 502: return "Bad Gateway";
 502         case 503: return "Service Unavailable";
 503         case 504: return "Gateway Timeout";
 504         case 505: return "HTTP Version Not Supported";
 505         default:  return "Unknown Error";
 506     }
 507 }
 508
 509 void
 510 tr_http_escape( struct evbuffer  * out,
 511                 const char * str, int len, bool escape_slashes )
 512 {
 513     const char * end;
 514
 515     if( ( len < 0 ) && ( str != NULL ) )
 516         len = strlen( str );
 517
 518     for( end=str+len; str && str!=end; ++str ) {
 519         if(    ( *str == ',' )
 520             || ( *str == '-' )
 521             || ( *str == '.' )
 522             || ( ( '0' <= *str ) && ( *str <= '9' ) )
 523             || ( ( 'A' <= *str ) && ( *str <= 'Z' ) )
 524             || ( ( 'a' <= *str ) && ( *str <= 'z' ) )
 525             || ( ( *str == '/' ) && ( !escape_slashes ) ) )
 526             evbuffer_add_printf( out, "%c", *str );
 527         else
 528             evbuffer_add_printf( out, "%%%02X", (unsigned)(*str&0xFF) );
 529     }
 530 }
 531
 532 char *
 533 tr_http_unescape( const char * str, int len )
 534 {
 535     char * tmp = curl_unescape( str, len );
 536     char * ret = tr_strdup( tmp );
 537     curl_free( tmp );
 538     return ret;
 539 }
 540
 541 static int
 542 is_rfc2396_alnum( uint8_t ch )
 543 {
 544     return ( '0' <= ch && ch <= '9' )
 545         || ( 'A' <= ch && ch <= 'Z' )
 546         || ( 'a' <= ch && ch <= 'z' )
 547         || ch == '.'
 548         || ch == '-'
 549         || ch == '_'
 550         || ch == '~';
 551 }
 552
 553 void
 554 tr_http_escape_sha1( char * out, const uint8_t * sha1_digest )
 555 {
 556     const uint8_t * in = sha1_digest;
 557     const uint8_t * end = in + SHA_DIGEST_LENGTH;
 558
 559     while( in != end )
 560         if( is_rfc2396_alnum( *in ) )
 561             *out++ = (char) *in++;
 562         else
 563             out += tr_snprintf( out, 4, "%%%02x", (unsigned int)*in++ );
 564
 565     *out = '\0';
 566 }