libtorrent/src/http_connection.cpp

   1 /*
   2
   3 Copyright (c) 2007, Arvid Norberg
   4 All rights reserved.
   5
   6 Redistribution and use in source and binary forms, with or without
   7 modification, are permitted provided that the following conditions
   8 are met:
   9
  10     * Redistributions of source code must retain the above copyright
  11       notice, this list of conditions and the following disclaimer.
  12     * Redistributions in binary form must reproduce the above copyright
  13       notice, this list of conditions and the following disclaimer in
  14       the documentation and/or other materials provided with the distribution.
  15     * Neither the name of the author nor the names of its
  16       contributors may be used to endorse or promote products derived
  17       from this software without specific prior written permission.
  18
  19 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  20 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  21 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  22 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  23 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  24 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  25 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  26 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  27 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  28 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  29 POSSIBILITY OF SUCH DAMAGE.
  30
  31 */
  32
  33 #include "libtorrent/http_connection.hpp"
  34 #include "libtorrent/escape_string.hpp"
  35 #include "libtorrent/instantiate_connection.hpp"
  36 #include "libtorrent/gzip.hpp"
  37 #include "libtorrent/parse_url.hpp"
  38 #include "libtorrent/socket.hpp"
  39 #include "libtorrent/connection_queue.hpp"
  40
  41 #include <boost/bind.hpp>
  42 #include <boost/lexical_cast.hpp>
  43 #include <string>
  44 #include <algorithm>
  45
  46 using boost::bind;
  47
  48 namespace libtorrent {
  49
  50 enum { max_bottled_buffer = 1024 * 1024 };
  51
  52
  53 void http_connection::get(std::string const& url, time_duration timeout, int prio
  54         , proxy_settings const* ps, int handle_redirects, std::string const& user_agent
  55         , address const& bind_addr)
  56 {
  57         std::string protocol;
  58         std::string auth;
  59         std::string hostname;
  60         std::string path;
  61         char const* error;
  62         int port;
  63
  64         boost::tie(protocol, auth, hostname, port, path, error)
  65                 = parse_url_components(url);
  66
  67         if (error)
  68         {
  69                 callback(asio::error::socket_type_not_supported);
  70                 return;
  71         }
  72
  73         TORRENT_ASSERT(prio >= 0 && prio < 2);
  74
  75         bool ssl = false;
  76         if (protocol == "https") ssl = true;
  77 #ifndef TORRENT_USE_OPENSSL
  78         if (ssl)
  79         {
  80                 callback(asio::error::socket_type_not_supported);
  81                 return;
  82         }
  83 #endif
  84
  85         std::stringstream headers;
  86         if (ps && (ps->type == proxy_settings::http
  87                 || ps->type == proxy_settings::http_pw)
  88                 && !ssl)
  89         {
  90                 // if we're using an http proxy and not an ssl
  91                 // connection, just do a regular http proxy request
  92                 headers << "GET " << url << " HTTP/1.0\r\n";
  93                 if (ps->type == proxy_settings::http_pw)
  94                         headers << "Proxy-Authorization: Basic " << base64encode(
  95                                 ps->username + ":" + ps->password) << "\r\n";
  96                 hostname = ps->hostname;
  97                 port = ps->port;
  98                 ps = 0;
  99         }
 100         else
 101         {
 102                 headers << "GET " << path << " HTTP/1.0\r\n"
 103                         "Host:" << hostname << "\r\n";
 104         }
 105
 106         if (!auth.empty())
 107                 headers << "Authorization: Basic " << base64encode(auth) << "\r\n";
 108
 109         if (!user_agent.empty())
 110                 headers << "User-Agent: " << user_agent << "\r\n";
 111
 112         headers <<
 113                 "Connection: close\r\n"
 114                 "Accept-Encoding: gzip\r\n"
 115                 "\r\n";
 116
 117         sendbuffer = headers.str();
 118         m_url = url;
 119         start(hostname, boost::lexical_cast<std::string>(port), timeout, prio
 120                 , ps, ssl, handle_redirects, bind_addr);
 121 }
 122
 123 void http_connection::start(std::string const& hostname, std::string const& port
 124         , time_duration timeout, int prio, proxy_settings const* ps, bool ssl, int handle_redirects
 125         , address const& bind_addr)
 126 {
 127         TORRENT_ASSERT(prio >= 0 && prio < 2);
 128
 129         m_redirects = handle_redirects;
 130         if (ps) m_proxy = *ps;
 131
 132         m_timeout = timeout;
 133         error_code ec;
 134         m_timer.expires_from_now(m_timeout, ec);
 135         m_timer.async_wait(bind(&http_connection::on_timeout
 136                 , boost::weak_ptr<http_connection>(shared_from_this()), _1));
 137         m_called = false;
 138         m_parser.reset();
 139         m_recvbuffer.clear();
 140         m_read_pos = 0;
 141         m_priority = prio;
 142
 143         if (ec)
 144         {
 145                 callback(ec);
 146                 return;
 147         }
 148
 149         if (m_sock.is_open() && m_hostname == hostname && m_port == port
 150                 && m_ssl == ssl && m_bind_addr == bind_addr)
 151         {
 152                 async_write(m_sock, asio::buffer(sendbuffer)
 153                         , bind(&http_connection::on_write, shared_from_this(), _1));
 154         }
 155         else
 156         {
 157                 m_ssl = ssl;
 158                 m_bind_addr = bind_addr;
 159                 error_code ec;
 160                 m_sock.close(ec);
 161
 162 #ifdef TORRENT_USE_OPENSSL
 163                 if (m_ssl)
 164                 {
 165                         m_sock.instantiate<ssl_stream<socket_type> >(m_resolver.get_io_service());
 166                         ssl_stream<socket_type>* s = m_sock.get<ssl_stream<socket_type> >();
 167                         TORRENT_ASSERT(s);
 168                         bool ret = instantiate_connection(m_resolver.get_io_service(), m_proxy, s->next_layer());
 169                         TORRENT_ASSERT(ret);
 170                 }
 171                 else
 172                 {
 173                         m_sock.instantiate<socket_type>(m_resolver.get_io_service());
 174                         bool ret = instantiate_connection(m_resolver.get_io_service()
 175                                 , m_proxy, *m_sock.get<socket_type>());
 176                         TORRENT_ASSERT(ret);
 177                 }
 178 #else
 179                 bool ret = instantiate_connection(m_resolver.get_io_service(), m_proxy, m_sock);
 180                 TORRENT_ASSERT(ret);
 181 #endif
 182                 if (m_bind_addr != address_v4::any())
 183                 {
 184                         error_code ec;
 185                         m_sock.bind(tcp::endpoint(m_bind_addr, 0), ec);
 186                         if (ec)
 187                         {
 188                                 callback(ec);
 189                                 return;
 190                         }
 191                 }
 192
 193                 tcp::resolver::query query(hostname, port);
 194                 m_resolver.async_resolve(query, bind(&http_connection::on_resolve
 195                         , shared_from_this(), _1, _2));
 196                 m_hostname = hostname;
 197                 m_port = port;
 198         }
 199 }
 200
 201 void http_connection::on_connect_timeout()
 202 {
 203         if (m_connection_ticket > -1) m_cc.done(m_connection_ticket);
 204         m_connection_ticket = -1;
 205
 206         if (!m_endpoints.empty())
 207         {
 208                 error_code ec;
 209                 m_sock.close(ec);
 210         }
 211         else
 212         {
 213                 callback(asio::error::timed_out);
 214                 close();
 215         }
 216 }
 217
 218 void http_connection::on_timeout(boost::weak_ptr<http_connection> p
 219         , error_code const& e)
 220 {
 221         boost::shared_ptr<http_connection> c = p.lock();
 222         if (!c) return;
 223
 224         if (e == asio::error::operation_aborted) return;
 225
 226         if (c->m_last_receive + c->m_timeout < time_now())
 227         {
 228                 if (c->m_connection_ticket > -1 && !c->m_endpoints.empty())
 229                 {
 230                         error_code ec;
 231                         c->m_sock.close(ec);
 232                         c->m_timer.expires_at(c->m_last_receive + c->m_timeout, ec);
 233                         c->m_timer.async_wait(bind(&http_connection::on_timeout, p, _1));
 234                 }
 235                 else
 236                 {
 237                         c->callback(asio::error::timed_out);
 238                         c->close();
 239                 }
 240                 return;
 241         }
 242
 243         if (!c->m_sock.is_open()) return;
 244         error_code ec;
 245         c->m_timer.expires_at(c->m_last_receive + c->m_timeout, ec);
 246         c->m_timer.async_wait(bind(&http_connection::on_timeout, p, _1));
 247 }
 248
 249 void http_connection::close()
 250 {
 251         error_code ec;
 252         m_timer.cancel(ec);
 253         m_limiter_timer.cancel(ec);
 254         m_sock.close(ec);
 255         m_hostname.clear();
 256         m_port.clear();
 257         m_handler.clear();
 258         m_abort = true;
 259 }
 260
 261 void http_connection::on_resolve(error_code const& e
 262         , tcp::resolver::iterator i)
 263 {
 264         if (e)
 265         {
 266                 callback(e);
 267                 close();
 268                 return;
 269         }
 270         TORRENT_ASSERT(i != tcp::resolver::iterator());
 271
 272         std::transform(i, tcp::resolver::iterator(), std::back_inserter(m_endpoints)
 273                 , boost::bind(&tcp::resolver::iterator::value_type::endpoint, _1));
 274
 275         if (m_filter_handler) m_filter_handler(*this, m_endpoints);
 276         if (m_endpoints.empty())
 277         {
 278                 close();
 279                 return;
 280         }
 281
 282         // The following statement causes msvc to crash (ICE). Since it's not
 283         // necessary in the vast majority of cases, just ignore the endpoint
 284         // order for windows
 285 #if !defined _MSC_VER || _MSC_VER > 1310
 286         // sort the endpoints so that the ones with the same IP version as our
 287         // bound listen socket are first. So that when contacting a tracker,
 288         // we'll talk to it from the same IP that we're listening on
 289         std::partition(m_endpoints.begin(), m_endpoints.end()
 290                 , boost::bind(&address::is_v4, boost::bind(&tcp::endpoint::address, _1)) == m_bind_addr.is_v4());
 291 #endif
 292
 293         queue_connect();
 294 }
 295
 296 void http_connection::queue_connect()
 297 {
 298         TORRENT_ASSERT(!m_endpoints.empty());
 299         tcp::endpoint target = m_endpoints.front();
 300         m_endpoints.pop_front();
 301
 302         m_cc.enqueue(bind(&http_connection::connect, shared_from_this(), _1, target)
 303                 , bind(&http_connection::on_connect_timeout, shared_from_this())
 304                 , m_timeout, m_priority);
 305 }
 306
 307 void http_connection::connect(int ticket, tcp::endpoint target_address)
 308 {
 309         m_connection_ticket = ticket;
 310         m_sock.async_connect(target_address, boost::bind(&http_connection::on_connect
 311                 , shared_from_this(), _1));
 312 }
 313
 314 void http_connection::on_connect(error_code const& e)
 315 {
 316         if (m_connection_ticket >= 0)
 317         {
 318                 m_cc.done(m_connection_ticket);
 319                 m_connection_ticket = -1;
 320         }
 321
 322         m_last_receive = time_now();
 323         if (!e)
 324         {
 325                 if (m_connect_handler) m_connect_handler(*this);
 326                 async_write(m_sock, asio::buffer(sendbuffer)
 327                         , bind(&http_connection::on_write, shared_from_this(), _1));
 328         }
 329         else if (!m_endpoints.empty() && !m_abort)
 330         {
 331                 // The connection failed. Try the next endpoint in the list.
 332                 error_code ec;
 333                 m_sock.close(ec);
 334                 queue_connect();
 335         }
 336         else
 337         {
 338                 callback(e);
 339                 close();
 340         }
 341 }
 342
 343 void http_connection::callback(error_code const& e, char const* data, int size)
 344 {
 345         if (!m_bottled || !m_called)
 346         {
 347                 std::vector<char> buf;
 348                 if (m_bottled && m_parser.header_finished())
 349                 {
 350                         std::string const& encoding = m_parser.header("content-encoding");
 351                         if (encoding == "gzip" || encoding == "x-gzip")
 352                         {
 353                                 std::string error;
 354                                 if (inflate_gzip(data, size, buf, max_bottled_buffer, error))
 355                                 {
 356                                         if (m_handler) m_handler(asio::error::fault, m_parser, data, size, *this);
 357                                         close();
 358                                         return;
 359                                 }
 360                                 data = &buf[0];
 361                                 size = int(buf.size());
 362                         }
 363                 }
 364                 m_called = true;
 365                 error_code ec;
 366                 m_timer.cancel(ec);
 367                 if (m_handler) m_handler(e, m_parser, data, size, *this);
 368         }
 369 }
 370
 371 void http_connection::on_write(error_code const& e)
 372 {
 373         if (e)
 374         {
 375                 callback(e);
 376                 close();
 377                 return;
 378         }
 379
 380         std::string().swap(sendbuffer);
 381         m_recvbuffer.resize(4096);
 382
 383         int amount_to_read = m_recvbuffer.size() - m_read_pos;
 384         if (m_rate_limit > 0 && amount_to_read > m_download_quota)
 385         {
 386                 amount_to_read = m_download_quota;
 387                 if (m_download_quota == 0)
 388                 {
 389                         if (!m_limiter_timer_active)
 390                                 on_assign_bandwidth(error_code());
 391                         return;
 392                 }
 393         }
 394         m_sock.async_read_some(asio::buffer(&m_recvbuffer[0] + m_read_pos
 395                 , amount_to_read)
 396                 , bind(&http_connection::on_read
 397                 , shared_from_this(), _1, _2));
 398 }
 399
 400 void http_connection::on_read(error_code const& e
 401         , std::size_t bytes_transferred)
 402 {
 403         if (m_rate_limit)
 404         {
 405                 m_download_quota -= bytes_transferred;
 406                 TORRENT_ASSERT(m_download_quota >= 0);
 407         }
 408
 409         if (e == asio::error::eof)
 410         {
 411                 TORRENT_ASSERT(bytes_transferred == 0);
 412                 char const* data = 0;
 413                 std::size_t size = 0;
 414                 if (m_bottled && m_parser.header_finished())
 415                 {
 416                         data = m_parser.get_body().begin;
 417                         size = m_parser.get_body().left();
 418                 }
 419                 callback(e, data, size);
 420                 close();
 421                 return;
 422         }
 423
 424         if (e)
 425         {
 426                 TORRENT_ASSERT(bytes_transferred == 0);
 427                 callback(e);
 428                 close();
 429                 return;
 430         }
 431
 432         m_read_pos += bytes_transferred;
 433         TORRENT_ASSERT(m_read_pos <= int(m_recvbuffer.size()));
 434
 435         if (m_bottled || !m_parser.header_finished())
 436         {
 437                 libtorrent::buffer::const_interval rcv_buf(&m_recvbuffer[0]
 438                         , &m_recvbuffer[0] + m_read_pos);
 439                 bool error = false;
 440                 m_parser.incoming(rcv_buf, error);
 441                 if (error)
 442                 {
 443                         // HTTP parse error
 444                         error_code ec = asio::error::fault;
 445                         callback(ec, 0, 0);
 446                         return;
 447                 }
 448
 449                 // having a nonempty path means we should handle redirects
 450                 if (m_redirects && m_parser.header_finished())
 451                 {
 452                         int code = m_parser.status_code();
 453
 454                         if (code >= 300 && code < 400)
 455                         {
 456                                 // attempt a redirect
 457                                 std::string const& location = m_parser.header("location");
 458                                 if (location.empty())
 459                                 {
 460                                         // missing location header
 461                                         callback(asio::error::fault);
 462                                         close();
 463                                         return;
 464                                 }
 465
 466                                 error_code ec;
 467                                 m_sock.close(ec);
 468                                 using boost::tuples::ignore;
 469                                 char const* error;
 470                                 boost::tie(ignore, ignore, ignore, ignore, ignore, error)
 471                                         = parse_url_components(location);
 472                                 if (error == 0)
 473                                 {
 474                                         get(location, m_timeout, m_priority, &m_proxy, m_redirects - 1);
 475                                 }
 476                                 else
 477                                 {
 478                                         // some broken web servers send out relative paths
 479                                         // in the location header.
 480                                         std::string url = m_url;
 481                                         // remove the leaf filename
 482                                         std::size_t i = url.find_last_of('/');
 483                                         if (i == std::string::npos)
 484                                         {
 485                                                 url += '/';
 486                                         }
 487                                         else
 488                                         {
 489                                                 url.resize(i + 1);
 490                                         }
 491                                         url += location;
 492
 493                                         get(url, m_timeout, m_priority, &m_proxy, m_redirects - 1);
 494                                 }
 495                                 return;
 496                         }
 497
 498                         m_redirects = 0;
 499                 }
 500
 501                 if (!m_bottled && m_parser.header_finished())
 502                 {
 503                         if (m_read_pos > m_parser.body_start())
 504                                 callback(e, &m_recvbuffer[0] + m_parser.body_start()
 505                                         , m_read_pos - m_parser.body_start());
 506                         m_read_pos = 0;
 507                         m_last_receive = time_now();
 508                 }
 509                 else if (m_bottled && m_parser.finished())
 510                 {
 511                         error_code ec;
 512                         m_timer.cancel(ec);
 513                         callback(e, m_parser.get_body().begin, m_parser.get_body().left());
 514                 }
 515         }
 516         else
 517         {
 518                 TORRENT_ASSERT(!m_bottled);
 519                 callback(e, &m_recvbuffer[0], m_read_pos);
 520                 m_read_pos = 0;
 521                 m_last_receive = time_now();
 522         }
 523
 524         if (int(m_recvbuffer.size()) == m_read_pos)
 525                 m_recvbuffer.resize((std::min)(m_read_pos + 2048, int(max_bottled_buffer)));
 526         if (m_read_pos == max_bottled_buffer)
 527         {
 528                 callback(asio::error::eof);
 529                 close();
 530                 return;
 531         }
 532         int amount_to_read = m_recvbuffer.size() - m_read_pos;
 533         if (m_rate_limit > 0 && amount_to_read > m_download_quota)
 534         {
 535                 amount_to_read = m_download_quota;
 536                 if (m_download_quota == 0)
 537                 {
 538                         if (!m_limiter_timer_active)
 539                                 on_assign_bandwidth(error_code());
 540                         return;
 541                 }
 542         }
 543         m_sock.async_read_some(asio::buffer(&m_recvbuffer[0] + m_read_pos
 544                 , amount_to_read)
 545                 , bind(&http_connection::on_read
 546                 , shared_from_this(), _1, _2));
 547 }
 548
 549 void http_connection::on_assign_bandwidth(error_code const& e)
 550 {
 551         if ((e == asio::error::operation_aborted
 552                 && m_limiter_timer_active)
 553                 || !m_sock.is_open())
 554         {
 555                 callback(asio::error::eof);
 556                 return;
 557         }
 558         m_limiter_timer_active = false;
 559         if (e) return;
 560
 561         if (m_download_quota > 0) return;
 562
 563         m_download_quota = m_rate_limit / 4;
 564
 565         int amount_to_read = m_recvbuffer.size() - m_read_pos;
 566         if (amount_to_read > m_download_quota)
 567                 amount_to_read = m_download_quota;
 568
 569         if (!m_sock.is_open()) return;
 570
 571         m_sock.async_read_some(asio::buffer(&m_recvbuffer[0] + m_read_pos
 572                 , amount_to_read)
 573                 , bind(&http_connection::on_read
 574                 , shared_from_this(), _1, _2));
 575
 576         error_code ec;
 577         m_limiter_timer_active = true;
 578         m_limiter_timer.expires_from_now(milliseconds(250), ec);
 579         m_limiter_timer.async_wait(bind(&http_connection::on_assign_bandwidth
 580                 , shared_from_this(), _1));
 581 }
 582
 583 void http_connection::rate_limit(int limit)
 584 {
 585         if (!m_sock.is_open()) return;
 586
 587         if (!m_limiter_timer_active)
 588         {
 589                 error_code ec;
 590                 m_limiter_timer_active = true;
 591                 m_limiter_timer.expires_from_now(milliseconds(250), ec);
 592                 m_limiter_timer.async_wait(bind(&http_connection::on_assign_bandwidth
 593                         , shared_from_this(), _1));
 594         }
 595         m_rate_limit = limit;
 596 }
 597
 598 }
 599