source/http.cpp

   1 #include <iostream>
   2 #include <istream>
   3 #include <ostream>
   4
   5 #include <ail/http.hpp>
   6 #include <ail/string.hpp>
   7 #include <ail/zlib.hpp>
   8 #include <ail/file.hpp>
   9 #include <ail/net.hpp>
  10
  11 namespace ail
  12 {
  13         bool process_chunked_data(std::string const & data, std::string & content, std::size_t offset)
  14         {
  15                 std::string const target = "\r\n";
  16                 while(offset < data.size())
  17                 {
  18                         std::size_t newline_offset = data.find(target, offset);
  19                         if(newline_offset == std::string::npos)
  20                                 return false;
  21                         std::string number_string = data.substr(offset, newline_offset - offset);
  22                         long chunk_size;
  23                         if(!ail::string_to_number<long>(number_string, chunk_size, std::ios_base::hex))
  24                                 return false;
  25                         std::size_t chunk_offset = newline_offset + target.length();
  26                         std::size_t end_of_chunk = chunk_offset + chunk_size;
  27                         std::string chunk = data.substr(chunk_offset, end_of_chunk - chunk_offset);
  28                         content += chunk;
  29                         offset = end_of_chunk + target.length();
  30                 }
  31                 return true;
  32         }
  33
  34         http_client::http_client(boost::asio::io_service & io_service):
  35                 io_service(io_service),
  36                 resolver(io_service),
  37                 socket(io_service),
  38                 use_compression(true),
  39                 has_error_handler(false),
  40                 has_download_handler(false),
  41                 has_download_finished_handler(false),
  42                 user_agent(http_user_agent_firefox),
  43                 method("GET")
  44         {
  45         }
  46
  47         void http_client::use_post()
  48         {
  49                 method = "POST";
  50         }
  51
  52         void http_client::set_compression(bool new_use_compression)
  53         {
  54                 use_compression = new_use_compression;
  55         }
  56
  57         void http_client::add_post_data(std::string const & field, std::string const & value)
  58         {
  59                 form_data[field] = value;
  60         }
  61
  62         void http_client::start_download(std::string const & new_url)
  63         {
  64                 std::string const
  65                         http_string = "http",
  66                         https_string = "https",
  67                         protocol_separator = "://";
  68
  69                 std::size_t host_offset;
  70
  71                 url = new_url;
  72
  73                 std::size_t separator_offset = url.find(protocol_separator);
  74                 if(separator_offset == std::string::npos)
  75                 {
  76                         use_ssl = false;
  77                         host_offset = 0;
  78                 }
  79                 else
  80                 {
  81                         host_offset = separator_offset + protocol_separator.length();
  82                         std::string protocol = url.substr(0, separator_offset);
  83                         if(protocol == http_string)
  84                                 use_ssl = false;
  85                         else if(protocol == https_string)
  86                                 use_ssl = true;
  87                         else
  88                                 throw ail::exception("Invalid protocol specified for HTTP client");
  89                 }
  90
  91                 std::size_t path_offset = url.find('/', host_offset);
  92                 if(path_offset == std::string::npos)
  93                 {
  94                         host = url.substr(host_offset);
  95                         path = "/";
  96                 }
  97                 else
  98                 {
  99                         host = url.substr(host_offset, path_offset - host_offset);
 100                         path = url.substr(path_offset);
 101                 }
 102
 103                 boost::asio::ip::tcp::resolver::query query(host, "http");
 104                 resolver.async_resolve(query, boost::bind(&http_client::dns_event, this, boost::asio::placeholders::error, boost::asio::placeholders::iterator));
 105                 io_service.stop();
 106         }
 107
 108         bool http_client::start_download(std::string const & new_url, std::string const & file_name)
 109         {
 110                 if(!file_output.open_create(file_name))
 111                         return false;
 112                 start_download(new_url);
 113                 return true;
 114         }
 115
 116         void http_client::dns_event(boost::system::error_code const & error, boost::asio::ip::tcp::resolver::iterator endpoint_iterator)
 117         {
 118                 //std::cout << "DNS event" << std::endl;
 119                 if(!error)
 120                 {
 121                         boost::asio::ip::tcp::endpoint endpoint = *endpoint_iterator;
 122                         endpoint_iterator++;
 123                         socket.async_connect(endpoint, boost::bind(&http_client::connect_event, this, boost::asio::placeholders::error, endpoint_iterator));
 124                         io_service.stop();
 125                 }
 126                 else
 127                         error_occured(http_error_dns);
 128         }
 129
 130         void http_client::connect_event(boost::system::error_code const & error, boost::asio::ip::tcp::resolver::iterator endpoint_iterator)
 131         {
 132                 if(!error)
 133                 {
 134                         std::ostream request_stream(&request);
 135                         request_stream << method << " " << path << " HTTP/1.1\r\n";
 136                         switch(user_agent)
 137                         {
 138                                 case http_user_agent_firefox:
 139                                         request_stream << "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8 (.NET CLR 3.5.30729)\r\n";
 140                                         break;
 141                         }
 142                         request_stream << "Host: " << host << "\r\n";
 143                         //request_stream << "Accept: */*\r\n";
 144                         request_stream << "Accept: text/html, application/xml;q=0.9, application/xhtml+xml, image/png, image/jpeg, image/gif, image/x-xbitmap, */*;q=0.1\r\n";
 145                         request_stream << "Accept-Language: en-us,en;q=0.5\r\n";
 146                         request_stream << "Accept-Charset: iso-8859-1, utf-8, utf-16, *;q=0.1\r\n";
 147                         request_stream << "Accept-Encoding: deflate, gzip, x-gzip, identity, *;q=0\r\n";
 148                         //request_stream << "Connection: close\r\n";
 149                         //request_stream << "Connection: Keep-Alive, TE\r\n";
 150                         request_stream << "Connection: close, TE\r\n";
 151                         request_stream << "TE: ";
 152                         if(use_compression)
 153                                 request_stream << "deflate, gzip, ";
 154                         request_stream << "chunked, identity, trailers\r\n";
 155                         if(!referrer.empty())
 156                                 request_stream << "Referer: " << referrer << "\r\n";
 157                         if(!cookies.empty())
 158                         {
 159                                 request_stream << "Cookie: ";
 160                                 bool first = true;
 161                                 for(cookie_map::iterator i = cookies.begin(), end = cookies.end(); i != end; i++)
 162                                 {
 163                                         if(first)
 164                                                 first = false;
 165                                         else
 166                                                 request_stream << "; ";
 167                                         request_stream << i->first << "=" << i->second;
 168                                 }
 169                                 request_stream << "\r\n";
 170                         }
 171
 172                         std::string post_data;
 173
 174                         if(!form_data.empty())
 175                         {
 176                                 std::string const boundary_tag = "------------PDq3XENV0Hx17uo1Qpe5qN";
 177                                 for(form_map::iterator i = form_data.begin(), end = form_data.end(); i != end; i++)
 178                                         post_data += boundary_tag + "\r\nContent-Disposition: form-data; name=\"" + i->first + "\"\r\n\r\n" + i->second + "\r\n";
 179                                 post_data += boundary_tag + "--\r\n";
 180
 181                                 //std::cout << ail::consolify(post_data) << std::endl;
 182
 183                                 request_stream << "Content-Length: " << post_data.size() << "\r\n";
 184                                 request_stream << "Content-Type: multipart/form-data; boundary=" << boundary_tag << "\r\n";
 185                                 request_stream << "\r\n";
 186                                 request_stream << post_data;
 187                         }
 188                         else
 189                                 request_stream << "\r\n";
 190
 191                         boost::asio::async_write(socket, request, boost::bind(&http_client::write_event, this, boost::asio::placeholders::error));
 192                         io_service.stop();
 193                 }
 194                 else if(endpoint_iterator != boost::asio::ip::tcp::resolver::iterator())
 195                 {
 196                         socket.close();
 197                         boost::asio::ip::tcp::endpoint endpoint = *endpoint_iterator;
 198                         endpoint_iterator++;
 199                         socket.async_connect(endpoint, boost::bind(&http_client::connect_event, this, boost::asio::placeholders::error, endpoint_iterator));
 200                         io_service.stop();
 201                 }
 202                 else
 203                 {
 204                         std::cout << "Connect: " << error.message() << std::endl;
 205                         error_occured(http_error_connect);
 206                 }
 207         }
 208
 209         void http_client::write_event(boost::system::error_code const & error)
 210         {
 211                 if(!error)
 212                         read_header_data();
 213                 else
 214                         error_occured(http_error_write);
 215         }
 216
 217         void http_client::read_header_data()
 218         {
 219                 boost::asio::async_read(socket, response, boost::asio::transfer_at_least(1), boost::bind(&http_client::read_header_event, this, boost::asio::placeholders::error));
 220                 io_service.stop();
 221         }
 222
 223         void http_client::read_partial_data()
 224         {
 225                 boost::asio::async_read(socket, response, boost::asio::transfer_at_least(1), boost::bind(&http_client::partial_read_event, this, boost::asio::placeholders::error));
 226                 io_service.stop();
 227         }
 228
 229         void http_client::read_full_data()
 230         {
 231                 boost::asio::async_read(socket, response, boost::asio::transfer_all(), boost::bind(&http_client::full_read_event, this, boost::asio::placeholders::error));
 232                 io_service.stop();
 233         }
 234
 235         void http_client::read_header_event(boost::system::error_code const & error)
 236         {
 237                 long const
 238                         ok_code = 200,
 239                         moved_code = 301,
 240                         found_code = 302;
 241
 242                 bool download_finished = error == boost::asio::error::eof;
 243                 if(error && !download_finished)
 244                 {
 245                         error_occured(http_error_read);
 246                         return;
 247                 }
 248
 249                 extract_data(response, buffer);
 250
 251                 std::string const end_of_header_string = "\r\n\r\n";
 252
 253                 std::size_t end_of_header = buffer.find(end_of_header_string);
 254                 if(end_of_header == std::string::npos)
 255                 {
 256                         if(download_finished)
 257                                 error_occured(http_error_end_of_header);
 258                         else
 259                                 read_header_data();
 260                         return;
 261                 }
 262                 std::string header = buffer.substr(0, end_of_header);
 263                 buffer.erase(0, end_of_header + end_of_header_string.size());
 264
 265                 string_vector tokens = tokenise(header, "\r\n");
 266                 if(tokens.empty())
 267                 {
 268                         error_occured(http_error_header_lacks_lines);
 269                         return;
 270                 }
 271
 272                 //std::cout << "Header: " << header << std::endl;
 273
 274                 std::string const & status_line = tokens[0];
 275
 276                 //std::cout << status_line << std::endl;
 277
 278                 std::string http_code_string;
 279                 if(!extract_string(status_line, " ", " ", http_code_string))
 280                 {
 281                         error_occured(http_error_invalid_status_line);
 282                         return;
 283                 }
 284
 285                 //std::cout << "Code: " << http_code_string << std::endl;
 286
 287                 long http_code;
 288                 if(!string_to_number<long>(http_code_string, http_code))
 289                 {
 290                         error_occured(http_error_invalid_code_string);
 291                         return;
 292                 }
 293
 294                 if(http_code != ok_code && http_code != moved_code && http_code != found_code)
 295                 {
 296                         error_occured(http_error_status_code_error);
 297                         std::cout << buffer << std::endl;
 298                         return;
 299                 }
 300
 301                 encoding = http_encoding_none;
 302
 303                 chunked = false;
 304
 305                 for(std::size_t i = 1, end = tokens.size(); i < end; i++)
 306                 {
 307                         string_vector line_tokens = tokenise(tokens[i], ": ");
 308                         if(line_tokens.size() != 2)
 309                         {
 310                                 error_occured(http_error_header_token_error);
 311                                 return;
 312                         }
 313                         std::string const & field = line_tokens[0];
 314                         std::string const & value = line_tokens[1];
 315                         if(http_code == ok_code && field == "Content-Encoding")
 316                         {
 317                                 if(value == "gzip")
 318                                         encoding = http_encoding_gzip;
 319                                 else if(value == "deflate")
 320                                         encoding = http_encoding_deflate;
 321                         }
 322                         else if(http_code == ok_code && field == "Transfer-Encoding")
 323                         {
 324                                 if(value == "chunked")
 325                                         chunked = true;
 326                         }
 327                         else if((http_code == moved_code || http_code == found_code) && field == "Location")
 328                         {
 329                                 socket.close();
 330
 331                                 std::string new_url;
 332
 333                                 if(!value.empty() && value[0] == '/')
 334                                         new_url = (use_ssl ? "http" : "https") + std::string("://") + host + value;
 335                                 else
 336                                         new_url = value;
 337                                 start_download(new_url);
 338                                 return;
 339                         }
 340                 }
 341
 342                 if(download_finished)
 343                         process_data();
 344                 else
 345                 {
 346                         if(encoding == http_encoding_none && has_download_handler)
 347                         {
 348                                 process_partial_data();
 349                                 read_partial_data();
 350                         }
 351                         else
 352                                 read_full_data();
 353                 }
 354         }
 355
 356         void http_client::full_read_event(boost::system::error_code const & error)
 357         {
 358                 if(error != boost::asio::error::eof)
 359                 {
 360                         error_occured(http_error_read);
 361                         return;
 362                 }
 363
 364                 extract_data(response, buffer);
 365                 process_data();
 366         }
 367
 368         void http_client::process_partial_data()
 369         {
 370                 if(file_output.open())
 371                         file_output.write(buffer);
 372
 373                 if(has_download_handler)
 374                         download_handler(*this, buffer);
 375
 376                 buffer.clear();
 377         }
 378
 379         void http_client::partial_read_event(boost::system::error_code const & error)
 380         {
 381                 bool download_finished = error == boost::asio::error::eof;
 382                 if(error && !download_finished)
 383                 {
 384                         error_occured(http_error_read);
 385                         return;
 386                 }
 387
 388                 extract_data(response, buffer);
 389
 390                 process_partial_data();
 391
 392                 if(download_finished)
 393                 {
 394                         if(file_output.open())
 395                                 file_output.close();
 396
 397                         if(has_download_finished_handler)
 398                                 download_finished_handler(*this, buffer);
 399                 }
 400                 else
 401                         read_partial_data();
 402         }
 403
 404         void http_client::process_data()
 405         {
 406                 std::string
 407                         content,
 408                         dechunked_data,
 409
 410                         * input_pointer = &buffer;
 411
 412                 if(chunked)
 413                 {
 414                         if(!process_chunked_data(buffer, dechunked_data, 0))
 415                         {
 416                                 error_occured(http_error_chunk);
 417                                 return;
 418                         }
 419                         input_pointer = &dechunked_data;
 420                 }
 421
 422                 std::string & input = *input_pointer;
 423
 424                 switch(encoding)
 425                 {
 426                         case http_encoding_none:
 427                                 content = input;
 428                                 break;
 429
 430                         case http_encoding_gzip:
 431                         case http_encoding_deflate:
 432                                 try
 433                                 {
 434                                         decompress_gzip(input, content);
 435                                 }
 436                                 catch(exception &)
 437                                 {
 438                                         error_occured(http_error_compression);
 439                                         return;
 440                                 }
 441                                 break;
 442                 }
 443
 444                 if(has_download_handler)
 445                         download_handler(*this, content);
 446
 447                 if(has_download_finished_handler)
 448                         download_finished_handler(*this, content);
 449
 450                 buffer.clear();
 451         }
 452
 453         void http_client::error_occured(http_error_type the_error_code)
 454         {
 455                 //std::cout << "http_client Error: " << the_error_code << std::endl;
 456                 if(has_error_handler)
 457                         error_handler(*this, the_error_code);
 458         }
 459
 460         void http_client::set_error_handler(error_handler_type new_error_handler)
 461         {
 462                 error_handler = new_error_handler;
 463                 has_error_handler = true;
 464         }
 465
 466         void http_client::set_download_handler(download_handler_type new_download_handler)
 467         {
 468                 download_handler = new_download_handler;
 469                 has_download_handler = true;
 470         }
 471
 472         void http_client::set_download_finished_handler(download_finished_handler_type new_download_finished_handler)
 473         {
 474                 download_finished_handler = new_download_finished_handler;
 475                 has_download_finished_handler = true;
 476         }
 477
 478         void http_client::set_referrer(std::string const & new_referrer)
 479         {
 480                 referrer = new_referrer;
 481         }
 482
 483         std::string http_client::get_referrer() const
 484         {
 485                 return referrer;
 486         }
 487
 488         std::string http_client::get_url() const
 489         {
 490                 return url;
 491         }
 492 }