source/http.cpp

   1 #include <iostream>
   2 #include <istream>
   3 #include <ostream>
   4
   5 #include <ail/http.hpp>
   6 #include <ail/string.hpp>
   7 #include <ail/zlib.hpp>
   8 #include <ail/file.hpp>
   9 #include <ail/net.hpp>
  10
  11 namespace ail
  12 {
  13         bool process_chunked_data(std::string const & data, std::string & content, std::size_t offset)
  14         {
  15                 std::string const target = "\r\n";
  16                 while(offset < data.size())
  17                 {
  18                         std::size_t newline_offset = data.find(target, offset);
  19                         if(newline_offset == std::string::npos)
  20                                 return false;
  21                         std::string number_string = data.substr(offset, newline_offset - offset);
  22                         long chunk_size;
  23                         if(!ail::string_to_number<long>(number_string, chunk_size, std::ios_base::hex))
  24                                 return false;
  25                         std::size_t chunk_offset = newline_offset + target.length();
  26                         std::size_t end_of_chunk = chunk_offset + chunk_size;
  27                         std::string chunk = data.substr(chunk_offset, end_of_chunk - chunk_offset);
  28                         content += chunk;
  29                         offset = end_of_chunk + target.length();
  30                 }
  31                 return true;
  32         }
  33
  34         http_client::http_client(boost::asio::io_service & io_service):
  35                 io_service(io_service),
  36                 resolver(io_service),
  37                 socket(io_service),
  38                 has_error_handler(false),
  39                 has_download_handler(false),
  40                 has_download_finished_handler(false),
  41                 user_agent(http_user_agent_firefox),
  42                 method("GET")
  43         {
  44         }
  45
  46         void http_client::use_post()
  47         {
  48                 method = "POST";
  49         }
  50
  51         void http_client::add_post_data(std::string const & field, std::string const & value)
  52         {
  53                 form_data[field] = value;
  54         }
  55
  56         void http_client::start_download(std::string const & new_url)
  57         {
  58                 std::string const
  59                         http_string = "http",
  60                         https_string = "https",
  61                         protocol_separator = "://";
  62
  63                 std::size_t host_offset;
  64
  65                 url = new_url;
  66
  67                 std::size_t separator_offset = url.find(protocol_separator);
  68                 if(separator_offset == std::string::npos)
  69                 {
  70                         use_ssl = false;
  71                         host_offset = 0;
  72                 }
  73                 else
  74                 {
  75                         host_offset = separator_offset + protocol_separator.length();
  76                         std::string protocol = url.substr(0, separator_offset);
  77                         if(protocol == http_string)
  78                                 use_ssl = false;
  79                         else if(protocol == https_string)
  80                                 use_ssl = true;
  81                         else
  82                                 throw ail::exception("Invalid protocol specified for HTTP client");
  83                 }
  84
  85                 std::size_t path_offset = url.find('/', host_offset);
  86                 if(path_offset == std::string::npos)
  87                 {
  88                         host = url.substr(host_offset);
  89                         path = "/";
  90                 }
  91                 else
  92                 {
  93                         host = url.substr(host_offset, path_offset - host_offset);
  94                         path = url.substr(path_offset);
  95                 }
  96
  97                 boost::asio::ip::tcp::resolver::query query(host, "http");
  98                 resolver.async_resolve(query, boost::bind(&http_client::dns_event, this, boost::asio::placeholders::error, boost::asio::placeholders::iterator));
  99                 io_service.stop();
 100         }
 101
 102         bool http_client::start_download(std::string const & new_url, std::string const & file_name)
 103         {
 104                 if(!file_output.open_create(file_name))
 105                         return false;
 106                 start_download(new_url);
 107                 return true;
 108         }
 109
 110         void http_client::dns_event(boost::system::error_code const & error, boost::asio::ip::tcp::resolver::iterator endpoint_iterator)
 111         {
 112                 //std::cout << "DNS event" << std::endl;
 113                 if(!error)
 114                 {
 115                         boost::asio::ip::tcp::endpoint endpoint = *endpoint_iterator;
 116                         endpoint_iterator++;
 117                         socket.async_connect(endpoint, boost::bind(&http_client::connect_event, this, boost::asio::placeholders::error, endpoint_iterator));
 118                         io_service.stop();
 119                 }
 120                 else
 121                         error_occured(http_error_dns);
 122         }
 123
 124         void http_client::connect_event(boost::system::error_code const & error, boost::asio::ip::tcp::resolver::iterator endpoint_iterator)
 125         {
 126                 if(!error)
 127                 {
 128                         std::ostream request_stream(&request);
 129                         request_stream << method << " " << path << " HTTP/1.1\r\n";
 130                         switch(user_agent)
 131                         {
 132                                 case http_user_agent_firefox:
 133                                         request_stream << "User-Agent: Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8 (.NET CLR 3.5.30729)\r\n";
 134                                         break;
 135                         }
 136                         request_stream << "Host: " << host << "\r\n";
 137                         //request_stream << "Accept: */*\r\n";
 138                         request_stream << "Accept: text/html, application/xml;q=0.9, application/xhtml+xml, image/png, image/jpeg, image/gif, image/x-xbitmap, */*;q=0.1\r\n";
 139                         request_stream << "Accept-Language: en-us,en;q=0.5\r\n";
 140                         request_stream << "Accept-Charset: iso-8859-1, utf-8, utf-16, *;q=0.1\r\n";
 141                         request_stream << "Accept-Encoding: deflate, gzip, x-gzip, identity, *;q=0\r\n";
 142                         //request_stream << "Connection: close\r\n";
 143                         //request_stream << "Connection: Keep-Alive, TE\r\n";
 144                         request_stream << "Connection: close, TE\r\n";
 145                         request_stream << "TE: deflate, gzip, chunked, identity, trailers\r\n";
 146                         if(!referrer.empty())
 147                                 request_stream << "Referer: " << referrer << "\r\n";
 148                         if(!cookies.empty())
 149                         {
 150                                 request_stream << "Cookie: ";
 151                                 bool first = true;
 152                                 for(cookie_map::iterator i = cookies.begin(), end = cookies.end(); i != end; i++)
 153                                 {
 154                                         if(first)
 155                                                 first = false;
 156                                         else
 157                                                 request_stream << "; ";
 158                                         request_stream << i->first << "=" << i->second;
 159                                 }
 160                                 request_stream << "\r\n";
 161                         }
 162
 163                         std::string post_data;
 164
 165                         if(!form_data.empty())
 166                         {
 167                                 std::string const boundary_tag = "------------PDq3XENV0Hx17uo1Qpe5qN";
 168                                 for(form_map::iterator i = form_data.begin(), end = form_data.end(); i != end; i++)
 169                                         post_data += boundary_tag + "\r\nContent-Disposition: form-data; name=\"" + i->first + "\"\r\n\r\n" + i->second + "\r\n";
 170                                 post_data += boundary_tag + "--\r\n";
 171
 172                                 std::cout << ail::consolify(post_data) << std::endl;
 173
 174                                 request_stream << "Content-Length: " << post_data.size() << "\r\n";
 175                                 request_stream << "Content-Type: multipart/form-data; boundary=" << boundary_tag << "\r\n";
 176                                 request_stream << "\r\n";
 177                                 request_stream << post_data;
 178                         }
 179                         else
 180                                 request_stream << "\r\n";
 181
 182                         boost::asio::async_write(socket, request, boost::bind(&http_client::write_event, this, boost::asio::placeholders::error));
 183                         io_service.stop();
 184                 }
 185                 else if(endpoint_iterator != boost::asio::ip::tcp::resolver::iterator())
 186                 {
 187                         socket.close();
 188                         boost::asio::ip::tcp::endpoint endpoint = *endpoint_iterator;
 189                         endpoint_iterator++;
 190                         socket.async_connect(endpoint, boost::bind(&http_client::connect_event, this, boost::asio::placeholders::error, endpoint_iterator));
 191                         io_service.stop();
 192                 }
 193                 else
 194                 {
 195                         std::cout << "Connect: " << error.message() << std::endl;
 196                         error_occured(http_error_connect);
 197                 }
 198         }
 199
 200         void http_client::write_event(boost::system::error_code const & error)
 201         {
 202                 if(!error)
 203                         read_header_data();
 204                 else
 205                         error_occured(http_error_write);
 206         }
 207
 208         void http_client::read_header_data()
 209         {
 210                 boost::asio::async_read(socket, response, boost::asio::transfer_at_least(1), boost::bind(&http_client::read_header_event, this, boost::asio::placeholders::error));
 211                 io_service.stop();
 212         }
 213
 214         void http_client::read_partial_data()
 215         {
 216                 boost::asio::async_read(socket, response, boost::asio::transfer_at_least(1), boost::bind(&http_client::partial_read_event, this, boost::asio::placeholders::error));
 217                 io_service.stop();
 218         }
 219
 220         void http_client::read_full_data()
 221         {
 222                 boost::asio::async_read(socket, response, boost::asio::transfer_all(), boost::bind(&http_client::full_read_event, this, boost::asio::placeholders::error));
 223                 io_service.stop();
 224         }
 225
 226         void http_client::read_header_event(boost::system::error_code const & error)
 227         {
 228                 long const
 229                         ok_code = 200,
 230                         moved_code = 301,
 231                         found_code = 302;
 232
 233                 bool download_finished = error == boost::asio::error::eof;
 234                 if(error && !download_finished)
 235                 {
 236                         error_occured(http_error_read);
 237                         return;
 238                 }
 239
 240                 extract_data(response, buffer);
 241
 242                 std::string const end_of_header_string = "\r\n\r\n";
 243
 244                 std::size_t end_of_header = buffer.find(end_of_header_string);
 245                 if(end_of_header == std::string::npos)
 246                 {
 247                         if(download_finished)
 248                                 error_occured(http_error_end_of_header);
 249                         else
 250                                 read_header_data();
 251                         return;
 252                 }
 253                 std::string header = buffer.substr(0, end_of_header);
 254                 buffer.erase(0, end_of_header + end_of_header_string.size());
 255
 256                 string_vector tokens = tokenise(header, "\r\n");
 257                 if(tokens.empty())
 258                 {
 259                         error_occured(http_error_header_lacks_lines);
 260                         return;
 261                 }
 262
 263                 //std::cout << "Header: " << header << std::endl;
 264
 265                 std::string const & status_line = tokens[0];
 266
 267                 //std::cout << status_line << std::endl;
 268
 269                 std::string http_code_string;
 270                 if(!extract_string(status_line, " ", " ", http_code_string))
 271                 {
 272                         error_occured(http_error_invalid_status_line);
 273                         return;
 274                 }
 275
 276                 //std::cout << "Code: " << http_code_string << std::endl;
 277
 278                 long http_code;
 279                 if(!string_to_number<long>(http_code_string, http_code))
 280                 {
 281                         error_occured(http_error_invalid_code_string);
 282                         return;
 283                 }
 284
 285                 if(http_code != ok_code && http_code != moved_code && http_code != found_code)
 286                 {
 287                         error_occured(http_error_status_code_error);
 288                         std::cout << buffer << std::endl;
 289                         return;
 290                 }
 291
 292                 encoding = http_encoding_none;
 293
 294                 chunked = false;
 295
 296                 for(std::size_t i = 1, end = tokens.size(); i < end; i++)
 297                 {
 298                         string_vector line_tokens = tokenise(tokens[i], ": ");
 299                         if(line_tokens.size() != 2)
 300                         {
 301                                 error_occured(http_error_header_token_error);
 302                                 return;
 303                         }
 304                         std::string const & field = line_tokens[0];
 305                         std::string const & value = line_tokens[1];
 306                         if(http_code == ok_code && field == "Content-Encoding")
 307                         {
 308                                 if(value == "gzip")
 309                                         encoding = http_encoding_gzip;
 310                                 else if(value == "deflate")
 311                                         encoding = http_encoding_deflate;
 312                         }
 313                         else if(http_code == ok_code && field == "Transfer-Encoding")
 314                         {
 315                                 if(value == "chunked")
 316                                         chunked = true;
 317                         }
 318                         else if((http_code == moved_code || http_code == found_code) && field == "Location")
 319                         {
 320                                 socket.close();
 321
 322                                 std::string new_url;
 323
 324                                 if(!value.empty() && value[0] == '/')
 325                                         new_url = (use_ssl ? "http" : "https") + std::string("://") + host + value;
 326                                 else
 327                                         new_url = value;
 328                                 start_download(new_url);
 329                                 return;
 330                         }
 331                 }
 332
 333                 if(download_finished)
 334                         process_data();
 335                 else
 336                 {
 337                         if(encoding == http_encoding_none && has_download_handler)
 338                         {
 339                                 process_partial_data();
 340                                 read_partial_data();
 341                         }
 342                         else
 343                                 read_full_data();
 344                 }
 345         }
 346
 347         void http_client::full_read_event(boost::system::error_code const & error)
 348         {
 349                 if(error != boost::asio::error::eof)
 350                 {
 351                         error_occured(http_error_read);
 352                         return;
 353                 }
 354
 355                 extract_data(response, buffer);
 356                 process_data();
 357         }
 358
 359         void http_client::process_partial_data()
 360         {
 361                 if(file_output.open())
 362                         file_output.write(buffer);
 363
 364                 if(has_download_handler)
 365                         download_handler(*this, buffer);
 366
 367                 buffer.clear();
 368         }
 369
 370         void http_client::partial_read_event(boost::system::error_code const & error)
 371         {
 372                 bool download_finished = error == boost::asio::error::eof;
 373                 if(error && !download_finished)
 374                 {
 375                         error_occured(http_error_read);
 376                         return;
 377                 }
 378
 379                 extract_data(response, buffer);
 380
 381                 process_partial_data();
 382
 383                 if(download_finished)
 384                 {
 385                         if(file_output.open())
 386                                 file_output.close();
 387
 388                         if(has_download_finished_handler)
 389                                 download_finished_handler(*this, buffer);
 390                 }
 391                 else
 392                         read_partial_data();
 393         }
 394
 395         void http_client::process_data()
 396         {
 397                 std::string
 398                         content,
 399                         dechunked_data,
 400
 401                         * input_pointer = &buffer;
 402
 403                 if(chunked)
 404                 {
 405                         if(!process_chunked_data(buffer, dechunked_data, 0))
 406                         {
 407                                 error_occured(http_error_chunk);
 408                                 return;
 409                         }
 410                         input_pointer = &dechunked_data;
 411                 }
 412
 413                 std::string & input = *input_pointer;
 414
 415                 switch(encoding)
 416                 {
 417                         case http_encoding_none:
 418                                 content = input;
 419                                 break;
 420
 421                         case http_encoding_gzip:
 422                         case http_encoding_deflate:
 423                                 try
 424                                 {
 425                                         decompress_gzip(input, content);
 426                                 }
 427                                 catch(exception &)
 428                                 {
 429                                         error_occured(http_error_compression);
 430                                         return;
 431                                 }
 432                                 break;
 433                 }
 434
 435                 if(has_download_handler)
 436                         download_handler(*this, content);
 437
 438                 if(has_download_finished_handler)
 439                         download_finished_handler(*this, content);
 440
 441                 buffer.clear();
 442         }
 443
 444         void http_client::error_occured(http_error_type the_error_code)
 445         {
 446                 //std::cout << "http_client Error: " << the_error_code << std::endl;
 447                 if(has_error_handler)
 448                         error_handler(*this, the_error_code);
 449         }
 450
 451         void http_client::set_error_handler(error_handler_type new_error_handler)
 452         {
 453                 error_handler = new_error_handler;
 454                 has_error_handler = true;
 455         }
 456
 457         void http_client::set_download_handler(download_handler_type new_download_handler)
 458         {
 459                 download_handler = new_download_handler;
 460                 has_download_handler = true;
 461         }
 462
 463         void http_client::set_download_finished_handler(download_finished_handler_type new_download_finished_handler)
 464         {
 465                 download_finished_handler = new_download_finished_handler;
 466                 has_download_finished_handler = true;
 467         }
 468
 469         void http_client::set_referrer(std::string const & new_referrer)
 470         {
 471                 referrer = new_referrer;
 472         }
 473
 474         std::string http_client::get_referrer() const
 475         {
 476                 return referrer;
 477         }
 478
 479         std::string http_client::get_url() const
 480         {
 481                 return url;
 482         }
 483 }