rbutil/rbutilqt/base/httpget.cpp

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  *
   9  *   Copyright (C) 2007 by Dominik Riebeling
  10  *   $Id$
  11  *
  12  * All files in this archive are subject to the GNU General Public License.
  13  * See the file COPYING in the source tree root for full license agreement.
  14  *
  15  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  16  * KIND, either express or implied.
  17  *
  18  ****************************************************************************/
  19
  20 #include <QtCore>
  21 #include <QtNetwork>
  22 #include <QtDebug>
  23
  24 #include "httpget.h"
  25
  26 QDir HttpGet::m_globalCache; //< global cach path value for new objects
  27 QUrl HttpGet::m_globalProxy; //< global proxy value for new objects
  28 bool HttpGet::m_globalDumbCache = false; //< globally set cache "dumb" mode
  29 QString HttpGet::m_globalUserAgent; //< globally set user agent for requests
  30
  31 HttpGet::HttpGet(QObject *parent)
  32     : QObject(parent)
  33 {
  34     outputToBuffer = true;
  35     m_cached = false;
  36     m_dumbCache = m_globalDumbCache;
  37     getRequest = -1;
  38     headRequest = -1;
  39     // if a request is cancelled before a reponse is available return some
  40     // hint about this in the http response instead of nonsense.
  41     m_response = -1;
  42     m_useproxy = false;
  43
  44     // default to global proxy / cache if not empty.
  45     // proxy is automatically enabled, disable it by setting an empty proxy
  46     // cache is enabled to be in line, can get disabled with setCache(bool)
  47     if(!m_globalProxy.isEmpty())
  48         setProxy(m_globalProxy);
  49     m_usecache = false;
  50     m_cachedir = m_globalCache;
  51
  52     m_serverTimestamp = QDateTime();
  53
  54     connect(&http, SIGNAL(done(bool)), this, SLOT(httpDone(bool)));
  55     connect(&http, SIGNAL(dataReadProgress(int, int)),
  56             this, SIGNAL(dataReadProgress(int, int)));
  57     connect(&http, SIGNAL(requestFinished(int, bool)),
  58             this, SLOT(httpFinished(int, bool)));
  59     connect(&http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader&)),
  60             this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
  61 //    connect(&http, SIGNAL(stateChanged(int)), this, SLOT(httpState(int)));
  62     connect(&http, SIGNAL(requestStarted(int)), this, SLOT(httpStarted(int)));
  63
  64     connect(&http, SIGNAL(readyRead(const QHttpResponseHeader&)),
  65             this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
  66     connect(this, SIGNAL(headerFinished()), this, SLOT(getFileFinish()));
  67
  68 }
  69
  70
  71 //! @brief set cache path
  72 //  @param d new directory to use as cache path
  73 void HttpGet::setCache(const QDir& d)
  74 {
  75     m_cachedir = d;
  76     bool result;
  77     result = initializeCache(d);
  78     m_usecache = result;
  79 }
  80
  81
  82 /** @brief enable / disable cache useage
  83  *  @param c set cache usage
  84  */
  85 void HttpGet::setCache(bool c)
  86 {
  87     m_usecache = c;
  88     // make sure cache is initialized
  89     if(c)
  90         m_usecache = initializeCache(m_cachedir);
  91 }
  92
  93
  94 bool HttpGet::initializeCache(const QDir& d)
  95 {
  96     bool result;
  97     QString p = d.absolutePath() + "/rbutil-cache";
  98     if(QFileInfo(d.absolutePath()).isDir())
  99     {
 100         if(!QFileInfo(p).isDir())
 101             result = d.mkdir("rbutil-cache");
 102         else
 103             result = true;
 104     }
 105     else
 106         result = false;
 107
 108     return result;
 109
 110 }
 111
 112
 113 /** @brief read all downloaded data into a buffer
 114  *  @return data
 115  */
 116 QByteArray HttpGet::readAll()
 117 {
 118     return dataBuffer;
 119 }
 120
 121
 122 /** @brief get http error
 123  *  @return http error
 124  */
 125 QHttp::Error HttpGet::error()
 126 {
 127     return http.error();
 128 }
 129
 130
 131 void HttpGet::setProxy(const QUrl &proxy)
 132 {
 133     m_proxy = proxy;
 134     m_useproxy = true;
 135     http.setProxy(m_proxy.host(), m_proxy.port(),
 136                   m_proxy.userName(), m_proxy.password());
 137 }
 138
 139
 140 void HttpGet::setProxy(bool enable)
 141 {
 142     if(enable) {
 143         m_useproxy = true;
 144         http.setProxy(m_proxy.host(), m_proxy.port(),
 145                       m_proxy.userName(), m_proxy.password());
 146     }
 147     else {
 148         m_useproxy = false;
 149         http.setProxy("", 0);
 150     }
 151 }
 152
 153
 154 void HttpGet::setFile(QFile *file)
 155 {
 156     outputFile = file;
 157     outputToBuffer = false;
 158 }
 159
 160
 161 void HttpGet::abort()
 162 {
 163     qDebug() << "[HTTP] Aborting requests, pending:" << http.hasPendingRequests();
 164     http.abort();
 165     if(!outputToBuffer)
 166         outputFile->close();
 167 }
 168
 169
 170 bool HttpGet::getFile(const QUrl &url)
 171 {
 172     if (!url.isValid()) {
 173         qDebug() << "[HTTP] Error: Invalid URL" << endl;
 174         return false;
 175     }
 176
 177     if (url.scheme() != "http") {
 178         qDebug() << "[HTTP] Error: URL must start with 'http:'" << endl;
 179         return false;
 180     }
 181
 182     if (url.path().isEmpty()) {
 183         qDebug() << "[HTTP] Error: URL has no path" << endl;
 184         return false;
 185     }
 186     m_serverTimestamp = QDateTime();
 187     // if no output file was set write to buffer
 188     if(!outputToBuffer) {
 189         if (!outputFile->open(QIODevice::ReadWrite)) {
 190             qDebug() << "[HTTP] Error: Cannot open " << qPrintable(outputFile->fileName())
 191                      << " for writing: " << qPrintable(outputFile->errorString());
 192             return false;
 193         }
 194     }
 195     else {
 196         // output to buffer. Make sure buffer is empty so no old data gets
 197         // returned in case the object is reused.
 198         dataBuffer.clear();
 199     }
 200     qDebug() << "[HTTP] GET URI" << url.toEncoded();
 201     // create request
 202     http.setHost(url.host(), url.port(80));
 203     // construct query (if any)
 204     QList<QPair<QString, QString> > qitems = url.queryItems();
 205     if(url.hasQuery()) {
 206         m_query = "?";
 207         for(int i = 0; i < qitems.size(); i++)
 208             m_query += QUrl::toPercentEncoding(qitems.at(i).first, "/") + "="
 209                   + QUrl::toPercentEncoding(qitems.at(i).second, "/") + "&";
 210     }
 211
 212     // create hash used for caching
 213     m_hash = QCryptographicHash::hash(url.toEncoded(), QCryptographicHash::Md5).toHex();
 214     // RFC2616: the absoluteURI form must get used when the request is being
 215     // sent to a proxy.
 216     m_path.clear();
 217     if(m_useproxy)
 218         m_path = url.scheme() + "://" + url.host();
 219     m_path += QString(QUrl::toPercentEncoding(url.path(), "/"));
 220
 221     // construct request header
 222     m_header.setValue("Host", url.host());
 223     m_header.setValue("User-Agent", m_globalUserAgent);
 224     m_header.setValue("Connection", "Keep-Alive");
 225
 226     if(m_dumbCache || !m_usecache) {
 227         getFileFinish();
 228     }
 229     else {
 230         // schedule HTTP header request
 231         m_header.setRequest("HEAD", m_path + m_query);
 232         headRequest = http.request(m_header);
 233         qDebug() << "[HTTP] HEAD scheduled:  " << headRequest;
 234     }
 235
 236     return true;
 237 }
 238
 239
 240 void HttpGet::getFileFinish()
 241 {
 242     m_cachefile = m_cachedir.absolutePath() + "/rbutil-cache/" + m_hash;
 243     QString indexFile = m_cachedir.absolutePath() + "/rbutil-cache/cache.txt";
 244     if(m_usecache) {
 245         // check if the file is present in cache
 246         QFileInfo cachefile = QFileInfo(m_cachefile);
 247         if(cachefile.isReadable()
 248             && cachefile.size() > 0
 249             && cachefile.lastModified() > m_serverTimestamp) {
 250
 251             qDebug() << "[HTTP] Cache: up-to-date file found:" << m_cachefile;
 252
 253             getRequest = -1;
 254             QFile c(m_cachefile);
 255             if(!outputToBuffer) {
 256                 qDebug() << "[HTTP] Cache: copying file to output"
 257                          << outputFile->fileName();
 258                 c.open(QIODevice::ReadOnly);
 259                 outputFile->open(QIODevice::ReadWrite);
 260                 outputFile->write(c.readAll());
 261                 outputFile->close();
 262                 c.close();
 263             }
 264             else {
 265                 qDebug() << "[HTTP] Cache: reading file into buffer";
 266                 c.open(QIODevice::ReadOnly);
 267                 dataBuffer = c.readAll();
 268                 c.close();
 269             }
 270             m_response = 200; // fake "200 OK" HTTP response
 271             m_cached = true;
 272             httpDone(false); // we're done now. Handle http "done" signal.
 273             return;
 274         }
 275         else {
 276             // unlink old cache file
 277             if(cachefile.isReadable()) {
 278                 QFile(m_cachefile).remove();
 279                 qDebug() << "[HTTP] Cache: outdated, timestamp:"
 280                          << cachefile.lastModified();
 281             }
 282             qDebug() << "[HTTP] Cache: caching as" << m_cachefile;
 283             // update cache index file
 284             QFile idxFile(indexFile);
 285             idxFile.open(QIODevice::ReadOnly);
 286             QByteArray currLine;
 287             do {
 288                 QByteArray currLine = idxFile.readLine(1000);
 289                 if(currLine.startsWith(m_hash.toUtf8()))
 290                     break;
 291             } while(!currLine.isEmpty());
 292             idxFile.close();
 293             if(currLine.isEmpty()) {
 294                 idxFile.open(QIODevice::Append);
 295                 QString outline = m_hash + "\t" + m_header.value("Host") + "\t"
 296                     + m_path + "\t" + m_query + "\n";
 297                 idxFile.write(outline.toUtf8());
 298                 idxFile.close();
 299             }
 300         }
 301
 302     }
 303     else {
 304         qDebug() << "[HTTP] cache DISABLED";
 305     }
 306     // schedule GET request
 307
 308     m_header.setRequest("GET", m_path + m_query);
 309     if(outputToBuffer) {
 310         qDebug() << "[HTTP] downloading to buffer.";
 311         getRequest = http.request(m_header);
 312     }
 313     else {
 314         qDebug() << "[HTTP] downloading to file:"
 315             << qPrintable(outputFile->fileName());
 316        getRequest = http.request(m_header, 0, outputFile);
 317     }
 318     qDebug() << "[HTTP] GET scheduled:   " << getRequest;
 319
 320     return;
 321 }
 322
 323
 324 void HttpGet::httpDone(bool error)
 325 {
 326     if (error) {
 327         qDebug() << "[HTTP] Error:" << qPrintable(http.errorString()) << httpResponse();
 328     }
 329     if(!outputToBuffer)
 330         outputFile->close();
 331
 332     if(m_usecache && !m_cached && !error) {
 333         qDebug() << "[HTTP] creating cache file" << m_cachefile;
 334         QFile c(m_cachefile);
 335         c.open(QIODevice::ReadWrite);
 336         if(!outputToBuffer) {
 337             outputFile->open(QIODevice::ReadOnly | QIODevice::Truncate);
 338             c.write(outputFile->readAll());
 339             outputFile->close();
 340         }
 341         else
 342             c.write(dataBuffer);
 343
 344         c.close();
 345     }
 346     // take care of concurring requests. If there is still one running,
 347     // don't emit done(). That request will call this slot again.
 348     if(http.currentId() == 0 && !http.hasPendingRequests())
 349         emit done(error);
 350 }
 351
 352
 353 void HttpGet::httpFinished(int id, bool error)
 354 {
 355     qDebug() << "[HTTP] Request finished:" << id << "Error:" << error
 356              << "pending requests:" << http.hasPendingRequests();
 357     if(id == getRequest) {
 358         dataBuffer = http.readAll();
 359         emit requestFinished(id, error);
 360     }
 361
 362     if(id == headRequest) {
 363         QHttpResponseHeader h = http.lastResponse();
 364
 365         QString date = h.value("Last-Modified").simplified();
 366         if(date.isEmpty()) {
 367             m_serverTimestamp = QDateTime(); // no value = invalid
 368             emit headerFinished();
 369             return;
 370         }
 371         // to successfully parse the date strip weekday and timezone
 372         date.remove(0, date.indexOf(" ") + 1);
 373         if(date.endsWith("GMT"))
 374             date.truncate(date.indexOf(" GMT"));
 375         // distinguish input formats (see RFC1945)
 376         // RFC 850
 377         if(date.contains("-"))
 378             m_serverTimestamp = QLocale::c().toDateTime(date, "dd-MMM-yy hh:mm:ss");
 379         // asctime format
 380         else if(date.at(0).isLetter())
 381             m_serverTimestamp = QLocale::c().toDateTime(date, "MMM d hh:mm:ss yyyy");
 382         // RFC 822
 383         else
 384             m_serverTimestamp = QLocale::c().toDateTime(date, "dd MMM yyyy hh:mm:ss");
 385         qDebug() << "[HTTP] HEAD finished, server date:" << date << ", parsed:" << m_serverTimestamp;
 386         emit headerFinished();
 387         return;
 388     }
 389     if(id == getRequest)
 390         emit requestFinished(id, error);
 391 }
 392
 393 void HttpGet::httpStarted(int id)
 394 {
 395     qDebug() << "[HTTP] Request started: " << id << "Header req:"
 396              << headRequest << "Get req:" << getRequest;
 397 }
 398
 399
 400 QString HttpGet::errorString()
 401 {
 402     return http.errorString();
 403 }
 404
 405
 406 void HttpGet::httpResponseHeader(const QHttpResponseHeader &resp)
 407 {
 408     // if there is a network error abort all scheduled requests for
 409     // this download
 410     m_response = resp.statusCode();
 411
 412     // 301 -- moved permanently
 413     // 302 -- found
 414     // 303 -- see other
 415     // 307 -- moved temporarily
 416     // in all cases, header: location has the correct address so we can follow.
 417     if(m_response == 301 || m_response == 302 || m_response == 303 || m_response == 307) {
 418         //abort without sending any signals
 419         http.blockSignals(true);
 420         http.abort();
 421         http.blockSignals(false);
 422         // start new request with new url
 423         qDebug() << "[HTTP] response =" << m_response << "- following";
 424         getFile(resp.value("location") + m_query);
 425     }
 426     else if(m_response != 200) {
 427         // all other errors are fatal.
 428         http.abort();
 429         qDebug() << "[HTTP] Response error:" << m_response << resp.reasonPhrase();
 430     }
 431
 432 }
 433
 434
 435 int HttpGet::httpResponse()
 436 {
 437     return m_response;
 438 }
 439
 440
 441 void HttpGet::httpState(int state)
 442 {
 443     QString s[] = {"Unconnected", "HostLookup", "Connecting", "Sending",
 444         "Reading", "Connected", "Closing"};
 445     if(state <= 6)
 446         qDebug() << "[HTTP] State:" << s[state];
 447     else qDebug() << "[HTTP] State:" << state;
 448 }
 449