rbutil/rbutilqt/httpget.cpp

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  *
   9  *   Copyright (C) 2007 by Dominik Riebeling
  10  *   $Id$
  11  *
  12  * All files in this archive are subject to the GNU General Public License.
  13  * See the file COPYING in the source tree root for full license agreement.
  14  *
  15  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  16  * KIND, either express or implied.
  17  *
  18  ****************************************************************************/
  19
  20 #include <QtCore>
  21 #include <QtNetwork>
  22 #include <QtDebug>
  23
  24 #include "httpget.h"
  25
  26 QDir HttpGet::m_globalCache; //< global cach path value for new objects
  27 QUrl HttpGet::m_globalProxy; //< global proxy value for new objects
  28 bool HttpGet::m_globalDumbCache = false; //< globally set cache "dumb" mode
  29
  30 HttpGet::HttpGet(QObject *parent)
  31     : QObject(parent)
  32 {
  33     outputToBuffer = true;
  34     m_cached = false;
  35     m_dumbCache = m_globalDumbCache;
  36     getRequest = -1;
  37     // if a request is cancelled before a reponse is available return some
  38     // hint about this in the http response instead of nonsense.
  39     m_response = -1;
  40
  41     // default to global proxy / cache if not empty.
  42     // proxy is automatically enabled, disable it by setting an empty proxy
  43     // cache is enabled to be in line, can get disabled with setCache(bool)
  44     if(!m_globalProxy.isEmpty())
  45         setProxy(m_globalProxy);
  46     m_usecache = false;
  47     m_cachedir = m_globalCache;
  48     connect(&http, SIGNAL(done(bool)), this, SLOT(httpDone(bool)));
  49     connect(&http, SIGNAL(dataReadProgress(int, int)), this, SLOT(httpProgress(int, int)));
  50     connect(&http, SIGNAL(requestFinished(int, bool)), this, SLOT(httpFinished(int, bool)));
  51     connect(&http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader&)), this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
  52     connect(&http, SIGNAL(stateChanged(int)), this, SLOT(httpState(int)));
  53     connect(&http, SIGNAL(requestStarted(int)), this, SLOT(httpStarted(int)));
  54
  55     connect(&http, SIGNAL(readyRead(const QHttpResponseHeader&)), this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
  56
  57 }
  58
  59
  60 //! @brief set cache path
  61 //  @param d new directory to use as cache path
  62 void HttpGet::setCache(QDir d)
  63 {
  64     m_cachedir = d;
  65     bool result;
  66     result = initializeCache(d);
  67     qDebug() << "[HTTP]"<< __func__ << "(QDir)" << d.absolutePath() << result;
  68     m_usecache = result;
  69 }
  70
  71
  72 /** @brief enable / disable cache useage
  73  *  @param c set cache usage
  74  */
  75 void HttpGet::setCache(bool c)
  76 {
  77     qDebug() << "[HTTP]" << __func__ << "(bool) =" << c;
  78     m_usecache = c;
  79     // make sure cache is initialized
  80     if(c)
  81         m_usecache = initializeCache(m_cachedir);
  82 }
  83
  84
  85 bool HttpGet::initializeCache(const QDir& d)
  86 {
  87     bool result;
  88     QString p = d.absolutePath() + "/rbutil-cache";
  89     if(QFileInfo(d.absolutePath()).isDir())
  90     {
  91         if(!QFileInfo(p).isDir())
  92             result = d.mkdir("rbutil-cache");
  93         else
  94             result = true;
  95     }
  96     else
  97         result = false;
  98
  99     return result;
 100
 101 }
 102
 103
 104 /** @brief read all downloaded data into a buffer
 105  *  @return data
 106  */
 107 QByteArray HttpGet::readAll()
 108 {
 109     return dataBuffer;
 110 }
 111
 112
 113 /** @brief get http error
 114  *  @return http error
 115  */
 116 QHttp::Error HttpGet::error()
 117 {
 118     return http.error();
 119 }
 120
 121
 122 void HttpGet::httpProgress(int read, int total)
 123 {
 124     emit dataReadProgress(read, total);
 125 }
 126
 127
 128 void HttpGet::setProxy(const QUrl &proxy)
 129 {
 130     qDebug() << "[HTTP]" << __func__ << "(QUrl)" << proxy.toString();
 131     m_proxy = proxy;
 132     http.setProxy(m_proxy.host(), m_proxy.port(), m_proxy.userName(), m_proxy.password());
 133 }
 134
 135
 136 void HttpGet::setProxy(bool enable)
 137 {
 138     qDebug() << "[HTTP]" << __func__ << "(bool)" << enable;
 139     if(enable)
 140         http.setProxy(m_proxy.host(), m_proxy.port(), m_proxy.userName(), m_proxy.password());
 141     else
 142         http.setProxy("", 0);
 143 }
 144
 145
 146 void HttpGet::setFile(QFile *file)
 147 {
 148     outputFile = file;
 149     outputToBuffer = false;
 150     qDebug() << "[HTTP]" << __func__ << "(QFile*)" << outputFile->fileName();
 151 }
 152
 153
 154 void HttpGet::abort()
 155 {
 156     http.abort();
 157     if(!outputToBuffer)
 158         outputFile->close();
 159 }
 160
 161
 162 bool HttpGet::getFile(const QUrl &url)
 163 {
 164     if (!url.isValid()) {
 165         qDebug() << "[HTTP] Error: Invalid URL" << endl;
 166         return false;
 167     }
 168
 169     if (url.scheme() != "http") {
 170         qDebug() << "[HTTP] Error: URL must start with 'http:'" << endl;
 171         return false;
 172     }
 173
 174     if (url.path().isEmpty()) {
 175         qDebug() << "[HTTP] Error: URL has no path" << endl;
 176         return false;
 177     }
 178     // if no output file was set write to buffer
 179     if(!outputToBuffer) {
 180         if (!outputFile->open(QIODevice::ReadWrite)) {
 181             qDebug() << "[HTTP] Error: Cannot open " << qPrintable(outputFile->fileName())
 182                 << " for writing: " << qPrintable(outputFile->errorString())
 183                 << endl;
 184             return false;
 185         }
 186     }
 187     qDebug() << "[HTTP] downloading" << url.toEncoded();
 188     // create request
 189     http.setHost(url.host(), url.port(80));
 190     // construct query (if any)
 191     QList<QPair<QString, QString> > qitems = url.queryItems();
 192     if(url.hasQuery()) {
 193         m_query = "?";
 194         for(int i = 0; i < qitems.size(); i++)
 195             m_query += QUrl::toPercentEncoding(qitems.at(i).first, "/") + "="
 196                   + QUrl::toPercentEncoding(qitems.at(i).second, "/") + "&";
 197     }
 198
 199     // create hash used for caching
 200     m_hash = QCryptographicHash::hash(url.toEncoded(), QCryptographicHash::Md5).toHex();
 201     m_path = QString(QUrl::toPercentEncoding(url.path(), "/"));
 202
 203     if(m_dumbCache || !m_usecache) {
 204         getFileFinish();
 205     }
 206     else {
 207         // request HTTP header
 208         connect(this, SIGNAL(headerFinished()), this, SLOT(getFileFinish()));
 209         headRequest = http.head(m_path + m_query);
 210     }
 211
 212     return true;
 213 }
 214
 215
 216 void HttpGet::getFileFinish()
 217 {
 218     m_cachefile = m_cachedir.absolutePath() + "/rbutil-cache/" + m_hash;
 219     if(m_usecache) {
 220         // check if the file is present in cache
 221         qDebug() << "[HTTP] cache ENABLED";
 222         QFileInfo cachefile = QFileInfo(m_cachefile);
 223         if(cachefile.isReadable()
 224             && cachefile.size() > 0
 225             && cachefile.lastModified() > m_serverTimestamp) {
 226
 227             qDebug() << "[HTTP] cached file found:" << m_cachefile;
 228
 229             getRequest = -1;
 230             QFile c(m_cachefile);
 231             if(!outputToBuffer) {
 232                 qDebug() << "[HTTP] copying cache file to output" << outputFile->fileName();
 233                 c.open(QIODevice::ReadOnly);
 234                 outputFile->open(QIODevice::ReadWrite);
 235                 outputFile->write(c.readAll());
 236                 outputFile->close();
 237                 c.close();
 238             }
 239             else {
 240                 qDebug() << "[HTTP] reading cache file into buffer";
 241                 c.open(QIODevice::ReadOnly);
 242                 dataBuffer = c.readAll();
 243                 c.close();
 244             }
 245             m_response = 200; // fake "200 OK" HTTP response
 246             m_cached = true;
 247             httpDone(false); // we're done now. Fake http "done" signal.
 248             return;
 249         }
 250         else {
 251             if(cachefile.isReadable())
 252                 qDebug() << "[HTTP] file in cache timestamp:" << cachefile.lastModified();
 253             else
 254                 qDebug() << "[HTTP] file not in cache.";
 255             qDebug() << "[HTTP] server file timestamp:" << m_serverTimestamp;
 256             qDebug() << "[HTTP] downloading file to" << m_cachefile;
 257             // unlink old cache file
 258             if(cachefile.isReadable())
 259                 QFile(m_cachefile).remove();
 260         }
 261
 262     }
 263     else {
 264         qDebug() << "[HTTP] cache DISABLED";
 265     }
 266
 267     if(outputToBuffer) {
 268         qDebug() << "[HTTP] downloading to buffer.";
 269         getRequest = http.get(m_path + m_query);
 270     }
 271     else {
 272         qDebug() << "[HTTP] downloading to file:"
 273             << qPrintable(outputFile->fileName());
 274         getRequest = http.get(m_path + m_query, outputFile);
 275     }
 276     qDebug() << "[HTTP] GET request scheduled, id:" << getRequest;
 277
 278     return;
 279 }
 280
 281
 282 void HttpGet::httpDone(bool error)
 283 {
 284     if (error) {
 285         qDebug() << "[HTTP] Error: " << qPrintable(http.errorString()) << httpResponse();
 286     }
 287     if(!outputToBuffer)
 288         outputFile->close();
 289
 290     if(m_usecache && !m_cached) {
 291         qDebug() << "[HTTP] creating cache file" << m_cachefile;
 292         QFile c(m_cachefile);
 293         c.open(QIODevice::ReadWrite);
 294         if(!outputToBuffer) {
 295             outputFile->open(QIODevice::ReadOnly | QIODevice::Truncate);
 296             c.write(outputFile->readAll());
 297             outputFile->close();
 298         }
 299         else
 300             c.write(dataBuffer);
 301
 302         c.close();
 303     }
 304     m_serverTimestamp = QDateTime();
 305     // take care of concurring requests. If there is still one running,
 306     // don't emit done(). That request will call this slot again.
 307     if(http.currentId() == 0 && !http.hasPendingRequests())
 308         emit done(error);
 309 }
 310
 311
 312 void HttpGet::httpFinished(int id, bool error)
 313 {
 314     qDebug() << "[HTTP]" << __func__ << "(int, bool) =" << id << error;
 315     if(id == getRequest) {
 316         dataBuffer = http.readAll();
 317
 318         emit requestFinished(id, error);
 319     }
 320     qDebug() << "[HTTP] hasPendingRequests =" << http.hasPendingRequests();
 321
 322
 323     if(id == headRequest) {
 324         QHttpResponseHeader h = http.lastResponse();
 325
 326         QString date = h.value("Last-Modified").simplified();
 327         if(date.isEmpty()) {
 328             m_serverTimestamp = QDateTime(); // no value = invalid
 329             emit headerFinished();
 330             return;
 331         }
 332         // to successfully parse the date strip weekday and timezone
 333         date.remove(0, date.indexOf(" ") + 1);
 334         if(date.endsWith("GMT"))
 335             date.truncate(date.indexOf(" GMT"));
 336         // distinguish input formats (see RFC1945)
 337         // RFC 850
 338         if(date.contains("-"))
 339             m_serverTimestamp = QDateTime::fromString(date, "dd-MMM-yy hh:mm:ss");
 340         // asctime format
 341         else if(date.at(0).isLetter())
 342             m_serverTimestamp = QDateTime::fromString(date, "MMM d hh:mm:ss yyyy");
 343         // RFC 822
 344         else
 345             m_serverTimestamp = QDateTime::fromString(date, "dd MMM yyyy hh:mm:ss");
 346         qDebug() << "[HTTP] Header Request Date:" << date << ", parsed:" << m_serverTimestamp;
 347         emit headerFinished();
 348         return;
 349     }
 350     if(id == getRequest)
 351         emit requestFinished(id, error);
 352 }
 353
 354 void HttpGet::httpStarted(int id)
 355 {
 356     qDebug() << "[HTTP]" << __func__ << "(int) =" << id;
 357     qDebug() << "headRequest" << headRequest << "getRequest" << getRequest;
 358 }
 359
 360
 361 QString HttpGet::errorString()
 362 {
 363     return http.errorString();
 364 }
 365
 366
 367 void HttpGet::httpResponseHeader(const QHttpResponseHeader &resp)
 368 {
 369     // if there is a network error abort all scheduled requests for
 370     // this download
 371     m_response = resp.statusCode();
 372     if(m_response != 200) {
 373         qDebug() << "[HTTP] response error =" << m_response << resp.reasonPhrase();
 374         http.abort();
 375     }
 376     // 301 -- moved permanently
 377     // 302 -- found
 378     // 303 -- see other
 379     // 307 -- moved temporarily
 380     // in all cases, header: location has the correct address so we can follow.
 381     if(m_response == 301 || m_response == 302 || m_response == 303 || m_response == 307) {
 382         // start new request with new url
 383         qDebug() << "[HTTP] response =" << m_response << "- following";
 384         getFile(resp.value("location") + m_query);
 385     }
 386 }
 387
 388
 389 int HttpGet::httpResponse()
 390 {
 391     return m_response;
 392 }
 393
 394
 395 void HttpGet::httpState(int state)
 396 {
 397     QString s[] = {"Unconnected", "HostLookup", "Connecting", "Sending",
 398         "Reading", "Connected", "Closing"};
 399     if(state <= 6)
 400         qDebug() << "[HTTP]" << __func__ << "() = " << s[state];
 401     else qDebug() << "[HTTP]" << __func__ << "() = " << state;
 402 }
 403