rbutil/rbutilqt/httpget.cpp

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  *
   9  *   Copyright (C) 2007 by Dominik Riebeling
  10  *   $Id$
  11  *
  12  * All files in this archive are subject to the GNU General Public License.
  13  * See the file COPYING in the source tree root for full license agreement.
  14  *
  15  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  16  * KIND, either express or implied.
  17  *
  18  ****************************************************************************/
  19
  20 #include <QtCore>
  21 #include <QtNetwork>
  22 #include <QtDebug>
  23
  24 #include "httpget.h"
  25
  26 QDir HttpGet::m_globalCache; //< global cach path value for new objects
  27 QUrl HttpGet::m_globalProxy; //< global proxy value for new objects
  28 bool HttpGet::m_globalDumbCache = false; //< globally set cache "dumb" mode
  29
  30 HttpGet::HttpGet(QObject *parent)
  31     : QObject(parent)
  32 {
  33     outputToBuffer = true;
  34     m_cached = false;
  35     m_dumbCache = m_globalDumbCache;
  36     getRequest = -1;
  37     // if a request is cancelled before a reponse is available return some
  38     // hint about this in the http response instead of nonsense.
  39     m_response = -1;
  40
  41     // default to global proxy / cache if not empty.
  42     // proxy is automatically enabled, disable it by setting an empty proxy
  43     // cache is enabled to be in line, can get disabled with setCache(bool)
  44     if(!m_globalProxy.isEmpty())
  45         setProxy(m_globalProxy);
  46     m_usecache = false;
  47     m_cachedir = m_globalCache;
  48
  49     m_serverTimestamp = QDateTime();
  50
  51     connect(&http, SIGNAL(done(bool)), this, SLOT(httpDone(bool)));
  52     connect(&http, SIGNAL(dataReadProgress(int, int)), this, SLOT(httpProgress(int, int)));
  53     connect(&http, SIGNAL(requestFinished(int, bool)), this, SLOT(httpFinished(int, bool)));
  54     connect(&http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader&)), this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
  55     connect(&http, SIGNAL(stateChanged(int)), this, SLOT(httpState(int)));
  56     connect(&http, SIGNAL(requestStarted(int)), this, SLOT(httpStarted(int)));
  57
  58     connect(&http, SIGNAL(readyRead(const QHttpResponseHeader&)), this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
  59
  60 }
  61
  62
  63 //! @brief set cache path
  64 //  @param d new directory to use as cache path
  65 void HttpGet::setCache(QDir d)
  66 {
  67     m_cachedir = d;
  68     bool result;
  69     result = initializeCache(d);
  70     qDebug() << "[HTTP]"<< __func__ << "(QDir)" << d.absolutePath() << result;
  71     m_usecache = result;
  72 }
  73
  74
  75 /** @brief enable / disable cache useage
  76  *  @param c set cache usage
  77  */
  78 void HttpGet::setCache(bool c)
  79 {
  80     qDebug() << "[HTTP]" << __func__ << "(bool) =" << c;
  81     m_usecache = c;
  82     // make sure cache is initialized
  83     if(c)
  84         m_usecache = initializeCache(m_cachedir);
  85 }
  86
  87
  88 bool HttpGet::initializeCache(const QDir& d)
  89 {
  90     bool result;
  91     QString p = d.absolutePath() + "/rbutil-cache";
  92     if(QFileInfo(d.absolutePath()).isDir())
  93     {
  94         if(!QFileInfo(p).isDir())
  95             result = d.mkdir("rbutil-cache");
  96         else
  97             result = true;
  98     }
  99     else
 100         result = false;
 101
 102     return result;
 103
 104 }
 105
 106
 107 /** @brief read all downloaded data into a buffer
 108  *  @return data
 109  */
 110 QByteArray HttpGet::readAll()
 111 {
 112     return dataBuffer;
 113 }
 114
 115
 116 /** @brief get http error
 117  *  @return http error
 118  */
 119 QHttp::Error HttpGet::error()
 120 {
 121     return http.error();
 122 }
 123
 124
 125 void HttpGet::httpProgress(int read, int total)
 126 {
 127     emit dataReadProgress(read, total);
 128 }
 129
 130
 131 void HttpGet::setProxy(const QUrl &proxy)
 132 {
 133     qDebug() << "[HTTP]" << __func__ << "(QUrl)" << proxy.toString();
 134     m_proxy = proxy;
 135     http.setProxy(m_proxy.host(), m_proxy.port(), m_proxy.userName(), m_proxy.password());
 136 }
 137
 138
 139 void HttpGet::setProxy(bool enable)
 140 {
 141     qDebug() << "[HTTP]" << __func__ << "(bool)" << enable;
 142     if(enable)
 143         http.setProxy(m_proxy.host(), m_proxy.port(), m_proxy.userName(), m_proxy.password());
 144     else
 145         http.setProxy("", 0);
 146 }
 147
 148
 149 void HttpGet::setFile(QFile *file)
 150 {
 151     outputFile = file;
 152     outputToBuffer = false;
 153     qDebug() << "[HTTP]" << __func__ << "(QFile*)" << outputFile->fileName();
 154 }
 155
 156
 157 void HttpGet::abort()
 158 {
 159     http.abort();
 160     if(!outputToBuffer)
 161         outputFile->close();
 162 }
 163
 164
 165 bool HttpGet::getFile(const QUrl &url)
 166 {
 167     if (!url.isValid()) {
 168         qDebug() << "[HTTP] Error: Invalid URL" << endl;
 169         return false;
 170     }
 171
 172     if (url.scheme() != "http") {
 173         qDebug() << "[HTTP] Error: URL must start with 'http:'" << endl;
 174         return false;
 175     }
 176
 177     if (url.path().isEmpty()) {
 178         qDebug() << "[HTTP] Error: URL has no path" << endl;
 179         return false;
 180     }
 181     m_serverTimestamp = QDateTime();
 182     // if no output file was set write to buffer
 183     if(!outputToBuffer) {
 184         if (!outputFile->open(QIODevice::ReadWrite)) {
 185             qDebug() << "[HTTP] Error: Cannot open " << qPrintable(outputFile->fileName())
 186                 << " for writing: " << qPrintable(outputFile->errorString())
 187                 << endl;
 188             return false;
 189         }
 190     }
 191     qDebug() << "[HTTP] downloading" << url.toEncoded();
 192     // create request
 193     http.setHost(url.host(), url.port(80));
 194     // construct query (if any)
 195     QList<QPair<QString, QString> > qitems = url.queryItems();
 196     if(url.hasQuery()) {
 197         m_query = "?";
 198         for(int i = 0; i < qitems.size(); i++)
 199             m_query += QUrl::toPercentEncoding(qitems.at(i).first, "/") + "="
 200                   + QUrl::toPercentEncoding(qitems.at(i).second, "/") + "&";
 201     }
 202
 203     // create hash used for caching
 204     m_hash = QCryptographicHash::hash(url.toEncoded(), QCryptographicHash::Md5).toHex();
 205     m_path = QString(QUrl::toPercentEncoding(url.path(), "/"));
 206
 207     if(m_dumbCache || !m_usecache) {
 208         getFileFinish();
 209     }
 210     else {
 211         // request HTTP header
 212         connect(this, SIGNAL(headerFinished()), this, SLOT(getFileFinish()));
 213         headRequest = http.head(m_path + m_query);
 214     }
 215
 216     return true;
 217 }
 218
 219
 220 void HttpGet::getFileFinish()
 221 {
 222     m_cachefile = m_cachedir.absolutePath() + "/rbutil-cache/" + m_hash;
 223     if(m_usecache) {
 224         // check if the file is present in cache
 225         qDebug() << "[HTTP] cache ENABLED";
 226         QFileInfo cachefile = QFileInfo(m_cachefile);
 227         if(cachefile.isReadable()
 228             && cachefile.size() > 0
 229             && cachefile.lastModified() > m_serverTimestamp) {
 230
 231             qDebug() << "[HTTP] cached file found:" << m_cachefile;
 232
 233             getRequest = -1;
 234             QFile c(m_cachefile);
 235             if(!outputToBuffer) {
 236                 qDebug() << "[HTTP] copying cache file to output" << outputFile->fileName();
 237                 c.open(QIODevice::ReadOnly);
 238                 outputFile->open(QIODevice::ReadWrite);
 239                 outputFile->write(c.readAll());
 240                 outputFile->close();
 241                 c.close();
 242             }
 243             else {
 244                 qDebug() << "[HTTP] reading cache file into buffer";
 245                 c.open(QIODevice::ReadOnly);
 246                 dataBuffer = c.readAll();
 247                 c.close();
 248             }
 249             m_response = 200; // fake "200 OK" HTTP response
 250             m_cached = true;
 251             httpDone(false); // we're done now. Fake http "done" signal.
 252             return;
 253         }
 254         else {
 255             if(cachefile.isReadable())
 256                 qDebug() << "[HTTP] file in cache timestamp:" << cachefile.lastModified();
 257             else
 258                 qDebug() << "[HTTP] file not in cache.";
 259             qDebug() << "[HTTP] server file timestamp:" << m_serverTimestamp;
 260             qDebug() << "[HTTP] downloading file to" << m_cachefile;
 261             // unlink old cache file
 262             if(cachefile.isReadable())
 263                 QFile(m_cachefile).remove();
 264         }
 265
 266     }
 267     else {
 268         qDebug() << "[HTTP] cache DISABLED";
 269     }
 270
 271     if(outputToBuffer) {
 272         qDebug() << "[HTTP] downloading to buffer.";
 273         getRequest = http.get(m_path + m_query);
 274     }
 275     else {
 276         qDebug() << "[HTTP] downloading to file:"
 277             << qPrintable(outputFile->fileName());
 278         getRequest = http.get(m_path + m_query, outputFile);
 279     }
 280     qDebug() << "[HTTP] GET request scheduled, id:" << getRequest;
 281
 282     return;
 283 }
 284
 285
 286 void HttpGet::httpDone(bool error)
 287 {
 288     if (error) {
 289         qDebug() << "[HTTP] Error: " << qPrintable(http.errorString()) << httpResponse();
 290     }
 291     if(!outputToBuffer)
 292         outputFile->close();
 293
 294     if(m_usecache && !m_cached) {
 295         qDebug() << "[HTTP] creating cache file" << m_cachefile;
 296         QFile c(m_cachefile);
 297         c.open(QIODevice::ReadWrite);
 298         if(!outputToBuffer) {
 299             outputFile->open(QIODevice::ReadOnly | QIODevice::Truncate);
 300             c.write(outputFile->readAll());
 301             outputFile->close();
 302         }
 303         else
 304             c.write(dataBuffer);
 305
 306         c.close();
 307     }
 308     // take care of concurring requests. If there is still one running,
 309     // don't emit done(). That request will call this slot again.
 310     if(http.currentId() == 0 && !http.hasPendingRequests())
 311         emit done(error);
 312 }
 313
 314
 315 void HttpGet::httpFinished(int id, bool error)
 316 {
 317     qDebug() << "[HTTP]" << __func__ << "(int, bool) =" << id << error;
 318     if(id == getRequest) {
 319         dataBuffer = http.readAll();
 320
 321         emit requestFinished(id, error);
 322     }
 323     qDebug() << "[HTTP] hasPendingRequests =" << http.hasPendingRequests();
 324
 325
 326     if(id == headRequest) {
 327         QHttpResponseHeader h = http.lastResponse();
 328
 329         QString date = h.value("Last-Modified").simplified();
 330         if(date.isEmpty()) {
 331             m_serverTimestamp = QDateTime(); // no value = invalid
 332             emit headerFinished();
 333             return;
 334         }
 335         // to successfully parse the date strip weekday and timezone
 336         date.remove(0, date.indexOf(" ") + 1);
 337         if(date.endsWith("GMT"))
 338             date.truncate(date.indexOf(" GMT"));
 339         // distinguish input formats (see RFC1945)
 340         // RFC 850
 341         if(date.contains("-"))
 342             m_serverTimestamp = QDateTime::fromString(date, "dd-MMM-yy hh:mm:ss");
 343         // asctime format
 344         else if(date.at(0).isLetter())
 345             m_serverTimestamp = QDateTime::fromString(date, "MMM d hh:mm:ss yyyy");
 346         // RFC 822
 347         else
 348             m_serverTimestamp = QDateTime::fromString(date, "dd MMM yyyy hh:mm:ss");
 349         qDebug() << "[HTTP] Header Request Date:" << date << ", parsed:" << m_serverTimestamp;
 350         emit headerFinished();
 351         return;
 352     }
 353     if(id == getRequest)
 354         emit requestFinished(id, error);
 355 }
 356
 357 void HttpGet::httpStarted(int id)
 358 {
 359     qDebug() << "[HTTP]" << __func__ << "(int) =" << id;
 360     qDebug() << "headRequest" << headRequest << "getRequest" << getRequest;
 361 }
 362
 363
 364 QString HttpGet::errorString()
 365 {
 366     return http.errorString();
 367 }
 368
 369
 370 void HttpGet::httpResponseHeader(const QHttpResponseHeader &resp)
 371 {
 372     // if there is a network error abort all scheduled requests for
 373     // this download
 374     m_response = resp.statusCode();
 375     if(m_response != 200) {
 376         qDebug() << "[HTTP] response error =" << m_response << resp.reasonPhrase();
 377         http.abort();
 378     }
 379     // 301 -- moved permanently
 380     // 302 -- found
 381     // 303 -- see other
 382     // 307 -- moved temporarily
 383     // in all cases, header: location has the correct address so we can follow.
 384     if(m_response == 301 || m_response == 302 || m_response == 303 || m_response == 307) {
 385         // start new request with new url
 386         qDebug() << "[HTTP] response =" << m_response << "- following";
 387         getFile(resp.value("location") + m_query);
 388     }
 389 }
 390
 391
 392 int HttpGet::httpResponse()
 393 {
 394     return m_response;
 395 }
 396
 397
 398 void HttpGet::httpState(int state)
 399 {
 400     QString s[] = {"Unconnected", "HostLookup", "Connecting", "Sending",
 401         "Reading", "Connected", "Closing"};
 402     if(state <= 6)
 403         qDebug() << "[HTTP]" << __func__ << "() = " << s[state];
 404     else qDebug() << "[HTTP]" << __func__ << "() = " << state;
 405 }
 406