rbutil/rbutilqt/httpget.cpp

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  *
   9  *   Copyright (C) 2007 by Dominik Riebeling
  10  *   $Id$
  11  *
  12  * All files in this archive are subject to the GNU General Public License.
  13  * See the file COPYING in the source tree root for full license agreement.
  14  *
  15  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  16  * KIND, either express or implied.
  17  *
  18  ****************************************************************************/
  19
  20 #include <QtCore>
  21 #include <QtNetwork>
  22 #include <QtDebug>
  23
  24 #include "httpget.h"
  25
  26 QDir HttpGet::m_globalCache; //< global cach path value for new objects
  27 QUrl HttpGet::m_globalProxy; //< global proxy value for new objects
  28 bool HttpGet::m_globalDumbCache = false; //< globally set cache "dumb" mode
  29
  30 HttpGet::HttpGet(QObject *parent)
  31     : QObject(parent)
  32 {
  33     outputToBuffer = true;
  34     m_cached = false;
  35     m_dumbCache = m_globalDumbCache;
  36     getRequest = -1;
  37     headRequest = -1;
  38     // if a request is cancelled before a reponse is available return some
  39     // hint about this in the http response instead of nonsense.
  40     m_response = -1;
  41
  42     // default to global proxy / cache if not empty.
  43     // proxy is automatically enabled, disable it by setting an empty proxy
  44     // cache is enabled to be in line, can get disabled with setCache(bool)
  45     if(!m_globalProxy.isEmpty())
  46         setProxy(m_globalProxy);
  47     m_usecache = false;
  48     m_cachedir = m_globalCache;
  49
  50     m_serverTimestamp = QDateTime();
  51
  52     connect(&http, SIGNAL(done(bool)), this, SLOT(httpDone(bool)));
  53     connect(&http, SIGNAL(dataReadProgress(int, int)), this, SIGNAL(dataReadProgress(int, int)));
  54     connect(&http, SIGNAL(requestFinished(int, bool)), this, SLOT(httpFinished(int, bool)));
  55     connect(&http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader&)), this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
  56     connect(&http, SIGNAL(stateChanged(int)), this, SLOT(httpState(int)));
  57     connect(&http, SIGNAL(requestStarted(int)), this, SLOT(httpStarted(int)));
  58
  59     connect(&http, SIGNAL(readyRead(const QHttpResponseHeader&)), this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
  60
  61 }
  62
  63
  64 //! @brief set cache path
  65 //  @param d new directory to use as cache path
  66 void HttpGet::setCache(QDir d)
  67 {
  68     m_cachedir = d;
  69     bool result;
  70     result = initializeCache(d);
  71     qDebug() << "[HTTP]"<< __func__ << "(QDir)" << d.absolutePath() << result;
  72     m_usecache = result;
  73 }
  74
  75
  76 /** @brief enable / disable cache useage
  77  *  @param c set cache usage
  78  */
  79 void HttpGet::setCache(bool c)
  80 {
  81     qDebug() << "[HTTP]" << __func__ << "(bool) =" << c;
  82     m_usecache = c;
  83     // make sure cache is initialized
  84     if(c)
  85         m_usecache = initializeCache(m_cachedir);
  86 }
  87
  88
  89 bool HttpGet::initializeCache(const QDir& d)
  90 {
  91     bool result;
  92     QString p = d.absolutePath() + "/rbutil-cache";
  93     if(QFileInfo(d.absolutePath()).isDir())
  94     {
  95         if(!QFileInfo(p).isDir())
  96             result = d.mkdir("rbutil-cache");
  97         else
  98             result = true;
  99     }
 100     else
 101         result = false;
 102
 103     return result;
 104
 105 }
 106
 107
 108 /** @brief read all downloaded data into a buffer
 109  *  @return data
 110  */
 111 QByteArray HttpGet::readAll()
 112 {
 113     return dataBuffer;
 114 }
 115
 116
 117 /** @brief get http error
 118  *  @return http error
 119  */
 120 QHttp::Error HttpGet::error()
 121 {
 122     return http.error();
 123 }
 124
 125
 126 void HttpGet::setProxy(const QUrl &proxy)
 127 {
 128     qDebug() << "[HTTP]" << __func__ << "(QUrl)" << proxy.toString();
 129     m_proxy = proxy;
 130     http.setProxy(m_proxy.host(), m_proxy.port(), m_proxy.userName(), m_proxy.password());
 131 }
 132
 133
 134 void HttpGet::setProxy(bool enable)
 135 {
 136     qDebug() << "[HTTP]" << __func__ << "(bool)" << enable;
 137     if(enable)
 138         http.setProxy(m_proxy.host(), m_proxy.port(), m_proxy.userName(), m_proxy.password());
 139     else
 140         http.setProxy("", 0);
 141 }
 142
 143
 144 void HttpGet::setFile(QFile *file)
 145 {
 146     outputFile = file;
 147     outputToBuffer = false;
 148     qDebug() << "[HTTP]" << __func__ << "(QFile*)" << outputFile->fileName();
 149 }
 150
 151
 152 void HttpGet::abort()
 153 {
 154     http.abort();
 155     if(!outputToBuffer)
 156         outputFile->close();
 157 }
 158
 159
 160 bool HttpGet::getFile(const QUrl &url)
 161 {
 162     if (!url.isValid()) {
 163         qDebug() << "[HTTP] Error: Invalid URL" << endl;
 164         return false;
 165     }
 166
 167     if (url.scheme() != "http") {
 168         qDebug() << "[HTTP] Error: URL must start with 'http:'" << endl;
 169         return false;
 170     }
 171
 172     if (url.path().isEmpty()) {
 173         qDebug() << "[HTTP] Error: URL has no path" << endl;
 174         return false;
 175     }
 176     m_serverTimestamp = QDateTime();
 177     // if no output file was set write to buffer
 178     if(!outputToBuffer) {
 179         if (!outputFile->open(QIODevice::ReadWrite)) {
 180             qDebug() << "[HTTP] Error: Cannot open " << qPrintable(outputFile->fileName())
 181                 << " for writing: " << qPrintable(outputFile->errorString())
 182                 << endl;
 183             return false;
 184         }
 185     }
 186     qDebug() << "[HTTP] downloading" << url.toEncoded();
 187     // create request
 188     http.setHost(url.host(), url.port(80));
 189     // construct query (if any)
 190     QList<QPair<QString, QString> > qitems = url.queryItems();
 191     if(url.hasQuery()) {
 192         m_query = "?";
 193         for(int i = 0; i < qitems.size(); i++)
 194             m_query += QUrl::toPercentEncoding(qitems.at(i).first, "/") + "="
 195                   + QUrl::toPercentEncoding(qitems.at(i).second, "/") + "&";
 196     }
 197
 198     // create hash used for caching
 199     m_hash = QCryptographicHash::hash(url.toEncoded(), QCryptographicHash::Md5).toHex();
 200     m_path = QString(QUrl::toPercentEncoding(url.path(), "/"));
 201
 202     if(m_dumbCache || !m_usecache) {
 203         getFileFinish();
 204     }
 205     else {
 206         // request HTTP header
 207         connect(this, SIGNAL(headerFinished()), this, SLOT(getFileFinish()));
 208         headRequest = http.head(m_path + m_query);
 209     }
 210
 211     return true;
 212 }
 213
 214
 215 void HttpGet::getFileFinish()
 216 {
 217     m_cachefile = m_cachedir.absolutePath() + "/rbutil-cache/" + m_hash;
 218     if(m_usecache) {
 219         // check if the file is present in cache
 220         qDebug() << "[HTTP] cache ENABLED";
 221         QFileInfo cachefile = QFileInfo(m_cachefile);
 222         if(cachefile.isReadable()
 223             && cachefile.size() > 0
 224             && cachefile.lastModified() > m_serverTimestamp) {
 225
 226             qDebug() << "[HTTP] cached file found:" << m_cachefile;
 227
 228             getRequest = -1;
 229             QFile c(m_cachefile);
 230             if(!outputToBuffer) {
 231                 qDebug() << "[HTTP] copying cache file to output" << outputFile->fileName();
 232                 c.open(QIODevice::ReadOnly);
 233                 outputFile->open(QIODevice::ReadWrite);
 234                 outputFile->write(c.readAll());
 235                 outputFile->close();
 236                 c.close();
 237             }
 238             else {
 239                 qDebug() << "[HTTP] reading cache file into buffer";
 240                 c.open(QIODevice::ReadOnly);
 241                 dataBuffer = c.readAll();
 242                 c.close();
 243             }
 244             m_response = 200; // fake "200 OK" HTTP response
 245             m_cached = true;
 246             httpDone(false); // we're done now. Fake http "done" signal.
 247             return;
 248         }
 249         else {
 250             if(cachefile.isReadable())
 251                 qDebug() << "[HTTP] file in cache timestamp:" << cachefile.lastModified();
 252             else
 253                 qDebug() << "[HTTP] file not in cache.";
 254             qDebug() << "[HTTP] server file timestamp:" << m_serverTimestamp;
 255             qDebug() << "[HTTP] downloading file to" << m_cachefile;
 256             // unlink old cache file
 257             if(cachefile.isReadable())
 258                 QFile(m_cachefile).remove();
 259         }
 260
 261     }
 262     else {
 263         qDebug() << "[HTTP] cache DISABLED";
 264     }
 265
 266     if(outputToBuffer) {
 267         qDebug() << "[HTTP] downloading to buffer.";
 268         getRequest = http.get(m_path + m_query);
 269     }
 270     else {
 271         qDebug() << "[HTTP] downloading to file:"
 272             << qPrintable(outputFile->fileName());
 273         getRequest = http.get(m_path + m_query, outputFile);
 274     }
 275     qDebug() << "[HTTP] GET request scheduled, id:" << getRequest;
 276
 277     return;
 278 }
 279
 280
 281 void HttpGet::httpDone(bool error)
 282 {
 283     if (error) {
 284         qDebug() << "[HTTP] Error: " << qPrintable(http.errorString()) << httpResponse();
 285     }
 286     if(!outputToBuffer)
 287         outputFile->close();
 288
 289     if(m_usecache && !m_cached && !error) {
 290         qDebug() << "[HTTP] creating cache file" << m_cachefile;
 291         QFile c(m_cachefile);
 292         c.open(QIODevice::ReadWrite);
 293         if(!outputToBuffer) {
 294             outputFile->open(QIODevice::ReadOnly | QIODevice::Truncate);
 295             c.write(outputFile->readAll());
 296             outputFile->close();
 297         }
 298         else
 299             c.write(dataBuffer);
 300
 301         c.close();
 302     }
 303     // if cached file found and cache enabled ignore http errors
 304     if(m_usecache && m_cached && !http.hasPendingRequests()) {
 305         error = false;
 306     }
 307     // take care of concurring requests. If there is still one running,
 308     // don't emit done(). That request will call this slot again.
 309     if(http.currentId() == 0 && !http.hasPendingRequests())
 310         emit done(error);
 311 }
 312
 313
 314 void HttpGet::httpFinished(int id, bool error)
 315 {
 316     qDebug() << "[HTTP]" << __func__ << "(int, bool) =" << id << error;
 317     if(id == getRequest) {
 318         dataBuffer = http.readAll();
 319
 320         emit requestFinished(id, error);
 321     }
 322     qDebug() << "[HTTP] hasPendingRequests =" << http.hasPendingRequests();
 323
 324
 325     if(id == headRequest) {
 326         QHttpResponseHeader h = http.lastResponse();
 327
 328         QString date = h.value("Last-Modified").simplified();
 329         if(date.isEmpty()) {
 330             m_serverTimestamp = QDateTime(); // no value = invalid
 331             emit headerFinished();
 332             return;
 333         }
 334         // to successfully parse the date strip weekday and timezone
 335         date.remove(0, date.indexOf(" ") + 1);
 336         if(date.endsWith("GMT"))
 337             date.truncate(date.indexOf(" GMT"));
 338         // distinguish input formats (see RFC1945)
 339         // RFC 850
 340         if(date.contains("-"))
 341             m_serverTimestamp = QDateTime::fromString(date, "dd-MMM-yy hh:mm:ss");
 342         // asctime format
 343         else if(date.at(0).isLetter())
 344             m_serverTimestamp = QDateTime::fromString(date, "MMM d hh:mm:ss yyyy");
 345         // RFC 822
 346         else
 347             m_serverTimestamp = QDateTime::fromString(date, "dd MMM yyyy hh:mm:ss");
 348         qDebug() << "[HTTP] Header Request Date:" << date << ", parsed:" << m_serverTimestamp;
 349         emit headerFinished();
 350         return;
 351     }
 352     if(id == getRequest)
 353         emit requestFinished(id, error);
 354 }
 355
 356 void HttpGet::httpStarted(int id)
 357 {
 358     qDebug() << "[HTTP]" << __func__ << "(int) =" << id;
 359     qDebug() << "headRequest" << headRequest << "getRequest" << getRequest;
 360 }
 361
 362
 363 QString HttpGet::errorString()
 364 {
 365     return http.errorString();
 366 }
 367
 368
 369 void HttpGet::httpResponseHeader(const QHttpResponseHeader &resp)
 370 {
 371     // if there is a network error abort all scheduled requests for
 372     // this download
 373     m_response = resp.statusCode();
 374     if(m_response != 200) {
 375         qDebug() << "[HTTP] response error =" << m_response << resp.reasonPhrase();
 376         http.abort();
 377     }
 378     // 301 -- moved permanently
 379     // 302 -- found
 380     // 303 -- see other
 381     // 307 -- moved temporarily
 382     // in all cases, header: location has the correct address so we can follow.
 383     if(m_response == 301 || m_response == 302 || m_response == 303 || m_response == 307) {
 384         // start new request with new url
 385         qDebug() << "[HTTP] response =" << m_response << "- following";
 386         getFile(resp.value("location") + m_query);
 387     }
 388 }
 389
 390
 391 int HttpGet::httpResponse()
 392 {
 393     return m_response;
 394 }
 395
 396
 397 void HttpGet::httpState(int state)
 398 {
 399     QString s[] = {"Unconnected", "HostLookup", "Connecting", "Sending",
 400         "Reading", "Connected", "Closing"};
 401     if(state <= 6)
 402         qDebug() << "[HTTP]" << __func__ << "() = " << s[state];
 403     else qDebug() << "[HTTP]" << __func__ << "() = " << state;
 404 }
 405