rbutil/rbutilqt/base/httpget.cpp

   1 /***************************************************************************
   2  *             __________               __   ___.
   3  *   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
   4  *   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
   5  *   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
   6  *   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
   7  *                     \/            \/     \/    \/            \/
   8  *
   9  *   Copyright (C) 2007 by Dominik Riebeling
  10  *
  11  * All files in this archive are subject to the GNU General Public License.
  12  * See the file COPYING in the source tree root for full license agreement.
  13  *
  14  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
  15  * KIND, either express or implied.
  16  *
  17  ****************************************************************************/
  18
  19 #include <QtCore>
  20 #include <QtNetwork>
  21 #include <QtDebug>
  22
  23 #include "httpget.h"
  24
  25 QDir HttpGet::m_globalCache; //< global cach path value for new objects
  26 QUrl HttpGet::m_globalProxy; //< global proxy value for new objects
  27 bool HttpGet::m_globalDumbCache = false; //< globally set cache "dumb" mode
  28 QString HttpGet::m_globalUserAgent; //< globally set user agent for requests
  29
  30 HttpGet::HttpGet(QObject *parent)
  31     : QObject(parent)
  32 {
  33     outputToBuffer = true;
  34     m_cached = false;
  35     m_dumbCache = m_globalDumbCache;
  36     getRequest = -1;
  37     headRequest = -1;
  38     // if a request is cancelled before a reponse is available return some
  39     // hint about this in the http response instead of nonsense.
  40     m_response = -1;
  41     m_useproxy = false;
  42
  43     // default to global proxy / cache if not empty.
  44     // proxy is automatically enabled, disable it by setting an empty proxy
  45     // cache is enabled to be in line, can get disabled with setCache(bool)
  46     if(!m_globalProxy.isEmpty())
  47         setProxy(m_globalProxy);
  48     m_usecache = false;
  49     m_cachedir = m_globalCache;
  50
  51     m_serverTimestamp = QDateTime();
  52
  53     connect(&http, SIGNAL(done(bool)), this, SLOT(httpDone(bool)));
  54     connect(&http, SIGNAL(dataReadProgress(int, int)),
  55             this, SIGNAL(dataReadProgress(int, int)));
  56     connect(&http, SIGNAL(requestFinished(int, bool)),
  57             this, SLOT(httpFinished(int, bool)));
  58     connect(&http, SIGNAL(responseHeaderReceived(const QHttpResponseHeader&)),
  59             this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
  60 //    connect(&http, SIGNAL(stateChanged(int)), this, SLOT(httpState(int)));
  61     connect(&http, SIGNAL(requestStarted(int)), this, SLOT(httpStarted(int)));
  62
  63     connect(&http, SIGNAL(readyRead(const QHttpResponseHeader&)),
  64             this, SLOT(httpResponseHeader(const QHttpResponseHeader&)));
  65     connect(this, SIGNAL(headerFinished()), this, SLOT(getFileFinish()));
  66
  67 }
  68
  69
  70 //! @brief set cache path
  71 //  @param d new directory to use as cache path
  72 void HttpGet::setCache(const QDir& d)
  73 {
  74     m_cachedir = d;
  75     bool result;
  76     result = initializeCache(d);
  77     m_usecache = result;
  78 }
  79
  80
  81 /** @brief enable / disable cache useage
  82  *  @param c set cache usage
  83  */
  84 void HttpGet::setCache(bool c)
  85 {
  86     m_usecache = c;
  87     // make sure cache is initialized
  88     if(c)
  89         m_usecache = initializeCache(m_cachedir);
  90 }
  91
  92
  93 bool HttpGet::initializeCache(const QDir& d)
  94 {
  95     bool result;
  96     QString p = d.absolutePath() + "/rbutil-cache";
  97     if(QFileInfo(d.absolutePath()).isDir())
  98     {
  99         if(!QFileInfo(p).isDir())
 100             result = d.mkdir("rbutil-cache");
 101         else
 102             result = true;
 103     }
 104     else
 105         result = false;
 106
 107     return result;
 108
 109 }
 110
 111
 112 /** @brief read all downloaded data into a buffer
 113  *  @return data
 114  */
 115 QByteArray HttpGet::readAll()
 116 {
 117     return dataBuffer;
 118 }
 119
 120
 121 /** @brief get http error
 122  *  @return http error
 123  */
 124 QHttp::Error HttpGet::error()
 125 {
 126     return http.error();
 127 }
 128
 129
 130 void HttpGet::setProxy(const QUrl &proxy)
 131 {
 132     m_proxy = proxy;
 133     m_useproxy = true;
 134     http.setProxy(m_proxy.host(), m_proxy.port(),
 135                   m_proxy.userName(), m_proxy.password());
 136 }
 137
 138
 139 void HttpGet::setProxy(bool enable)
 140 {
 141     if(enable) {
 142         m_useproxy = true;
 143         http.setProxy(m_proxy.host(), m_proxy.port(),
 144                       m_proxy.userName(), m_proxy.password());
 145     }
 146     else {
 147         m_useproxy = false;
 148         http.setProxy("", 0);
 149     }
 150 }
 151
 152
 153 void HttpGet::setFile(QFile *file)
 154 {
 155     outputFile = file;
 156     outputToBuffer = false;
 157 }
 158
 159
 160 void HttpGet::abort()
 161 {
 162     qDebug() << "[HTTP] Aborting requests, pending:" << http.hasPendingRequests();
 163     http.abort();
 164     if(!outputToBuffer)
 165         outputFile->close();
 166 }
 167
 168
 169 bool HttpGet::getFile(const QUrl &url)
 170 {
 171     if (!url.isValid()) {
 172         qDebug() << "[HTTP] Error: Invalid URL" << endl;
 173         return false;
 174     }
 175
 176     if (url.scheme() != "http") {
 177         qDebug() << "[HTTP] Error: URL must start with 'http:'" << endl;
 178         return false;
 179     }
 180
 181     if (url.path().isEmpty()) {
 182         qDebug() << "[HTTP] Error: URL has no path" << endl;
 183         return false;
 184     }
 185     m_serverTimestamp = QDateTime();
 186     // if no output file was set write to buffer
 187     if(!outputToBuffer) {
 188         if (!outputFile->open(QIODevice::ReadWrite)) {
 189             qDebug() << "[HTTP] Error: Cannot open " << qPrintable(outputFile->fileName())
 190                      << " for writing: " << qPrintable(outputFile->errorString());
 191             return false;
 192         }
 193     }
 194     else {
 195         // output to buffer. Make sure buffer is empty so no old data gets
 196         // returned in case the object is reused.
 197         dataBuffer.clear();
 198     }
 199     qDebug() << "[HTTP] GET URI" << url.toEncoded();
 200     // create request
 201     http.setHost(url.host(), url.port(80));
 202     // construct query (if any)
 203     QList<QPair<QString, QString> > qitems = url.queryItems();
 204     if(url.hasQuery()) {
 205         m_query = "?";
 206         for(int i = 0; i < qitems.size(); i++)
 207             m_query += QUrl::toPercentEncoding(qitems.at(i).first, "/") + "="
 208                   + QUrl::toPercentEncoding(qitems.at(i).second, "/") + "&";
 209     }
 210
 211     // create hash used for caching
 212     m_hash = QCryptographicHash::hash(url.toEncoded(), QCryptographicHash::Md5).toHex();
 213     // RFC2616: the absoluteURI form must get used when the request is being
 214     // sent to a proxy.
 215     m_path.clear();
 216     if(m_useproxy)
 217         m_path = url.scheme() + "://" + url.host();
 218     m_path += QString(QUrl::toPercentEncoding(url.path(), "/"));
 219
 220     // construct request header
 221     m_header.setValue("Host", url.host());
 222     m_header.setValue("User-Agent", m_globalUserAgent);
 223     m_header.setValue("Connection", "Keep-Alive");
 224
 225     if(m_dumbCache || !m_usecache) {
 226         getFileFinish();
 227     }
 228     else {
 229         // schedule HTTP header request
 230         m_header.setRequest("HEAD", m_path + m_query);
 231         headRequest = http.request(m_header);
 232         qDebug() << "[HTTP] HEAD scheduled:  " << headRequest;
 233     }
 234
 235     return true;
 236 }
 237
 238
 239 void HttpGet::getFileFinish()
 240 {
 241     m_cachefile = m_cachedir.absolutePath() + "/rbutil-cache/" + m_hash;
 242     QString indexFile = m_cachedir.absolutePath() + "/rbutil-cache/cache.txt";
 243     if(m_usecache) {
 244         // check if the file is present in cache
 245         QFileInfo cachefile = QFileInfo(m_cachefile);
 246         if(cachefile.isReadable()
 247             && cachefile.size() > 0
 248             && cachefile.lastModified() > m_serverTimestamp) {
 249
 250             qDebug() << "[HTTP] Cache: up-to-date file found:" << m_cachefile;
 251
 252             getRequest = -1;
 253             QFile c(m_cachefile);
 254             if(!outputToBuffer) {
 255                 qDebug() << "[HTTP] Cache: copying file to output"
 256                          << outputFile->fileName();
 257                 c.open(QIODevice::ReadOnly);
 258                 outputFile->open(QIODevice::ReadWrite);
 259                 outputFile->write(c.readAll());
 260                 outputFile->close();
 261                 c.close();
 262             }
 263             else {
 264                 qDebug() << "[HTTP] Cache: reading file into buffer";
 265                 c.open(QIODevice::ReadOnly);
 266                 dataBuffer = c.readAll();
 267                 c.close();
 268             }
 269             m_response = 200; // fake "200 OK" HTTP response
 270             m_cached = true;
 271             httpDone(false); // we're done now. Handle http "done" signal.
 272             return;
 273         }
 274         else {
 275             // unlink old cache file
 276             if(cachefile.isReadable()) {
 277                 QFile(m_cachefile).remove();
 278                 qDebug() << "[HTTP] Cache: outdated, timestamp:"
 279                          << cachefile.lastModified();
 280             }
 281             qDebug() << "[HTTP] Cache: caching as" << m_cachefile;
 282             // update cache index file
 283             QFile idxFile(indexFile);
 284             idxFile.open(QIODevice::ReadOnly);
 285             QByteArray currLine;
 286             do {
 287                 QByteArray currLine = idxFile.readLine(1000);
 288                 if(currLine.startsWith(m_hash.toUtf8()))
 289                     break;
 290             } while(!currLine.isEmpty());
 291             idxFile.close();
 292             if(currLine.isEmpty()) {
 293                 idxFile.open(QIODevice::Append);
 294                 QString outline = m_hash + "\t" + m_header.value("Host") + "\t"
 295                     + m_path + "\t" + m_query + "\n";
 296                 idxFile.write(outline.toUtf8());
 297                 idxFile.close();
 298             }
 299         }
 300
 301     }
 302     else {
 303         qDebug() << "[HTTP] cache DISABLED";
 304     }
 305     // schedule GET request
 306
 307     m_header.setRequest("GET", m_path + m_query);
 308     if(outputToBuffer) {
 309         qDebug() << "[HTTP] downloading to buffer.";
 310         getRequest = http.request(m_header);
 311     }
 312     else {
 313         qDebug() << "[HTTP] downloading to file:"
 314             << qPrintable(outputFile->fileName());
 315        getRequest = http.request(m_header, 0, outputFile);
 316     }
 317     qDebug() << "[HTTP] GET scheduled:   " << getRequest;
 318
 319     return;
 320 }
 321
 322
 323 void HttpGet::httpDone(bool error)
 324 {
 325     if (error) {
 326         qDebug() << "[HTTP] Error:" << qPrintable(http.errorString()) << httpResponse();
 327     }
 328     if(!outputToBuffer)
 329         outputFile->close();
 330
 331     if(m_usecache && !m_cached && !error) {
 332         qDebug() << "[HTTP] creating cache file" << m_cachefile;
 333         QFile c(m_cachefile);
 334         c.open(QIODevice::ReadWrite);
 335         if(!outputToBuffer) {
 336             outputFile->open(QIODevice::ReadOnly | QIODevice::Truncate);
 337             c.write(outputFile->readAll());
 338             outputFile->close();
 339         }
 340         else
 341             c.write(dataBuffer);
 342
 343         c.close();
 344     }
 345     // take care of concurring requests. If there is still one running,
 346     // don't emit done(). That request will call this slot again.
 347     if(http.currentId() == 0 && !http.hasPendingRequests())
 348         emit done(error);
 349 }
 350
 351
 352 void HttpGet::httpFinished(int id, bool error)
 353 {
 354     qDebug() << "[HTTP] Request finished:" << id << "Error:" << error
 355              << "pending requests:" << http.hasPendingRequests();
 356     if(id == getRequest) {
 357         dataBuffer = http.readAll();
 358         emit requestFinished(id, error);
 359     }
 360
 361     if(id == headRequest) {
 362         QHttpResponseHeader h = http.lastResponse();
 363
 364         QString date = h.value("Last-Modified").simplified();
 365         if(date.isEmpty()) {
 366             m_serverTimestamp = QDateTime(); // no value = invalid
 367             emit headerFinished();
 368             return;
 369         }
 370         // to successfully parse the date strip weekday and timezone
 371         date.remove(0, date.indexOf(" ") + 1);
 372         if(date.endsWith("GMT"))
 373             date.truncate(date.indexOf(" GMT"));
 374         // distinguish input formats (see RFC1945)
 375         // RFC 850
 376         if(date.contains("-"))
 377             m_serverTimestamp = QLocale::c().toDateTime(date, "dd-MMM-yy hh:mm:ss");
 378         // asctime format
 379         else if(date.at(0).isLetter())
 380             m_serverTimestamp = QLocale::c().toDateTime(date, "MMM d hh:mm:ss yyyy");
 381         // RFC 822
 382         else
 383             m_serverTimestamp = QLocale::c().toDateTime(date, "dd MMM yyyy hh:mm:ss");
 384         qDebug() << "[HTTP] HEAD finished, server date:" << date << ", parsed:" << m_serverTimestamp;
 385         emit headerFinished();
 386         return;
 387     }
 388     if(id == getRequest)
 389         emit requestFinished(id, error);
 390 }
 391
 392 void HttpGet::httpStarted(int id)
 393 {
 394     qDebug() << "[HTTP] Request started: " << id << "Header req:"
 395              << headRequest << "Get req:" << getRequest;
 396 }
 397
 398
 399 QString HttpGet::errorString()
 400 {
 401     return http.errorString();
 402 }
 403
 404
 405 void HttpGet::httpResponseHeader(const QHttpResponseHeader &resp)
 406 {
 407     // if there is a network error abort all scheduled requests for
 408     // this download
 409     m_response = resp.statusCode();
 410
 411     // 301 -- moved permanently
 412     // 302 -- found
 413     // 303 -- see other
 414     // 307 -- moved temporarily
 415     // in all cases, header: location has the correct address so we can follow.
 416     if(m_response == 301 || m_response == 302 || m_response == 303 || m_response == 307) {
 417         //abort without sending any signals
 418         http.blockSignals(true);
 419         http.abort();
 420         http.blockSignals(false);
 421         // start new request with new url
 422         qDebug() << "[HTTP] response =" << m_response << "- following";
 423         getFile(resp.value("location") + m_query);
 424     }
 425     else if(m_response != 200) {
 426         // all other errors are fatal.
 427         http.abort();
 428         qDebug() << "[HTTP] Response error:" << m_response << resp.reasonPhrase();
 429     }
 430
 431 }
 432
 433
 434 int HttpGet::httpResponse()
 435 {
 436     return m_response;
 437 }
 438
 439
 440 void HttpGet::httpState(int state)
 441 {
 442     QString s[] = {"Unconnected", "HostLookup", "Connecting", "Sending",
 443         "Reading", "Connected", "Closing"};
 444     if(state <= 6)
 445         qDebug() << "[HTTP] State:" << s[state];
 446     else qDebug() << "[HTTP] State:" << state;
 447 }
 448