kget/core/linkimporter.cpp

   1 /* This file is part of the KDE project
   2
   3    Copyright (C) 2008 Javier Goday <jgoday @ gmail.com>
   4    First Url regular expression taken from urlview tool by Michael Elkins <me@cs.hmc.edu>.
   5    Regular expression improved by FiNex.
   6    Improvements to regular expression and slotReadFile by Frantisek Ziacik
   7
   8    This program is free software; you can redistribute it and/or
   9    modify it under the terms of the GNU General Public
  10    License as published by the Free Software Foundation; either
  11    version 2 of the License, or (at your option) any later version.
  12 */
  13 #include "linkimporter.h"
  14
  15 #include <QFile>
  16 #include <QIODevice>
  17 #include <QList>
  18 #include <QMap>
  19 #include <QRegExp>
  20 #include <QDir>
  21 #include <QTextStream>
  22 #include <QStringList>
  23
  24 #include <KDebug>
  25 #include <KLocale>
  26 #include <kio/copyjob.h>
  27 #include <kio/netaccess.h>
  28
  29 //static QString REGULAR_EXPRESSION = "(((https?|ftp|gopher)://|(mailto|file|news):)[^’ <>\"]+|(www|web|w3).[-a-z0-9.]+)[^’ .,;<>\":]";
  30 // static QString REGULAR_EXPRESSION = "((http|https|ftp|ftps)+([\\:\\w\\d:#@%/;$()~_?\\+-=\\\\.&])*)";
  31 static QString REGULAR_EXPRESSION = "(\\w+[:]//)?(((([\\w-]+[.]){1,}(ac|ad|ae|af|ag|ai|al|am|an|ao|aq|ar|as|at|au|aw|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|com|cr|cs|cu|cv|cx|cy|cz|de|dj|dk|dm|do|dz|ec|edu|ee|eg|eh|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gd|ge|gf|gg|gh|gi|gl|gm|gn|gov|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|int|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|mg|mh|mil|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|net|nf|ng|ni|nl|no|np|nr|nt|nu|nz|om|org|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|sv|st|sy|sz|tc|td|tf|tg|th|tj|tk|tm|tn|to|tp|tr|tt|tv|tw|tz|ua|ug|uk|um|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|yu|za|zm|zw|aero|biz|coop|info|museum|name|pro|travel))|([0-9]+[.][0-9]+[.][0-9]+[.][0-9]+)))([:][0-9]*)?([?/][\\w~#\\-;%?@&=/.+]*)?(?!\\w)";
  32
  33 LinkImporter::LinkImporter(const KUrl &url, QObject *parent) : QThread(parent),
  34     m_url(url),
  35     m_transfers(),
  36     m_tempFile()
  37 {
  38 }
  39
  40 LinkImporter::LinkImporter(QObject *parent) : QThread(parent),
  41     m_url(),
  42     m_transfers(),
  43     m_tempFile()
  44 {
  45 }
  46
  47 LinkImporter::~LinkImporter()
  48 {
  49 }
  50
  51 void LinkImporter::checkClipboard(const QString &clipboardContent)
  52 {
  53     QRegExp rx(REGULAR_EXPRESSION);
  54
  55     int regexPos = 0;
  56
  57     while ((regexPos = rx.indexIn(clipboardContent, regexPos)) > -1) {
  58         QString link = rx.capturedTexts()[0];
  59
  60         addTransfer(link);
  61
  62         regexPos += rx.matchedLength();
  63     }
  64
  65     emit finished();
  66 }
  67
  68 void LinkImporter::run()
  69 {
  70     if(!m_url.isLocalFile() && !m_tempFile.isEmpty()) {
  71         slotReadFile(KUrl(m_tempFile));
  72     }
  73     else {
  74         slotReadFile(m_url);
  75     }
  76
  77     emit finished();
  78 }
  79
  80 void LinkImporter::copyRemoteFile()
  81 {
  82     m_tempFile = QString("%1/%2.tmp").arg(QDir::tempPath()).arg("importer_aux");
  83
  84     KUrl aux(m_tempFile);
  85     KIO::CopyJob *job = KIO::copy(m_url, aux, KIO::HideProgressInfo);
  86
  87     QMap<QString, QString> metaData;
  88     bool ok = KIO::NetAccess::synchronousRun(job, 0, 0, 0, &metaData);
  89     if(!ok) {
  90         emit error(ki18n("Error trying to get %1").subs(m_url.url()));
  91     }
  92 }
  93
  94 void LinkImporter::slotReadFile(const QUrl &url)
  95 {
  96     QRegExp rx(REGULAR_EXPRESSION);
  97     QFile file(url.path());
  98
  99     if (!file.open(QIODevice::ReadOnly | QIODevice::Text))
 100         return;
 101
 102     QTextStream in(&file);
 103     quint64 size = file.size();
 104     quint64 position = 0;
 105
 106     while (!in.atEnd()) {
 107         QString line = in.readLine();
 108         int regexPos = 0;
 109         quint64 lastPosition = position;
 110
 111         while ((regexPos = rx.indexIn(line, regexPos)) > -1) {
 112             QString link = rx.capturedTexts()[0];
 113
 114             addTransfer(link);
 115
 116             regexPos += rx.matchedLength();
 117             position = lastPosition + regexPos;
 118
 119             emit progress(position * 100 / size);
 120         }
 121
 122         position += line.size();
 123
 124         emit progress(position * 100 / size);
 125     }
 126
 127     if(!m_url.isLocalFile()) {
 128         file.remove();
 129     }
 130 }
 131
 132 void LinkImporter::addTransfer(QString &link)
 133 {
 134     KUrl auxUrl;
 135
 136     if (link.contains("://")) {
 137         auxUrl = KUrl(link);
 138     } else {
 139         auxUrl = KUrl(QString("http://") + link);
 140     }
 141
 142     if(!link.isEmpty() && auxUrl.isValid() && m_transfers.indexOf(link) < 0 &&
 143        !auxUrl.scheme().isEmpty() && !auxUrl.host().isEmpty()) {
 144         m_transfers << link;
 145     }
 146 }
 147
 148 #include "linkimporter.moc"