mailimporter/filter_pmail.cpp

   1 /***************************************************************************
   2                           FilterPMail.cxx  -  Pegasus-Mail import
   3                              -------------------
   4     begin                : Sat Jan 6 2001
   5     copyright            : (C) 2001 by Holger Schurig <holgerschurig@gmx.de>
   6                            (C) 2005 by Danny Kukawka <danny.kukawka@web.de>
   7  ***************************************************************************/
   8
   9 /***************************************************************************
  10  *                                                                         *
  11  *   This program is free software; you can redistribute it and/or modify  *
  12  *   it under the terms of the GNU General Public License as published by  *
  13  *   the Free Software Foundation; either version 2 of the License, or     *
  14  *   (at your option) any later version.                                   *
  15  *                                                                         *
  16  ***************************************************************************/
  17
  18 #include <KLocalizedString>
  19 #include <kfiledialog.h>
  20 #include <QRegExp>
  21 #include <QPointer>
  22 #include <QTemporaryFile>
  23 #include <qdebug.h>
  24
  25 #include "filter_pmail.h"
  26
  27 using namespace MailImporter;
  28
  29 FilterPMail::FilterPMail() :
  30     Filter(i18n("Import Folders From Pegasus-Mail"),
  31            i18n("Holger Schurig <br>( rewritten by Danny Kukawka )"),
  32            i18n("<p>Select the Pegasus-Mail directory on your system (containing *.CNM, *.PMM and *.MBX files). "
  33                 "On many systems this is stored in C:\\pmail\\mail or C:\\pmail\\mail\\admin</p>"
  34                 "<p><b>Note:</b> Since it is possible to recreate the folder structure, the folders "
  35                 "will be stored under: \"PegasusMail-Import\".</p>"))
  36 {}
  37
  38 FilterPMail::~FilterPMail()
  39 {
  40 }
  41
  42 void FilterPMail::import()
  43 {
  44     // Select directory from where I have to import files
  45     QPointer<KFileDialog> kfd = new KFileDialog(QDir::homePath(), QString(), 0);
  46     kfd->setMode(KFile::Directory | KFile::LocalOnly);
  47     if (kfd->exec()) {
  48         const QString maildir = kfd->selectedFile();
  49
  50         importMails(maildir);
  51     }
  52     delete kfd;
  53 }
  54
  55 void FilterPMail::importMails(const QString  &chosenDir)
  56 {
  57     setMailDir(chosenDir);
  58     if (mailDir().isEmpty()) {
  59         filterInfo()->alert(i18n("No directory selected."));
  60         return;
  61     }
  62
  63     // Count total number of files to be processed
  64     filterInfo()->addInfoLogEntry(i18n("Counting files..."));
  65     dir.setPath(mailDir());
  66     const QStringList files = dir.entryList(QStringList() << QLatin1String("*.[cC][nN][mM]") << QLatin1String("*.[pP][mM][mM]") << QLatin1String("*.[mM][bB][xX]"), QDir::Files, QDir::Name);
  67     totalFiles = files.count();
  68     currentFile = 0;
  69     qDebug() << "Count is" << totalFiles;
  70
  71     if (!(folderParsed = parseFolderMatrix(mailDir()))) {
  72         filterInfo()->addErrorLogEntry(i18n("Cannot parse the folder structure; continuing import without subfolder support."));
  73     }
  74
  75     filterInfo()->addInfoLogEntry(i18n("Importing new mail files ('.cnm')..."));
  76     processFiles(QLatin1String("*.[cC][nN][mM]"), &FilterPMail::importNewMessage);
  77     filterInfo()->addInfoLogEntry(i18n("Importing mail folders ('.pmm')..."));
  78     processFiles(QLatin1String("*.[pP][mM][mM]"), &FilterPMail::importMailFolder);
  79     filterInfo()->addInfoLogEntry(i18n("Importing 'UNIX' mail folders ('.mbx')..."));
  80     processFiles(QLatin1String("*.[mM][bB][xX]"), &FilterPMail::importUnixMailFolder);
  81
  82     filterInfo()->addInfoLogEntry(i18n("Finished importing emails from %1", mailDir()));
  83     filterInfo()->setCurrent(100);
  84     filterInfo()->setOverall(100);
  85 }
  86
  87 /** this looks for all files with the filemask 'mask' and calls the 'workFunc' on each of them */
  88 void FilterPMail::processFiles(const QString &mask, void(FilterPMail::* workFunc)(const QString &))
  89 {
  90     if (filterInfo()->shouldTerminate()) {
  91         return;
  92     }
  93
  94     const QStringList files = dir.entryList(QStringList(mask), QDir::Files, QDir::Name);
  95     //qDebug() <<"Mask is" << mask <<" count is" << files.count();
  96     QStringList::ConstIterator end = files.constEnd();
  97     for (QStringList::ConstIterator mailFile = files.constBegin(); mailFile != end; ++mailFile) {
  98         // Notify current file
  99         QFileInfo mailfilem_filterInfoo(*mailFile);
 100         filterInfo()->setFrom(mailfilem_filterInfoo.fileName());
 101
 102         // Clear the other fields
 103         filterInfo()->setTo(QString());
 104         filterInfo()->setCurrent(QString());
 105         filterInfo()->setCurrent(-1);
 106
 107         // call worker function, increase progressbar
 108         (this->*workFunc)(dir.filePath(*mailFile));
 109         ++currentFile;
 110         filterInfo()->setOverall((int)((float) currentFile / totalFiles * 100));
 111         filterInfo()->setCurrent(100);
 112         if (filterInfo()->shouldTerminate()) {
 113             return;
 114         }
 115     }
 116 }
 117
 118 /** this function imports one *.CNM message */
 119 void FilterPMail::importNewMessage(const QString &file)
 120 {
 121     QString destFolder(QLatin1String("PegasusMail-Import/New Messages"));
 122     filterInfo()->setTo(destFolder);
 123
 124     /* comment by Danny Kukawka:
 125     * addMessage() == old function, need more time and check for duplicates
 126     * addMessage_fastImport == new function, faster and no check for duplicates
 127     */
 128     if (filterInfo()->removeDupMessage()) {
 129         addMessage(destFolder, file);
 130     } else {
 131         addMessage_fastImport(destFolder, file);
 132     }
 133 }
 134
 135 /** this function imports one mail folder file (*.PMM) */
 136 void FilterPMail::importMailFolder(const QString &file)
 137 {
 138     // Format of a PMM file:
 139     // First comes a header with 128 bytes. At the beginning is the name of
 140     // the folder. Then there are some unknown bytes (strings). At offset 128
 141     // the first message starts.
 142     //
 143     // Each message is terminated by a 0x1A byte. The next message follows
 144     // immediately.
 145     //
 146     // The last message is followed by a 0x1A, too.
 147     //
 148     // 000000 6d 61 69 6c 73 65 72 76 65 72 2d 70 72 6f 6a 65    mailserver-proje
 149     // 000010 63 74 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ct..............
 150     // 000020 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
 151     // 000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
 152     // 000040 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
 153     // 000050 00 00 00 00 00 00 36 30 34 37 35 37 32 45 3a 36    ......6047572E:6
 154     // 000060 46 34 39 3a 46 4f 4c 30 31 33 35 35 00 00 00 00    F49:FOL01355....
 155     // 000070 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
 156     // 000080 52 65 74 75 72 6e 2d 50 61 74 68 3a 20 3c 75 72    Return-Path: <ur
 157     // ...
 158     // 000cb0 2d 2d 2d 2d 2d 2d 2d 2d 2d 2b 0d 0a 1a 52 65 74    ---------+...Ret
 159     // 000cc0 75 72 6e 2d 50 61 74 68 3a 20 3c 62 6f 75 6e 63    urn-Path: <bounc
 160     // ...
 161     // 04dc50 46 30 33 38 44 2e 36 31 35 44 37 34 44 30 2d 2d    F038D.615D74D0--
 162     // 04dc60 0d 0a 1a
 163
 164     struct {
 165         char folder[86];
 166         char id[42];
 167     } pmm_head;
 168
 169     long l = 0;
 170     QFile f(file);
 171     if (!f.open(QIODevice::ReadOnly)) {
 172         filterInfo()->alert(i18n("Unable to open %1, skipping", file));
 173     } else {
 174         // Get folder name
 175         l = f.read((char *) &pmm_head, sizeof(pmm_head));
 176         QString folder(i18nc("define folder name when we will import pegasus mail", "PegasusMail-Import") + QLatin1Char('/'));
 177         if (folderParsed) {
 178             folder.append(getFolderName(QString::fromLatin1(pmm_head.id)));
 179         } else {
 180             folder.append(QString::fromLatin1(pmm_head.folder));
 181         }
 182         filterInfo()->setTo(folder);
 183         filterInfo()->addInfoLogEntry(i18n("Importing %1", QString::fromLatin1("../") + QString::fromLatin1(pmm_head.folder)));
 184
 185         QByteArray input(MAX_LINE, '\0');
 186         bool first_msg = true;
 187
 188         while (!f.atEnd()) {
 189             QTemporaryFile tempfile;
 190             tempfile.open();
 191             filterInfo()->setCurrent((int)(((float) f.pos() / f.size()) * 100));
 192
 193             if (!first_msg) {
 194                 // set the filepos back to last line minus the separate char (0x1a)
 195                 f.seek(f.pos() - l + 1);
 196             }
 197
 198             // no problem to loose the last line in file. This only contains a separate char
 199             while (! f.atEnd() && (l = f.readLine(input.data(), MAX_LINE))) {
 200                 if (filterInfo()->shouldTerminate()) {
 201                     return;
 202                 }
 203                 if (input.at(0) == 0x1a) {
 204                     break;
 205                 } else {
 206                     tempfile.write(input, l);
 207                 }
 208             }
 209             tempfile.flush();
 210
 211             if (filterInfo()->removeDupMessage()) {
 212                 addMessage(folder, tempfile.fileName());
 213             } else {
 214                 addMessage_fastImport(folder, tempfile.fileName());
 215             }
 216
 217             first_msg = false;
 218         }
 219     }
 220     f.close();
 221 }
 222
 223 /** imports a 'unix' format mail folder (*.MBX) */
 224 void FilterPMail::importUnixMailFolder(const QString &file)
 225 {
 226     struct {
 227         char folder[58];
 228         char id[31];
 229     } pmg_head;
 230
 231     QFile f;
 232     QString folder(QLatin1String("PegasusMail-Import/")), s(file), separate;
 233     QByteArray line(MAX_LINE, '\0');
 234     int n = 0, l = 0;
 235
 236     /** Get the folder name */
 237     s.replace(QRegExp(QLatin1String("mbx$")), QLatin1String("pmg"));
 238     s.replace(QRegExp(QLatin1String("MBX$")), QLatin1String("PMG"));
 239     f.setFileName(s);
 240     if (! f.open(QIODevice::ReadOnly)) {
 241         filterInfo()->alert(i18n("Unable to open %1, skipping", s));
 242         return;
 243     } else {
 244         f.read((char *) &pmg_head, sizeof(pmg_head));
 245         f.close();
 246
 247         if (folderParsed) {
 248             folder.append(getFolderName(QString::fromLatin1(pmg_head.id)));
 249         } else {
 250             folder.append(QString::fromLatin1(pmg_head.folder));
 251         }
 252
 253         filterInfo()->setTo(folder);
 254         filterInfo()->setTo(folder);
 255     }
 256
 257     /** Read in the mbox */
 258     f.setFileName(file);
 259     if (! f.open(QIODevice::ReadOnly)) {
 260         filterInfo()->alert(i18n("Unable to open %1, skipping", s));
 261     } else {
 262         filterInfo()->addInfoLogEntry(i18n("Importing %1", QLatin1String("../") + QString::fromLatin1(pmg_head.folder)));
 263         l = f.readLine(line.data(), MAX_LINE); // read the first line which is unneeded
 264         while (! f.atEnd()) {
 265             QTemporaryFile tempfile;
 266             tempfile.open();
 267
 268             // we lost the last line, which is the first line of the new message in
 269             // this lopp, but this is ok, because this is the separate line with
 270             // "From ???@???" and we can forget them
 271             while (! f.atEnd() && (l = f.readLine(line.data(), MAX_LINE)) && ((separate = QString::fromLatin1(line.data())).left(5) != QLatin1String("From "))) {
 272                 tempfile.write(line.data(), l);
 273                 if (filterInfo()->shouldTerminate()) {
 274                     return;
 275                 }
 276             }
 277             tempfile.flush();
 278             if (filterInfo()->removeDupMessage()) {
 279                 addMessage(folder, tempfile.fileName());
 280             } else {
 281                 addMessage_fastImport(folder, tempfile.fileName());
 282             }
 283
 284             n++;
 285             filterInfo()->setCurrent(i18n("Message %1", n));
 286             filterInfo()->setCurrent((int)(((float) f.pos() / f.size()) * 100));
 287         }
 288     }
 289     f.close();
 290 }
 291
 292 /** Parse the m_filterInfoormation about folderstructure to folderMatrix */
 293 bool FilterPMail::parseFolderMatrix(const QString  &chosendir)
 294 {
 295     qDebug() << "Start parsing the foldermatrix.";
 296     filterInfo()->addInfoLogEntry(i18n("Parsing the folder structure..."));
 297
 298     QFile hierarch(chosendir + QLatin1String("/hierarch.pm"));
 299     if (! hierarch.open(QIODevice::ReadOnly)) {
 300         filterInfo()->alert(i18n("Unable to open %1, skipping", chosendir + QLatin1String("hierarch.pm")));
 301         return false;
 302     } else {
 303         QStringList tmpList;
 304         QByteArray tmpRead;
 305         while (!hierarch.atEnd()) {
 306             tmpRead = hierarch.readLine();
 307             if (tmpRead.isEmpty()) {
 308                 break;
 309             }
 310             QString tmpArray[5];
 311             tmpRead.remove(tmpRead.length() - 2, 2);
 312             QStringList tmpList = QString::fromLatin1(tmpRead).split(QLatin1Char(','), QString::SkipEmptyParts);
 313             int i = 0;
 314             QStringList::ConstIterator end(tmpList.constEnd());
 315             for (QStringList::ConstIterator it = tmpList.constBegin(); it != end; ++it, ++i) {
 316                 QString _tmp = *it;
 317                 if (i < 5) {
 318                     tmpArray[i] = _tmp.remove(QLatin1Char('\"'));
 319                 } else {
 320                     hierarch.close();
 321                     return false;
 322                 }
 323             }
 324             folderMatrix.append(FolderStructure(tmpArray));
 325         }
 326     }
 327     hierarch.close();
 328     return true;
 329 }
 330
 331 /** get the foldername for a given file ID from folderMatrix */
 332 QString FilterPMail::getFolderName(const QString &ID)
 333 {
 334     bool found = false;
 335     QString folder;
 336     QString search = ID;
 337
 338     while (!found) {
 339         FolderStructureIterator end(folderMatrix.end());
 340         for (FolderStructureIterator it = folderMatrix.begin(); it != end; ++it) {
 341             FolderStructure tmp = *it;
 342
 343             QString _ID = tmp[2];
 344             if (_ID == search) {
 345                 QString _type = tmp[0] + tmp[1];
 346                 if ((_type == QLatin1String("21"))) {
 347                     found = true;
 348                     break;
 349                 } else {
 350                     folder.prepend((tmp[4] + QLatin1Char('/')));
 351                     search = tmp[3];
 352                 }
 353             }
 354         }
 355     }
 356     return folder;
 357 }