mailimporter/filter_pmail.cpp

   1 /***************************************************************************
   2                           FilterPMail.cxx  -  Pegasus-Mail import
   3                              -------------------
   4     begin                : Sat Jan 6 2001
   5     copyright            : (C) 2001 by Holger Schurig <holgerschurig@gmx.de>
   6                            (C) 2005 by Danny Kukawka <danny.kukawka@web.de>
   7  ***************************************************************************/
   8
   9 /***************************************************************************
  10  *                                                                         *
  11  *   This program is free software; you can redistribute it and/or modify  *
  12  *   it under the terms of the GNU General Public License as published by  *
  13  *   the Free Software Foundation; either version 2 of the License, or     *
  14  *   (at your option) any later version.                                   *
  15  *                                                                         *
  16  ***************************************************************************/
  17
  18 #include <klocale.h>
  19 #include <kfiledialog.h>
  20 #include <QRegExp>
  21 #include <QPointer>
  22 #include <ktemporaryfile.h>
  23 #include <kdebug.h>
  24
  25 #include "filter_pmail.h"
  26
  27 using namespace MailImporter;
  28
  29 FilterPMail::FilterPMail() :
  30     Filter(i18n("Import Folders From Pegasus-Mail"),
  31            i18n("Holger Schurig <br>( rewritten by Danny Kukawka )"),
  32            i18n("<p>Select the Pegasus-Mail directory on your system (containing *.CNM, *.PMM and *.MBX files). "
  33                 "On many systems this is stored in C:\\pmail\\mail or C:\\pmail\\mail\\admin</p>"
  34                 "<p><b>Note:</b> Since it is possible to recreate the folder structure, the folders "
  35                 "will be stored under: \"PegasusMail-Import\".</p>"))
  36 {}
  37
  38 FilterPMail::~FilterPMail()
  39 {
  40 }
  41
  42 void FilterPMail::import()
  43 {
  44     // Select directory from where I have to import files
  45     QPointer<KFileDialog> kfd = new KFileDialog( QDir::homePath(), QString(), 0 );
  46     kfd->setMode(KFile::Directory | KFile::LocalOnly);
  47     if (kfd->exec()) {
  48         const QString maildir = kfd->selectedFile();
  49
  50         importMails( maildir );
  51     }
  52     delete kfd;
  53 }
  54
  55 void FilterPMail::importMails( const QString  &chosenDir )
  56 {
  57     setMailDir( chosenDir );
  58     if (mailDir().isEmpty()) {
  59         filterInfo()->alert(i18n("No directory selected."));
  60         return;
  61     }
  62
  63     // Count total number of files to be processed
  64     filterInfo()->addInfoLogEntry(i18n("Counting files..."));
  65     dir.setPath (mailDir());
  66     const QStringList files = dir.entryList(QStringList()<<QLatin1String("*.[cC][nN][mM]")<<QLatin1String("*.[pP][mM][mM]")<<QLatin1String("*.[mM][bB][xX]"), QDir::Files, QDir::Name);
  67     totalFiles = files.count();
  68     currentFile = 0;
  69     kDebug() <<"Count is" << totalFiles;
  70
  71     if(!(folderParsed = parseFolderMatrix(mailDir()))) {
  72         filterInfo()->addErrorLogEntry(i18n("Cannot parse the folder structure; continuing import without subfolder support."));
  73     }
  74
  75     filterInfo()->addInfoLogEntry(i18n("Importing new mail files ('.cnm')..."));
  76     processFiles(QLatin1String("*.[cC][nN][mM]"), &FilterPMail::importNewMessage);
  77     filterInfo()->addInfoLogEntry(i18n("Importing mail folders ('.pmm')..."));
  78     processFiles(QLatin1String("*.[pP][mM][mM]"), &FilterPMail::importMailFolder);
  79     filterInfo()->addInfoLogEntry(i18n("Importing 'UNIX' mail folders ('.mbx')..."));
  80     processFiles(QLatin1String("*.[mM][bB][xX]"), &FilterPMail::importUnixMailFolder);
  81
  82     filterInfo()->addInfoLogEntry( i18n("Finished importing emails from %1", mailDir() ));
  83     filterInfo()->setCurrent(100);
  84     filterInfo()->setOverall(100);
  85 }
  86
  87 /** this looks for all files with the filemask 'mask' and calls the 'workFunc' on each of them */
  88 void FilterPMail::processFiles(const QString &mask, void(FilterPMail::* workFunc)(const QString&) )
  89 {
  90     if (filterInfo()->shouldTerminate())
  91         return;
  92
  93     const QStringList files = dir.entryList(QStringList(mask), QDir::Files, QDir::Name);
  94     //kDebug() <<"Mask is" << mask <<" count is" << files.count();
  95     QStringList::ConstIterator end = files.constEnd();
  96     for ( QStringList::ConstIterator mailFile = files.constBegin(); mailFile != end; ++mailFile ) {
  97         // Notify current file
  98         QFileInfo mailfilem_filterInfoo(*mailFile);
  99         filterInfo()->setFrom(mailfilem_filterInfoo.fileName());
 100
 101         // Clear the other fields
 102         filterInfo()->setTo(QString());
 103         filterInfo()->setCurrent(QString());
 104         filterInfo()->setCurrent(-1);
 105
 106         // call worker function, increase progressbar
 107         (this->*workFunc)(dir.filePath(*mailFile));
 108         ++currentFile;
 109         filterInfo()->setOverall( (int) ((float) currentFile / totalFiles * 100));
 110         filterInfo()->setCurrent( 100 );
 111         if (filterInfo()->shouldTerminate()) return;
 112     }
 113 }
 114
 115
 116 /** this function imports one *.CNM message */
 117 void FilterPMail::importNewMessage(const QString &file)
 118 {
 119     QString destFolder(QLatin1String("PegasusMail-Import/New Messages"));
 120     filterInfo()->setTo(destFolder);
 121
 122     /* comment by Danny Kukawka:
 123    * addMessage() == old function, need more time and check for duplicates
 124    * addMessage_fastImport == new function, faster and no check for duplicates
 125    */
 126     if(filterInfo()->removeDupMessage())
 127         addMessage( destFolder, file );
 128     else
 129         addMessage_fastImport( destFolder, file );
 130 }
 131
 132
 133 /** this function imports one mail folder file (*.PMM) */
 134 void FilterPMail::importMailFolder(const QString &file)
 135 {
 136     // Format of a PMM file:
 137     // First comes a header with 128 bytes. At the beginning is the name of
 138     // the folder. Then there are some unknown bytes (strings). At offset 128
 139     // the first message starts.
 140     //
 141     // Each message is terminated by a 0x1A byte. The next message follows
 142     // immediately.
 143     //
 144     // The last message is followed by a 0x1A, too.
 145     //
 146     // 000000 6d 61 69 6c 73 65 72 76 65 72 2d 70 72 6f 6a 65    mailserver-proje
 147     // 000010 63 74 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ct..............
 148     // 000020 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
 149     // 000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
 150     // 000040 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
 151     // 000050 00 00 00 00 00 00 36 30 34 37 35 37 32 45 3a 36    ......6047572E:6
 152     // 000060 46 34 39 3a 46 4f 4c 30 31 33 35 35 00 00 00 00    F49:FOL01355....
 153     // 000070 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
 154     // 000080 52 65 74 75 72 6e 2d 50 61 74 68 3a 20 3c 75 72    Return-Path: <ur
 155     // ...
 156     // 000cb0 2d 2d 2d 2d 2d 2d 2d 2d 2d 2b 0d 0a 1a 52 65 74    ---------+...Ret
 157     // 000cc0 75 72 6e 2d 50 61 74 68 3a 20 3c 62 6f 75 6e 63    urn-Path: <bounc
 158     // ...
 159     // 04dc50 46 30 33 38 44 2e 36 31 35 44 37 34 44 30 2d 2d    F038D.615D74D0--
 160     // 04dc60 0d 0a 1a
 161
 162     struct {
 163         char folder[86];
 164         char id[42];
 165     } pmm_head;
 166
 167     long l = 0;
 168     QFile f(file);
 169     if (!f.open(QIODevice::ReadOnly)) {
 170         filterInfo()->alert(i18n("Unable to open %1, skipping", file));
 171     } else {
 172         // Get folder name
 173         l = f.read((char *) &pmm_head, sizeof(pmm_head));
 174         QString folder(i18nc("define folder name when we will import pegasus mail","PegasusMail-Import") + QLatin1Char('/'));
 175         if(folderParsed)
 176             folder.append(getFolderName(QString::fromLatin1(pmm_head.id)));
 177         else
 178             folder.append(QString::fromLatin1(pmm_head.folder));
 179         filterInfo()->setTo(folder);
 180         filterInfo()->addInfoLogEntry(i18n("Importing %1", QString::fromLatin1("../") + QString::fromLatin1(pmm_head.folder)));
 181
 182         QByteArray input(MAX_LINE,'\0');
 183         bool first_msg = true;
 184
 185         while (!f.atEnd()) {
 186             KTemporaryFile tempfile;
 187             tempfile.open();
 188             filterInfo()->setCurrent( (int) ( ( (float) f.pos() / f.size() ) * 100 ) );
 189
 190             if(!first_msg) {
 191                 // set the filepos back to last line minus the separate char (0x1a)
 192                 f.seek(f.pos() - l + 1);
 193             }
 194
 195             // no problem to loose the last line in file. This only contains a separate char
 196             while ( ! f.atEnd() && (l = f.readLine(input.data(),MAX_LINE))) {
 197                 if (filterInfo()->shouldTerminate()){
 198                     return;
 199                 }
 200                 if ( input.at( 0 ) == 0x1a ) {
 201                     break;
 202                 } else {
 203                     tempfile.write( input, l );
 204                 }
 205             }
 206             tempfile.flush();
 207
 208             if(filterInfo()->removeDupMessage())
 209                 addMessage( folder, tempfile.fileName() );
 210             else
 211                 addMessage_fastImport( folder, tempfile.fileName() );
 212
 213             first_msg = false;
 214         }
 215     }
 216     f.close();
 217 }
 218
 219
 220 /** imports a 'unix' format mail folder (*.MBX) */
 221 void FilterPMail::importUnixMailFolder(const QString &file)
 222 {
 223     struct {
 224         char folder[58];
 225         char id[31];
 226     } pmg_head;
 227
 228     QFile f;
 229     QString folder(QLatin1String("PegasusMail-Import/")), s(file), separate;
 230     QByteArray line(MAX_LINE,'\0');
 231     int n = 0, l = 0;
 232
 233     /** Get the folder name */
 234     s.replace( QRegExp(QLatin1String("mbx$")), QLatin1String("pmg"));
 235     s.replace( QRegExp(QLatin1String("MBX$")), QLatin1String("PMG"));
 236     f.setFileName(s);
 237     if (! f.open( QIODevice::ReadOnly ) ) {
 238         filterInfo()->alert( i18n("Unable to open %1, skipping", s ) );
 239         return;
 240     } else {
 241         f.read((char *) &pmg_head, sizeof(pmg_head));
 242         f.close();
 243
 244         if(folderParsed)
 245             folder.append(getFolderName(QString::fromLatin1(pmg_head.id)));
 246         else
 247             folder.append(QString::fromLatin1(pmg_head.folder));
 248
 249         filterInfo()->setTo(folder);
 250         filterInfo()->setTo(folder);
 251     }
 252
 253     /** Read in the mbox */
 254     f.setFileName(file);
 255     if (! f.open( QIODevice::ReadOnly ) ) {
 256         filterInfo()->alert( i18n("Unable to open %1, skipping", s ) );
 257     } else {
 258         filterInfo()->addInfoLogEntry(i18n("Importing %1", QLatin1String("../") + QString::fromLatin1(pmg_head.folder)));
 259         l = f.readLine( line.data(),MAX_LINE); // read the first line which is unneeded
 260         while ( ! f.atEnd() ) {
 261             KTemporaryFile tempfile;
 262             tempfile.open();
 263
 264             // we lost the last line, which is the first line of the new message in
 265             // this lopp, but this is ok, because this is the separate line with
 266             // "From ???@???" and we can forget them
 267             while ( ! f.atEnd() && (l = f.readLine(line.data(),MAX_LINE)) &&((separate = QString::fromLatin1(line.data())).left(5) != QLatin1String("From "))) {
 268                 tempfile.write(line.data(), l);
 269                 if (filterInfo()->shouldTerminate()){
 270                     return;
 271                 }
 272             }
 273             tempfile.flush();
 274             if(filterInfo()->removeDupMessage())
 275                 addMessage( folder, tempfile.fileName() );
 276             else
 277                 addMessage_fastImport( folder, tempfile.fileName() );
 278
 279             n++;
 280             filterInfo()->setCurrent(i18n("Message %1", n));
 281             filterInfo()->setCurrent( (int) ( ( (float) f.pos() / f.size() ) * 100 ) );
 282         }
 283     }
 284     f.close();
 285 }
 286
 287 /** Parse the m_filterInfoormation about folderstructure to folderMatrix */
 288 bool FilterPMail::parseFolderMatrix( const QString  &chosendir )
 289 {
 290     kDebug() <<"Start parsing the foldermatrix.";
 291     filterInfo()->addInfoLogEntry(i18n("Parsing the folder structure..."));
 292
 293     QFile hierarch(chosendir + QLatin1String("/hierarch.pm"));
 294     if (! hierarch.open( QIODevice::ReadOnly ) ) {
 295         filterInfo()->alert( i18n("Unable to open %1, skipping",chosendir + QLatin1String("hierarch.pm") ) );
 296         return false;
 297     } else {
 298         QStringList tmpList;
 299         QByteArray tmpRead;
 300         while ( !hierarch.atEnd() ) {
 301             tmpRead = hierarch.readLine();
 302             if ( tmpRead.isEmpty() )
 303                 break;
 304             QString tmpArray[5];
 305             tmpRead.remove(tmpRead.length() -2,2);
 306             QStringList tmpList = QString::fromLatin1(tmpRead).split(QLatin1Char(','), QString::SkipEmptyParts);
 307             int i = 0;
 308             QStringList::ConstIterator end( tmpList.constEnd() );
 309             for ( QStringList::ConstIterator it = tmpList.constBegin(); it != end; ++it, ++i) {
 310                 QString _tmp = *it;
 311                 if(i < 5) tmpArray[i] = _tmp.remove(QLatin1Char('\"'));
 312                 else {
 313                     hierarch.close();
 314                     return false;
 315                 }
 316             }
 317             folderMatrix.append(FolderStructure(tmpArray));
 318         }
 319     }
 320     hierarch.close();
 321     return true;
 322 }
 323
 324 /** get the foldername for a given file ID from folderMatrix */
 325 QString FilterPMail::getFolderName(const QString &ID)
 326 {
 327     bool found = false;
 328     QString folder;
 329     QString search = ID;
 330
 331     while (!found)
 332     {
 333         FolderStructureIterator end( folderMatrix.end() );
 334         for ( FolderStructureIterator it = folderMatrix.begin(); it != end; ++it) {
 335             FolderStructure tmp = *it;
 336
 337             QString _ID = tmp[2];
 338             if(_ID == search) {
 339                 QString _type = tmp[0] + tmp[1];
 340                 if(( _type == QLatin1String("21"))) {
 341                     found = true;
 342                     break;
 343                 }
 344                 else {
 345                     folder.prepend((tmp[4] + QLatin1Char('/')));
 346                     search = tmp[3];
 347                 }
 348             }
 349         }
 350     }
 351     return folder;
 352 }