kmailcvt/filter_pmail.cxx

   1 /***************************************************************************
   2                           FilterPMail.cxx  -  Pegasus-Mail import
   3                              -------------------
   4     begin                : Sat Jan 6 2001
   5     copyright            : (C) 2001 by Holger Schurig <holgerschurig@gmx.de>
   6                            (C) 2005 by Danny Kukawka <danny.kukawka@web.de>
   7  ***************************************************************************/
   8
   9 /***************************************************************************
  10  *                                                                         *
  11  *   This program is free software; you can redistribute it and/or modify  *
  12  *   it under the terms of the GNU General Public License as published by  *
  13  *   the Free Software Foundation; either version 2 of the License, or     *
  14  *   (at your option) any later version.                                   *
  15  *                                                                         *
  16  ***************************************************************************/
  17
  18 #include <klocale.h>
  19 #include <kfiledialog.h>
  20 #include <QRegExp>
  21 #include <ktemporaryfile.h>
  22 #include <kdebug.h>
  23
  24 #include "filter_pmail.hxx"
  25
  26
  27 FilterPMail::FilterPMail() :
  28         Filter(i18n("Import Folders From Pegasus-Mail"),
  29                "Holger Schurig <br>( rewritten by Danny Kukawka )",
  30                i18n("<p>Select the Pegasus-Mail directory on your system (containing *.CNM, *.PMM and *.MBX files). "
  31                     "On many systems this is stored in C:\\pmail\\mail or C:\\pmail\\mail\\admin</p>"
  32                     "<p><b>Note:</b> Since it is possible to recreate the folder structure, the folders "
  33                     "will be stored under: \"PegasusMail-Import\".</p>"))
  34 {}
  35
  36 FilterPMail::~FilterPMail()
  37 {
  38 }
  39
  40 void FilterPMail::import(FilterInfo *info)
  41 {
  42     inf = info;
  43
  44     // Select directory from where I have to import files
  45     KFileDialog *kfd;
  46     kfd = new KFileDialog( QDir::homePath(), "", 0 );
  47     kfd->setMode(KFile::Directory | KFile::LocalOnly);
  48     kfd->exec();
  49     chosenDir = kfd->selectedFile();
  50     delete kfd;
  51     if (chosenDir.isEmpty()) {
  52         info->alert(i18n("No directory selected."));
  53         return;
  54     }
  55
  56     // Count total number of files to be processed
  57     info->addLog(i18n("Counting files..."));
  58     dir.setPath (chosenDir);
  59     const QStringList files = dir.entryList(QStringList("*.[cC][nN][mM]")<<"*.[pP][mM][mM]"<<"*.[mM][bB][xX]", QDir::Files, QDir::Name);
  60     totalFiles = files.count();
  61     currentFile = 0;
  62     kDebug() <<"Count is" << totalFiles;
  63
  64     if(!(folderParsed = parseFolderMatrix())) {
  65         info->addLog(i18n("Cannot parse the folder structure; continuing import without subfolder support."));
  66     }
  67
  68     info->addLog(i18n("Importing new mail files ('.cnm')..."));
  69     processFiles("*.[cC][nN][mM]", &FilterPMail::importNewMessage);
  70     info->addLog(i18n("Importing mail folders ('.pmm')..."));
  71     processFiles("*.[pP][mM][mM]", &FilterPMail::importMailFolder);
  72     info->addLog(i18n("Importing 'UNIX' mail folders ('.mbx')..."));
  73     processFiles("*.[mM][bB][xX]", &FilterPMail::importUnixMailFolder);
  74
  75     info->addLog( i18n("Finished importing emails from %1", chosenDir ));
  76     info->setCurrent(100);
  77     info->setOverall(100);
  78     delete kfd;
  79 }
  80
  81 /** this looks for all files with the filemask 'mask' and calls the 'workFunc' on each of them */
  82 void FilterPMail::processFiles(const QString& mask, void(FilterPMail::* workFunc)(const QString&) )
  83 {
  84     if (inf->shouldTerminate()) return;
  85
  86     const QStringList files = dir.entryList(QStringList(mask), QDir::Files, QDir::Name);
  87     //kDebug() <<"Mask is" << mask <<" count is" << files.count();
  88     for ( QStringList::ConstIterator mailFile = files.constBegin(); mailFile != files.constEnd(); ++mailFile ) {
  89         // Notify current file
  90         QFileInfo mailfileinfo(*mailFile);
  91         inf->setFrom(mailfileinfo.fileName());
  92
  93         // Clear the other fields
  94         inf->setTo(QString());
  95         inf->setCurrent(QString());
  96         inf->setCurrent(-1);
  97
  98         // call worker function, increase progressbar
  99         (this->*workFunc)(dir.filePath(*mailFile));
 100         ++currentFile;
 101         inf->setOverall( (int) ((float) currentFile / totalFiles * 100));
 102         inf->setCurrent( 100 );
 103         if (inf->shouldTerminate()) return;
 104     }
 105 }
 106
 107
 108 /** this function imports one *.CNM message */
 109 void FilterPMail::importNewMessage(const QString& file)
 110 {
 111     QString destFolder("PegasusMail-Import/New Messages");
 112     inf->setTo(destFolder);
 113
 114     /* comment by Danny Kukawka:
 115      * addMessage() == old function, need more time and check for duplicates
 116      * addMessage_fastImport == new function, faster and no check for duplicates
 117      */
 118     if(inf->removeDupMsg)
 119         addMessage( inf, destFolder, file );
 120     else
 121         addMessage_fastImport( inf, destFolder, file );
 122 }
 123
 124
 125 /** this function imports one mail folder file (*.PMM) */
 126 void FilterPMail::importMailFolder(const QString& file)
 127 {
 128     // Format of a PMM file:
 129     // First comes a header with 128 bytes. At the beginning is the name of
 130     // the folder. Then there are some unknown bytes (strings). At offset 128
 131     // the first message starts.
 132     //
 133     // Each message is terminated by a 0x1A byte. The next message follows
 134     // immediately.
 135     //
 136     // The last message is followed by a 0x1A, too.
 137     //
 138     // 000000 6d 61 69 6c 73 65 72 76 65 72 2d 70 72 6f 6a 65    mailserver-proje
 139     // 000010 63 74 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ct..............
 140     // 000020 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
 141     // 000030 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
 142     // 000040 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
 143     // 000050 00 00 00 00 00 00 36 30 34 37 35 37 32 45 3a 36    ......6047572E:6
 144     // 000060 46 34 39 3a 46 4f 4c 30 31 33 35 35 00 00 00 00    F49:FOL01355....
 145     // 000070 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00    ................
 146     // 000080 52 65 74 75 72 6e 2d 50 61 74 68 3a 20 3c 75 72    Return-Path: <ur
 147     // ...
 148     // 000cb0 2d 2d 2d 2d 2d 2d 2d 2d 2d 2b 0d 0a 1a 52 65 74    ---------+...Ret
 149     // 000cc0 75 72 6e 2d 50 61 74 68 3a 20 3c 62 6f 75 6e 63    urn-Path: <bounc
 150     // ...
 151     // 04dc50 46 30 33 38 44 2e 36 31 35 44 37 34 44 30 2d 2d    F038D.615D74D0--
 152     // 04dc60 0d 0a 1a
 153
 154     struct {
 155         char folder[86];
 156         char id[42];
 157     } pmm_head;
 158
 159     long l = 0;
 160     QFile f(file);
 161     if (!f.open(QIODevice::ReadOnly)) {
 162         inf->alert(i18n("Unable to open %1, skipping", file));
 163     } else {
 164         // Get folder name
 165         l = f.read((char *) &pmm_head, sizeof(pmm_head));
 166         QString folder("PegasusMail-Import/");
 167         if(folderParsed)
 168             folder.append(getFolderName((QString)pmm_head.id));
 169         else
 170             folder.append(pmm_head.folder);
 171         inf->setTo(folder);
 172         inf->addLog(i18n("Importing %1", QString("../") + QString(pmm_head.folder)));
 173
 174         QByteArray input(MAX_LINE,'\0');
 175         bool first_msg = true;
 176
 177         while (!f.atEnd()) {
 178             KTemporaryFile tempfile;
 179             tempfile.open();
 180             inf->setCurrent( (int) ( ( (float) f.pos() / f.size() ) * 100 ) );
 181
 182             if(!first_msg) {
 183                 // set the filepos back to last line minus the separate char (0x1a)
 184                 f.seek(f.pos() - l + 1);
 185             }
 186
 187             // no problem to loose the last line in file. This only contains a separate char
 188             while ( ! f.atEnd() &&  (l = f.readLine(input.data(),MAX_LINE))) {
 189                     if (inf->shouldTerminate()){
 190                         return;
 191                     }
 192                     if ( input.at( 0 ) == 0x1a ) {
 193                         break;
 194                     } else {
 195                         tempfile.write( input, l );
 196                     }
 197             }
 198             tempfile.flush();
 199
 200             if(inf->removeDupMsg)
 201                 addMessage( inf, folder, tempfile.fileName() );
 202             else
 203                 addMessage_fastImport( inf, folder, tempfile.fileName() );
 204
 205             first_msg = false;
 206         }
 207     }
 208     f.close();
 209 }
 210
 211
 212 /** imports a 'unix' format mail folder (*.MBX) */
 213 void FilterPMail::importUnixMailFolder(const QString& file)
 214 {
 215     struct {
 216         char folder[58];
 217         char id[31];
 218     } pmg_head;
 219
 220     QFile f;
 221     QString folder("PegasusMail-Import/"), s(file), separate;
 222     QByteArray line(MAX_LINE,'\0');
 223     int n = 0, l = 0;
 224
 225     /** Get the folder name */
 226     s.replace( QRegExp("mbx$"), "pmg");
 227     s.replace( QRegExp("MBX$"), "PMG");
 228     f.setFileName(s);
 229     if (! f.open( QIODevice::ReadOnly ) ) {
 230         inf->alert( i18n("Unable to open %1, skipping", s ) );
 231         return;
 232     } else {
 233         f.read((char *) &pmg_head, sizeof(pmg_head));
 234         f.close();
 235
 236          if(folderParsed)
 237             folder.append(getFolderName((QString)pmg_head.id));
 238         else
 239             folder.append(pmg_head.folder);
 240
 241         inf->setTo(folder);
 242         inf->setTo(folder);
 243     }
 244
 245     /** Read in the mbox */
 246     f.setFileName(file);
 247     if (! f.open( QIODevice::ReadOnly ) ) {
 248         inf->alert( i18n("Unable to open %1, skipping", s ) );
 249     } else {
 250         inf->addLog(i18n("Importing %1", QString("../") + QString(pmg_head.folder)));
 251         l = f.readLine( line.data(),MAX_LINE); // read the first line which is unneeded
 252         while ( ! f.atEnd() ) {
 253             KTemporaryFile tempfile;
 254             tempfile.open();
 255
 256             // we lost the last line, which is the first line of the new message in
 257             // this lopp, but this is ok, because this is the separate line with
 258             // "From ???@???" and we can forget them
 259             while ( ! f.atEnd() &&  (l = f.readLine(line.data(),MAX_LINE)) && ((separate = line.data()).left(5) != "From ")) {
 260                 tempfile.write(line.data(), l);
 261                 if (inf->shouldTerminate()){
 262                     return;
 263                 }
 264             }
 265             tempfile.flush();
 266             if(inf->removeDupMsg)
 267                 addMessage( inf, folder, tempfile.fileName() );
 268             else
 269                 addMessage_fastImport( inf, folder, tempfile.fileName() );
 270
 271             n++;
 272             inf->setCurrent(i18n("Message %1", n));
 273             inf->setCurrent( (int) ( ( (float) f.pos() / f.size() ) * 100 ) );
 274         }
 275     }
 276     f.close();
 277 }
 278
 279 /** Parse the information about folderstructure to folderMatrix */
 280 bool FilterPMail::parseFolderMatrix()
 281 {
 282     kDebug() <<"Start parsing the foldermatrix.";
 283     inf->addLog(i18n("Parsing the folder structure..."));
 284
 285     QFile hierarch(chosenDir + "/hierarch.pm");
 286     if (! hierarch.open( QIODevice::ReadOnly ) ) {
 287         inf->alert( i18n("Unable to open %1, skipping", chosenDir + "hierarch.pm" ) );
 288         return false;
 289     } else {
 290         QStringList tmpList;
 291         QByteArray tmpRead;
 292         while ( !hierarch.atEnd() ) {
 293             tmpRead = hierarch.readLine();
 294             if ( tmpRead.isEmpty() )
 295                 break;
 296             QString tmpArray[5];
 297             tmpRead.remove(tmpRead.length() -2,2);
 298             QStringList tmpList = QString(tmpRead).split(',', QString::SkipEmptyParts);
 299             int i = 0;
 300             for ( QStringList::Iterator it = tmpList.begin(); it != tmpList.end(); ++it, i++) {
 301                 QString _tmp = *it;
 302                 if(i < 5) tmpArray[i] = _tmp.remove('\"');
 303                 else {
 304                     hierarch.close();
 305                     return false;
 306                 }
 307             }
 308             folderMatrix.append(FolderStructure(tmpArray));
 309         }
 310     }
 311     hierarch.close();
 312     return true;
 313 }
 314
 315 /** get the foldername for a given file ID from folderMatrix */
 316 QString FilterPMail::getFolderName(QString ID)
 317 {
 318     bool found = false;
 319     QString folder;
 320     QString search = ID;
 321
 322     while (!found)
 323     {
 324         for ( FolderStructureIterator it = folderMatrix.begin(); it != folderMatrix.end(); ++it) {
 325             FolderStructure tmp = *it;
 326
 327             QString _ID = tmp[2];
 328             if(_ID == search) {
 329                 QString _type = tmp[0] + tmp[1];
 330                 if(( _type == "21")) {
 331                     found = true;
 332                     break;
 333                 }
 334                 else {
 335                     folder.prepend((tmp[4] + '/'));
 336                     search = tmp[3];
 337                 }
 338             }
 339         }
 340     }
 341     return folder;
 342 }