mailimporter/filter_oe.cpp

   1 /***************************************************************************
   2                           filter_oe.cxx  -  Outlook Express mail import
   3                              -------------------
   4     begin                : Sat Feb 1 2003
   5     copyright            : (C) 2003 by Laurence Anderson <l.d.anderson@warwick.ac.uk>
   6                            (C) 2005 by Danny Kukawka <danny.Kukawka@web.de>
   7  ***************************************************************************/
   8
   9 /***************************************************************************
  10  *                                                                         *
  11  *   This program is free software; you can redistribute it and/or modify  *
  12  *   it under the terms of the GNU General Public License as published by  *
  13  *   the Free Software Foundation; either version 2 of the License, or     *
  14  *   (at your option) any later version.                                   *
  15  *                                                                         *
  16  ***************************************************************************/
  17
  18 // This filter was created by looking at libdbx  &liboe
  19
  20 #include <klocale.h>
  21 #include <kfiledialog.h>
  22 #include <ktemporaryfile.h>
  23 #include <kdebug.h>
  24
  25 #include "filter_oe.h"
  26
  27 #define OE4_SIG_1 0x36464d4a
  28 #define OE4_SIG_2 0x00010003
  29 #define OE5_SIG_1 0xfe12adcf
  30 #define OE5_EMAIL_SIG_2 0x6f74fdc5
  31 #define OE5_FOLDER_SIG_2 0x6f74fdc6
  32 #define OE5_SIG_3 0x11d1e366
  33 #define OE5_SIG_4 0xc0004e9a
  34 #define MBX_MAILMAGIC 0x7F007F00
  35
  36 using namespace MailImporter;
  37
  38 FilterOE::FilterOE() :
  39     Filter( i18n("Import Outlook Express Emails"),
  40             i18n("Laurence Anderson <br>( Filter enhanced by Danny Kukawka )</p>"),
  41             i18n("<p><b>Outlook Express 4/5/6 import filter</b></p>"
  42                  "<p>You will need to locate the folder where the mailbox has been "
  43                  "stored by searching for .dbx or .mbx files under "
  44                  "<ul><li><i>C:\\Windows\\Application Data</i> in Windows 9x</li>"
  45                  "<li><i>Documents and Settings</i> in Windows 2000 or later</li></ul></p>"
  46                  "<p><b>Note:</b> Since it is possible to recreate the folder structure, the folders from "
  47                  "Outlook Express 5 and 6 will be stored under: \"OE-Import\" in your local folder.</p>" ))
  48 {
  49 }
  50
  51 FilterOE::~FilterOE()
  52 {
  53 }
  54
  55 void FilterOE::import()
  56 {
  57     // Select directory containing plain text emails
  58     const QString maildir = KFileDialog::getExistingDirectory(QDir::homePath(),filterInfo()->parent());
  59     importMails(maildir);
  60 }
  61
  62 void FilterOE::importMails(const QString  &maildir)
  63 {
  64     setMailDir(maildir);
  65     if (mailDir().isEmpty()) { // No directory selected
  66         filterInfo()->alert(i18n("No directory selected."));
  67         return;
  68     }
  69
  70     QDir dir (mailDir());
  71     QStringList files = dir.entryList(QStringList(QLatin1String("*.[dDmM][bB][xX]")), QDir::Files, QDir::Name);
  72     if (files.isEmpty()) {
  73         filterInfo()->alert(i18n("No Outlook Express mailboxes found in directory %1.", mailDir()));
  74         return;
  75     }
  76
  77     totalFiles = files.count();
  78     currentFile = 0;
  79     count0x04 = 0;
  80     count0x84 = 0;
  81     parsedFolder = false;
  82
  83     filterInfo()->setOverall(0);
  84
  85     /** search the folderfile to recreate folder struct */
  86
  87     for ( QStringList::Iterator mailFile = files.begin(); mailFile != files.end(); ++mailFile ) {
  88         if(*mailFile == QLatin1String( "Folders.dbx" )) {
  89             filterInfo()->addInfoLogEntry(i18n("Import folder structure..."));
  90             importMailBox(dir.filePath(*mailFile));
  91             if(!folderStructure.isEmpty())
  92                 parsedFolder = true;
  93             // remove file from QStringList::files, no longer needed
  94             files.erase(mailFile);
  95             currentIsFolderFile = false;
  96             break;
  97         }
  98     }
  99
 100     int n=0;
 101     QStringList::ConstIterator end( files.constEnd() );
 102     for ( QStringList::ConstIterator mailFile = files.constBegin(); mailFile != end; ++mailFile ) {
 103         if ( filterInfo()->shouldTerminate() )
 104             break;
 105         importMailBox(dir.filePath(*mailFile));
 106         filterInfo()->setOverall(100 * ++n  / files.count());
 107     }
 108
 109     filterInfo()->setOverall(100);
 110     filterInfo()->setCurrent(100);
 111     filterInfo()->addInfoLogEntry(i18n("Finished importing Outlook Express emails"));
 112     if (filterInfo()->shouldTerminate())
 113         filterInfo()->addInfoLogEntry( i18n("Finished import, canceled by user."));
 114
 115     kDebug() <<"total emails in current file:" << totalEmails;
 116     kDebug() <<"0x84 Mails:" << count0x84;
 117     kDebug() <<"0x04 Mails:" << count0x04;
 118 }
 119
 120 void FilterOE::importMailBox( const QString &fileName)
 121 {
 122     QFile mailfile(fileName);
 123     QFileInfo mailfileinfo(fileName);
 124     QString _nameOfFile = fileName;
 125     _nameOfFile.remove( mailDir() );
 126     _nameOfFile.remove( QLatin1Char('/') );
 127     filterInfo()->setFrom(mailfileinfo.fileName());
 128
 129     if (!mailfile.open(QIODevice::ReadOnly)) {
 130         filterInfo()->addErrorLogEntry(i18n("Unable to open mailbox %1", fileName));
 131         return;
 132     }
 133     QDataStream mailbox(&mailfile);
 134     mailbox.setByteOrder(QDataStream::LittleEndian);
 135
 136     // Parse magic
 137     quint32 sig_block1, sig_block2;
 138     mailbox >> sig_block1 >> sig_block2;
 139     if (sig_block1 == OE4_SIG_1 && sig_block2 == OE4_SIG_2) {
 140         folderName = QLatin1String("OE-Import/") + mailfileinfo.completeBaseName();
 141         filterInfo()->addInfoLogEntry(i18n("Importing OE4 Mailbox %1", QLatin1String("../") + _nameOfFile));
 142         filterInfo()->setTo(folderName);
 143         mbxImport(mailbox);
 144         return;
 145     } else {
 146         quint32 sig_block3, sig_block4;
 147         mailbox >> sig_block3 >> sig_block4;
 148         if (sig_block1 == OE5_SIG_1 && sig_block3 == OE5_SIG_3 && sig_block4 == OE5_SIG_4) {
 149             if (sig_block2 == OE5_EMAIL_SIG_2) {
 150                 folderName = QLatin1String("OE-Import/") + mailfileinfo.completeBaseName();
 151                 if(parsedFolder) {
 152                     const QString _tmpFolder = getFolderName(_nameOfFile);
 153                     if(!_tmpFolder.isEmpty()) folderName = QLatin1String("OE-Import/") + _tmpFolder;
 154                 }
 155                 filterInfo()->addInfoLogEntry(i18n("Importing OE5+ Mailbox %1", QLatin1String("../") + _nameOfFile));
 156                 filterInfo()->setTo(folderName);
 157                 dbxImport(mailbox);
 158                 return;
 159             } else if (sig_block2 == OE5_FOLDER_SIG_2) {
 160                 if(!parsedFolder) {
 161                     filterInfo()->addInfoLogEntry(i18n("Importing OE5+ Folder file %1", QLatin1String("../") + _nameOfFile));
 162                     currentIsFolderFile = true;
 163                     dbxImport(mailbox);
 164                     currentIsFolderFile = false;
 165                 }
 166                 return;
 167             }
 168         }
 169     }
 170     // filterInfo()->addInfoLogEntry(i18n("File %1 does not seem to be an Outlook Express mailbox").arg("../" + _nameOfFile));
 171 }
 172
 173 /* ------------------- MBX support ------------------- */
 174
 175 void FilterOE::mbxImport( QDataStream &ds)
 176 {
 177     quint32 msgCount, lastMsgNum, fileSize;
 178
 179     // Read the header
 180     ds >> msgCount >> lastMsgNum >> fileSize;
 181     ds.device()->seek( ds.device()->pos() + 64 ); // Skip 0's
 182     kDebug() <<"This mailbox has" << msgCount <<" messages";
 183     if (msgCount == 0)
 184         return; // Don't import empty mailbox
 185
 186     quint32 msgMagic;
 187     ds >> msgMagic; // Read first magic
 188
 189     while (!ds.atEnd()) {
 190
 191         quint32 msgNumber, msgSize, msgTextSize;
 192         KTemporaryFile tmp;
 193         tmp.open();
 194         QDataStream dataStream(&tmp);
 195         dataStream.setByteOrder(QDataStream::LittleEndian);
 196
 197         // Read the messages
 198         ds >> msgNumber >> msgSize >> msgTextSize; // All seem to be lies...?
 199         do {
 200             ds >> msgMagic;
 201             if (msgMagic != MBX_MAILMAGIC)
 202                 dataStream << msgMagic;
 203             else
 204                 break;
 205         } while ( !ds.atEnd() );
 206         tmp.flush();
 207         /* comment by Danny Kukawka:
 208      * addMessage() == old function, need more time and check for duplicates
 209      * addMessage_fastImport == new function, faster and no check for duplicates
 210      */
 211         if(filterInfo()->removeDupMessage())
 212             addMessage( folderName, tmp.fileName() );
 213         else
 214             addMessage_fastImport( folderName, tmp.fileName() );
 215
 216         if(filterInfo()->shouldTerminate())
 217             return;
 218     }
 219 }
 220
 221 /* ------------------- DBX support ------------------- */
 222
 223 void FilterOE::dbxImport( QDataStream &ds)
 224 {
 225     // Get item count  &offset of index
 226     quint32 itemCount, indexPtr;
 227     ds.device()->seek(0xc4);
 228     ds >> itemCount;
 229     ds.device()->seek(0xe4);
 230     ds >> indexPtr;
 231     kDebug() <<"Item count is" << itemCount <<", Index at" << indexPtr;
 232
 233     if (itemCount == 0)
 234         return; // Empty file
 235     totalEmails = itemCount;
 236     currentEmail = 0;
 237     // Parse the indexes
 238     ds.device()->seek(indexPtr);
 239     dbxReadIndex(ds, indexPtr);
 240 }
 241
 242 void FilterOE::dbxReadIndex( QDataStream &ds, int filePos)
 243 {
 244
 245     if(filterInfo()->shouldTerminate()) return;
 246     quint32 self, unknown, nextIndexPtr, parent, indexCount;
 247     quint8 unknown2, ptrCount;
 248     quint16 unknown3;
 249     int wasAt = ds.device()->pos();
 250     ds.device()->seek(filePos);
 251
 252
 253     kDebug() <<"Reading index of file" << folderName;
 254     ds >> self >> unknown >> nextIndexPtr >> parent >> unknown2 >> ptrCount >> unknown3 >> indexCount; // _dbx_tableindexstruct
 255
 256     kDebug() <<"This index has" << (int) ptrCount <<" data pointers";
 257     for (int count = 0; count < ptrCount; ++count) {
 258         if(filterInfo()->shouldTerminate())
 259             return;
 260         quint32 dataIndexPtr, anotherIndexPtr, anotherIndexCount; // _dbx_indexstruct
 261         ds >> dataIndexPtr >> anotherIndexPtr >> anotherIndexCount;
 262
 263         if (anotherIndexCount > 0) {
 264             kDebug() <<"Recursing to another table @" << anotherIndexPtr;
 265             dbxReadIndex(ds, anotherIndexPtr);
 266         }
 267         kDebug() <<"Data index @" << dataIndexPtr;
 268         dbxReadDataBlock( ds, dataIndexPtr);
 269     }
 270
 271     if (indexCount > 0) { // deal with nextTablePtr
 272         kDebug() <<"Recuring to next table @" << nextIndexPtr;
 273         dbxReadIndex(ds, nextIndexPtr);
 274     }
 275
 276     ds.device()->seek(wasAt); // Restore file position to same as when function called
 277 }
 278
 279 void FilterOE::dbxReadDataBlock(QDataStream &ds, int filePos)
 280 {
 281     quint32 curOffset, blockSize;
 282     quint16 unknown;
 283     quint8 count, unknown2;
 284     int wasAt = ds.device()->pos();
 285
 286     QString folderEntry[4];
 287
 288     ds.device()->seek(filePos);
 289
 290     ds >> curOffset >> blockSize >> unknown >> count >> unknown2; // _dbx_email_headerstruct
 291     kDebug() <<"Data block has" << (int) count <<" elements";
 292
 293     for (int c = 0; c < count; c++) {
 294         if(filterInfo()->shouldTerminate()) return;
 295         quint8 type;  // _dbx_email_pointerstruct
 296         quint32 value; // Actually 24 bit
 297
 298         ds >> type >> value;
 299         value &= 0xffffff;
 300         ds.device()->seek(ds.device()->pos() - 1); // We only wanted 3 bytes
 301
 302         if(!currentIsFolderFile) {
 303             if (type == 0x84) { // It's an email!
 304                 kDebug() <<"**** Offset of emaildata (0x84)" << value <<" ****";
 305                 dbxReadEmail( ds, value);
 306                 ++count0x84;
 307             } else if( type == 0x04) {
 308                 int currentFilePos = ds.device()->pos();
 309                 ds.device()->seek(filePos + 12 + value + (count*4) );
 310                 quint32 newOFF;
 311                 ds >> newOFF;
 312                 kDebug() <<"**** Offset of emaildata (0x04)" <<  newOFF;
 313                 ds.device()->seek(currentFilePos);
 314                 dbxReadEmail(ds, newOFF);
 315                 ++count0x04;
 316             }
 317         }
 318         else {
 319             // this is a folderfile
 320             if(type == 0x02) {
 321                 // kDebug() <<"**** FOLDER: descriptive name ****";
 322                 folderEntry[0] = parseFolderString(ds, filePos + 12 + value + (count*4) );
 323             } else if (type == 0x03) {
 324                 // kDebug() <<"**** FOLDER: filename ****";
 325                 folderEntry[1] = parseFolderString(ds, filePos + 12 + value + (count*4) );
 326
 327             } else if (type == 0x80) {
 328                 // kDebug() <<"**** FOLDER: current ID ****";
 329                 folderEntry[2] = QString::number(value);
 330
 331             } else if (type == 0x81) {
 332                 // kDebug() <<"**** FOLDER: parent ID ****";
 333                 folderEntry[3] =  QString::number(value);
 334             }
 335         }
 336     }
 337     if(currentIsFolderFile) {
 338         folderStructure.append(FolderStructure(folderEntry));
 339     }
 340     ds.device()->seek(wasAt); // Restore file position to same as when function called
 341 }
 342
 343 void FilterOE::dbxReadEmail(QDataStream &ds, int filePos)
 344 {
 345     if(filterInfo()->shouldTerminate()) return;
 346     quint32 self, nextAddressOffset, nextAddress=0;
 347     quint16 blockSize;
 348     quint8 intCount, unknown;
 349     KTemporaryFile tmp;
 350     tmp.open();
 351     bool _break = false;
 352     int wasAt = ds.device()->pos();
 353     ds.device()->seek(filePos);
 354     QDataStream tempDs (&tmp);
 355
 356     do {
 357         ds >> self >> nextAddressOffset >> blockSize >> intCount >> unknown >> nextAddress; // _dbx_block_hdrstruct
 358         QByteArray blockBuffer(blockSize,'\0');
 359         ds.readRawData(blockBuffer.data(), blockSize);
 360         tempDs.writeRawData(blockBuffer.data(), blockSize);
 361         // to detect incomplete mails or corrupted archives. See Bug #86119
 362         if(ds.atEnd()) {
 363             _break = true;
 364             break;
 365         }
 366         ds.device()->seek(nextAddress);
 367     } while (nextAddress != 0);
 368     tmp.flush();
 369
 370     if(!_break) {
 371         if(filterInfo()->removeDupMessage())
 372             addMessage( folderName, tmp.fileName() );
 373         else
 374             addMessage_fastImport( folderName, tmp.fileName() );
 375
 376         currentEmail++;
 377         int currentPercentage = (int) ( ( (float) currentEmail / totalEmails ) * 100 );
 378         filterInfo()->setCurrent(currentPercentage);
 379         ds.device()->seek(wasAt);
 380     }
 381 }
 382
 383 /* ------------------- FolderFile support ------------------- */
 384 QString FilterOE::parseFolderString( QDataStream &ds, int filePos )
 385 {
 386     char tmp;
 387     QString returnString;
 388     int wasAt = ds.device()->pos();
 389     ds.device()->seek(filePos);
 390
 391     // read while != 0x00
 392     while( !ds.device()->atEnd() ) {
 393         ds.device()->getChar(&tmp);
 394         if( tmp != 0x00) {
 395             returnString += QLatin1Char(tmp);
 396         }
 397         else break;
 398     }
 399     ds.device()->seek(wasAt);
 400     return returnString;
 401 }
 402
 403 /** get the foldername for a given file ID from folderMatrix */
 404 QString FilterOE::getFolderName(const QString &filename)
 405 {
 406     bool found = false;
 407     bool foundFilename = false;
 408     QString folder;
 409     // we must do this because folder with more than one upper letter
 410     // at start have maybe not a file named like the folder !!!
 411     QString search = filename.toLower();
 412
 413     while (!found)
 414     {
 415         for ( FolderStructureIterator it = folderStructure.begin(); it != folderStructure.end(); ++it) {
 416             FolderStructure tmp = *it;
 417             if(foundFilename == false) {
 418                 QString _tmpFileName = tmp[1];
 419                 _tmpFileName = _tmpFileName.toLower();
 420                 if(_tmpFileName == search) {
 421                     folder.prepend( tmp[0] + QString::fromLatin1("/") );
 422                     search = tmp[3];
 423                     foundFilename = true;
 424                 }
 425             } else {
 426                 QString _currentID = tmp[2];
 427                 QString _parentID = tmp[3];
 428                 if(_currentID == search) {
 429                     if(_parentID.isEmpty()) { // this is the root of the folder
 430                         found = true;
 431                         break;
 432                     } else {
 433                         folder.prepend( tmp[0] + QString::fromLatin1("/") );
 434                         search = tmp[3];
 435                     }
 436                 }
 437             }
 438         }
 439         // need to break the while loop maybe in some cases
 440         if((foundFilename == false) && (folder.isEmpty())) return folder;
 441     }
 442     return folder;
 443 }