DB/NewImageFinder.cpp

   1 /* Copyright (C) 2003-2006 Jesper K. Pedersen <blackie@kde.org>
   2
   3    This program is free software; you can redistribute it and/or
   4    modify it under the terms of the GNU General Public
   5    License as published by the Free Software Foundation; either
   6    version 2 of the License, or (at your option) any later version.
   7
   8    This program is distributed in the hope that it will be useful,
   9    but WITHOUT ANY WARRANTY; without even the implied warranty of
  10    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11    General Public License for more details.
  12
  13    You should have received a copy of the GNU General Public License
  14    along with this program; see the file COPYING.  If not, write to
  15    the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  16    Boston, MA 02110-1301, USA.
  17 */
  18 #include "NewImageFinder.h"
  19
  20 #include <sys/types.h>
  21 #include <dirent.h>
  22 #include <stdio.h>
  23
  24 #include "DB/ImageDB.h"
  25 #include "DB/ResultId.h"
  26 #include "DB/Result.h"
  27
  28 #include <qfileinfo.h>
  29 #include <QStringList>
  30 #include <QProgressDialog>
  31 #include <klocale.h>
  32 #include <qapplication.h>
  33 #include <qeventloop.h>
  34 #include <kmessagebox.h>
  35 #include "DB/MD5Map.h"
  36
  37 #include "config-kpa-exiv2.h"
  38 #ifdef HAVE_EXIV2
  39 #  include "Exif/Database.h"
  40 #endif
  41
  42 #include "ImageManager/Manager.h"
  43 #include "ImageManager/RawImageDecoder.h"
  44 #include "Settings/SettingsData.h"
  45 #include "Utilities/Util.h"
  46
  47 using namespace DB;
  48
  49 bool NewImageFinder::findImages()
  50 {
  51     // Load the information from the XML file.
  52     QSet<QString> loadedFiles;
  53
  54     // TODO: maybe the databas interface should allow to query if it
  55     // knows about an image ? Here we've to iterate through all of them and it
  56     // might be more efficient do do this in the database without fetching the
  57     // whole info.
  58     Q_FOREACH(
  59         const DB::ImageInfoPtr& info,
  60         DB::ImageDB::instance()->images().fetchInfos()) {
  61         loadedFiles.insert(info->fileName(DB::AbsolutePath));
  62     }
  63
  64     _pendingLoad.clear();
  65     searchForNewFiles( loadedFiles, Settings::SettingsData::instance()->imageDirectory() );
  66     loadExtraFiles();
  67
  68     // To avoid deciding if the new images are shown in a given thumbnail view or in a given search
  69     // we rather just go to home.
  70     return (!_pendingLoad.isEmpty()); // returns if new images was found.
  71 }
  72
  73 // FastDir is used in place of QDir because QDir stat()s every file in
  74 // the directory, even if we tell it not to restrict anything.  When
  75 // scanning for new images, we don't want to look at files we already
  76 // have in our database, and we also don't want to look at files whose
  77 // names indicate that we don't care about them.  So what we do is
  78 // simply read the names from the directory and let the higher layers
  79 // decide what to do with them.
  80 //
  81 // On my sample database with ~20,000 images, this improves the time
  82 // to rescan for images on a cold system from about 100 seconds to
  83 // about 3 seconds.
  84 //
  85 // -- Robert Krawitz, rlk@alum.mit.edu 2007-07-22
  86 class FastDir
  87 {
  88 public:
  89     FastDir(const QString &path);
  90     QStringList entryList() const;
  91 private:
  92     const QString _path;
  93 };
  94
  95 FastDir::FastDir(const QString &path)
  96   : _path(path)
  97 {
  98 }
  99
 100 QStringList FastDir::entryList() const
 101 {
 102     QStringList answer;
 103     DIR *dir;
 104     dirent *file;
 105     dir = opendir( QFile::encodeName(_path) );
 106     if ( !dir )
 107         return answer; // cannot read the directory
 108
 109 #if defined(QT_THREAD_SUPPORT) && defined(_POSIX_THREAD_SAFE_FUNCTIONS) && !defined(Q_OS_CYGWIN)
 110     union dirent_buf {
 111         struct dirent mt_file;
 112         char b[sizeof(struct dirent) + MAXNAMLEN + 1];
 113     } *u = new union dirent_buf;
 114     while ( readdir_r(dir, &(u->mt_file), &file ) == 0 && file )
 115 #else
 116     while ( (file = readdir(dir)) )
 117 #endif // QT_THREAD_SUPPORT && _POSIX_THREAD_SAFE_FUNCTIONS
 118         answer.append(QFile::decodeName(file->d_name));
 119 #if defined(QT_THREAD_SUPPORT) && defined(_POSIX_THREAD_SAFE_FUNCTIONS) && !defined(Q_OS_CYGWIN)
 120     delete u;
 121 #endif
 122     (void) closedir(dir);
 123     return answer;
 124 }
 125
 126 void NewImageFinder::searchForNewFiles( const QSet<QString>& loadedFiles, QString directory )
 127 {
 128     qApp->processEvents( QEventLoop::AllEvents );
 129     if ( directory.endsWith( QString::fromLatin1("/") ) )
 130         directory = directory.mid( 0, directory.length()-1 );
 131
 132     QString imageDir = Settings::SettingsData::instance()->imageDirectory();
 133     if ( imageDir.endsWith( QString::fromLatin1("/") ) )
 134         imageDir = imageDir.mid( 0, imageDir.length()-1 );
 135
 136     FastDir dir( directory );
 137     QStringList dirList = dir.entryList( );
 138     ImageManager::RAWImageDecoder dec;   // TODO: DEPENDENCY: DB:: should not reference other directories
 139     QStringList excluded;
 140     excluded << Settings::SettingsData::instance()->excludeDirectories();
 141     excluded = excluded.at(0).split(QString::fromLatin1(","));
 142     for( QStringList::const_iterator it = dirList.constBegin(); it != dirList.constEnd(); ++it ) {
 143         QString file = directory + QString::fromLatin1("/") + *it;
 144         if ( (*it) == QString::fromLatin1(".") || (*it) == QString::fromLatin1("..") ||
 145                 excluded.contains( (*it) ) || loadedFiles.contains( file ) ||
 146                 dec._skipThisFile(loadedFiles, file) ||
 147                 (*it) == QString::fromLatin1("CategoryImages") )
 148             continue;
 149
 150         QFileInfo fi( file );
 151
 152             if ( !fi.isReadable() )
 153                 continue;
 154
 155         if ( fi.isFile() ) {
 156             QString baseName = file.mid( imageDir.length()+1 );
 157             if ( ! DB::ImageDB::instance()->isBlocking( baseName ) ) {
 158                 if ( Utilities::canReadImage(file) )
 159                     _pendingLoad.append( qMakePair( baseName, DB::Image ) );
 160                 else if ( Utilities::isVideo( file ) )
 161                     _pendingLoad.append( qMakePair( baseName, DB::Video ) );
 162             }
 163         } else if ( fi.isDir() )  {
 164             searchForNewFiles( loadedFiles, file );
 165         }
 166     }
 167 }
 168
 169 void NewImageFinder::loadExtraFiles()
 170 {
 171     // FIXME: should be converted to a threadpool for SMP stuff and whatnot :]
 172     QProgressDialog dialog;
 173     dialog.setLabelText( i18n("<p><b>Loading information from new files</b></p>"
 174                               "<p>Depending on the number of images, this may take some time.<br/>"
 175                               "However, there is only a delay when new images are found.</p>") );
 176     dialog.setMaximum( _pendingLoad.count() );
 177     dialog.setMinimumDuration( 1000 );
 178
 179     int count = 0;
 180     ImageInfoList newImages;
 181     for( LoadList::Iterator it = _pendingLoad.begin(); it != _pendingLoad.end(); ++it, ++count ) {
 182         dialog.setValue( count ); // ensure to call setProgress(0)
 183         qApp->processEvents( QEventLoop::AllEvents );
 184
 185         if ( dialog.wasCanceled() )
 186             return;
 187         ImageInfoPtr info = loadExtraFile( (*it).first, (*it).second );
 188         if ( info ) {
 189             markUnTagged(info);
 190             newImages.append(info);
 191         }
 192     }
 193     DB::ImageDB::instance()->addImages( newImages );
 194 }
 195
 196
 197 ImageInfoPtr NewImageFinder::loadExtraFile( const QString& relativeNewFileName, DB::MediaType type )
 198 {
 199     QString absoluteNewFileName = Utilities::absoluteImageFileName( relativeNewFileName );
 200     MD5 sum = Utilities::MD5Sum( absoluteNewFileName );
 201     if ( DB::ImageDB::instance()->md5Map()->contains( sum ) ) {
 202         QString relativeMatchedFileName = DB::ImageDB::instance()->md5Map()->lookup(sum);
 203         QString absoluteMatchedFileName = Utilities::absoluteImageFileName( relativeMatchedFileName );
 204         QFileInfo fi( absoluteMatchedFileName );
 205
 206         if ( !fi.exists() ) {
 207             // The file we had a collapse with didn't exists anymore so it is likely moved to this new name
 208             ImageInfoPtr info = DB::ImageDB::instance()->info( relativeMatchedFileName, DB::RelativeToImageRoot );
 209             if ( !info )
 210                 qWarning("How did that happen? We couldn't find info for the images %s", qPrintable(relativeMatchedFileName));
 211             else {
 212                 info->delaySavingChanges(true);
 213                 fi = QFileInfo ( relativeMatchedFileName );
 214                 if ( info->label() == fi.completeBaseName() ) {
 215                     fi = QFileInfo( relativeNewFileName );
 216                     info->setLabel( fi.completeBaseName() );
 217                 }
 218
 219                 DB::ImageDB::instance()->renameImage( info, relativeNewFileName );
 220
 221                 // We need to insert the new name into the MD5 map,
 222                 // as it is a map, the value for the moved file will automatically be deleted.
 223                 DB::ImageDB::instance()->md5Map()->insert( sum, info->fileName(DB::RelativeToImageRoot) );
 224
 225 #ifdef HAVE_EXIV2
 226                 Exif::Database::instance()->remove( absoluteMatchedFileName );
 227                 Exif::Database::instance()->add( absoluteNewFileName );
 228 #endif
 229                 return DB::ImageInfoPtr();
 230             }
 231         }
 232     }
 233
 234     QString err = Settings::SettingsData::instance()->modifiedFileComponent();
 235     QRegExp modifiedFileComponent =
 236         QRegExp(Settings::SettingsData::instance()->modifiedFileComponent());
 237
 238     // check to see if this is a new version of a previous image
 239     ImageInfoPtr info = ImageInfoPtr(new ImageInfo( relativeNewFileName, type ));
 240     ImageInfoPtr originalInfo;
 241     QString originalFileName;
 242
 243     if (Settings::SettingsData::instance()->detectModifiedFiles()) {
 244         // should be cached because loading once per image is expensive
 245         QString err = Settings::SettingsData::instance()->modifiedFileComponent();
 246         QRegExp modifiedFileComponent =
 247             QRegExp(Settings::SettingsData::instance()->modifiedFileComponent());
 248         // requires at least *something* in the modifiedFileComponent
 249         if (err.length() >= 0 &&
 250             relativeNewFileName.contains(modifiedFileComponent)) {
 251
 252             originalFileName = relativeNewFileName;
 253             QString originalFileComponent =
 254                 Settings::SettingsData::instance()->originalFileComponent();
 255             originalFileName.replace(modifiedFileComponent, originalFileComponent);
 256
 257             MD5 originalSum = Utilities::MD5Sum( Utilities::absoluteImageFileName( originalFileName ) );
 258             if ( DB::ImageDB::instance()->md5Map()->contains( originalSum ) ) {
 259                 // we have a previous copy of this file; copy it's data
 260                 // from the original.
 261                 originalInfo = DB::ImageDB::instance()->info( originalFileName, DB::RelativeToImageRoot );
 262                 if ( !originalInfo ) {
 263                     qWarning("How did that happen? We couldn't find info for the original image %s; can't copy the original data to %s", qPrintable(originalFileName), qPrintable(relativeNewFileName));
 264                 } else {
 265                     info->copyExtraData(*originalInfo);
 266                 }
 267
 268                 /* if requested to move, then delete old data from original */
 269                 if (Settings::SettingsData::instance()->moveOriginalContents() ) {
 270                     originalInfo->removeExtraData();
 271                 }
 272             }
 273         }
 274     }
 275
 276     // also inserts image into exif db if present:
 277     info->setMD5Sum(sum);
 278     DB::ImageDB::instance()->md5Map()->insert( sum, info->fileName(DB::RelativeToImageRoot) );
 279
 280     if (originalInfo &&
 281         Settings::SettingsData::instance()->autoStackNewFiles() ) {
 282         // we have to do this immediately to get the ids
 283         ImageInfoList newImages;
 284         markUnTagged(info);
 285         newImages.append(info);
 286         DB::ImageDB::instance()->addImages( newImages );
 287
 288         // stack the files together
 289         DB::ResultId olderfile = DB::ImageDB::instance()->ID_FOR_FILE(originalFileName);
 290         DB::ResultId newerfile = DB::ImageDB::instance()->ID_FOR_FILE(info->fileName(DB::AbsolutePath));
 291         DB::Result tostack = DB::Result();
 292
 293         tostack.append(newerfile);
 294         tostack.append(olderfile);
 295         DB::ImageDB::instance()->stack(tostack);
 296
 297         // ordering: XXX we ideally want to place the new image right
 298         // after the older one in the list.
 299
 300         // XXX: deal with already-stacked items; currently a silent fail
 301
 302         info = NULL;  // we already added it, so don't process again
 303     }
 304
 305     return info;
 306 }
 307
 308 bool  NewImageFinder::calculateMD5sums(
 309     const DB::Result& list,
 310     DB::MD5Map* md5Map,
 311     bool* wasCanceled)
 312 {
 313     // FIXME: should be converted to a threadpool for SMP stuff and whatnot :]
 314     QProgressDialog dialog;
 315     dialog.setLabelText(
 316         i18n("<p><b>Calculating checksum for %1 files<b></p>"
 317              "<p>By storing a checksum for each image "
 318              "KPhotoAlbum is capable of finding images "
 319              "even when you have moved them on the disk.</p>", list.size()));
 320     dialog.setMaximum(list.size());
 321     dialog.setMinimumDuration( 1000 );
 322
 323     int count = 0;
 324     QStringList cantRead;
 325     bool dirty = false;
 326
 327     Q_FOREACH(DB::ImageInfoPtr info, list.fetchInfos()) {
 328         const QString absoluteFileName = info->fileName(DB::AbsolutePath );
 329         if ( count % 10 == 0 ) {
 330             dialog.setValue( count ); // ensure to call setProgress(0)
 331             qApp->processEvents( QEventLoop::AllEvents );
 332
 333             if ( dialog.wasCanceled() ) {
 334                 if ( wasCanceled )
 335                     *wasCanceled = true;
 336                 return dirty;
 337             }
 338         }
 339
 340         MD5 md5 = Utilities::MD5Sum( absoluteFileName );
 341         if (md5.isNull()) {
 342             cantRead << absoluteFileName;
 343             continue;
 344         }
 345
 346         if  ( info->MD5Sum() != md5 ) {
 347             info->setMD5Sum( md5 );
 348             dirty = true;
 349             ImageManager::Manager::instance()->removeThumbnail( absoluteFileName );
 350         }
 351
 352         md5Map->insert( md5, info->fileName(DB::RelativeToImageRoot) );
 353
 354         ++count;
 355     }
 356     if ( wasCanceled )
 357         *wasCanceled = false;
 358
 359     if ( !cantRead.empty() )
 360         KMessageBox::informationList( 0, i18n("Following files could not be read:"), cantRead );
 361
 362     return dirty;
 363 }
 364
 365 void DB::NewImageFinder::markUnTagged( ImageInfoPtr info )
 366 {
 367     if ( Settings::SettingsData::instance()->hasUntaggedCategoryFeatureConfigured() ) {
 368         info->addCategoryInfo( Settings::SettingsData::instance()->untaggedCategory(),
 369                            Settings::SettingsData::instance()->untaggedTag() );
 370     }
 371 }