src/scancontroller.cpp

   1 /***************************************************************************
   2  *   Copyright (C) 2003-2007 by The Amarok Developers                      *
   3  *                                                                         *
   4  *   This program is free software; you can redistribute it and/or modify  *
   5  *   it under the terms of the GNU General Public License as published by  *
   6  *   the Free Software Foundation; either version 2 of the License, or     *
   7  *   (at your option) any later version.                                   *
   8  *                                                                         *
   9  *   This program is distributed in the hope that it will be useful,       *
  10  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
  11  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
  12  *   GNU General Public License for more details.                          *
  13  *                                                                         *
  14  *   You should have received a copy of the GNU General Public License     *
  15  *   along with this program; if not, write to the                         *
  16  *   Free Software Foundation, Inc.,                                       *
  17  *   51 Franklin Steet, Fifth Floor, Boston, MA  02111-1307, USA.          *
  18  ***************************************************************************/
  19
  20 #define DEBUG_PREFIX "ScanController"
  21
  22 #include "amarok.h"
  23 #include "amarokconfig.h"
  24 #include "collectiondb.h"
  25 #include "debug.h"
  26 #include "metabundle.h"
  27 #include "mountpointmanager.h"
  28 #include "playlist.h"
  29 #include "playlistbrowser.h"
  30 #include "scancontroller.h"
  31 #include "statusbar.h"
  32
  33 #include <QByteArray>
  34 #include <QFileInfo>
  35 #include <QTextCodec>
  36
  37 #include <kapplication.h>
  38 #include <klocale.h>
  39 #include <kmessagebox.h>
  40 ////////////////////////////////////////////////////////////////////////////////
  41 // class ScanController
  42 ////////////////////////////////////////////////////////////////////////////////
  43
  44 ScanController* ScanController::currController = 0;
  45
  46 ScanController* ScanController::instance()
  47 {
  48     return currController;
  49 }
  50
  51 void ScanController::setInstance( ScanController* curr )
  52 {
  53     currController = curr;
  54 }
  55
  56 ScanController::ScanController( CollectionDB* parent, bool incremental, const QStringList& folders )
  57     : DependentJob( parent, "CollectionScanner" )
  58     , QXmlDefaultHandler()
  59     , m_scanner( new Amarok::ProcIO() )
  60     , m_folders( folders )
  61     , m_incremental( incremental )
  62     , m_hasChanged( false )
  63     , m_source( new QXmlInputSource() )
  64     , m_reader( new QXmlSimpleReader() )
  65     , m_tablesCreated( false )
  66     , m_scanCount( 0 )
  67 {
  68     DEBUG_BLOCK
  69
  70     ScanController::setInstance( this );
  71     m_reader->setContentHandler( this );
  72
  73     connect( this, SIGNAL( scanDone( bool ) ), MountPointManager::instance(), SLOT( updateStatisticsURLs( bool ) ) );
  74
  75     connect( m_scanner, SIGNAL( readReady( K3ProcIO* ) ), SLOT( slotReadReady() ) );
  76
  77     *m_scanner << "amarokcollectionscanner";
  78     *m_scanner << "--nocrashhandler"; // We want to be able to catch SIGSEGV
  79
  80     // K3Process must be started from the GUI thread, so we're invoking the scanner
  81     // here in the ctor:
  82     if( incremental )
  83     {
  84         setDescription( i18n( "Updating Collection" ) );
  85         initIncremental();
  86     }
  87     else
  88     {
  89         setDescription( i18n( "Building Collection" ) );
  90         *m_scanner << "-p";
  91         if( AmarokConfig::scanRecursively() ) *m_scanner << "-r";
  92         *m_scanner << m_folders;
  93         m_scanner->start();
  94     }
  95 }
  96
  97
  98 ScanController::~ScanController()
  99 {
 100     DEBUG_BLOCK
 101
 102     if( !isAborted() && !m_crashedFiles.empty() ) {
 103         KMessageBox::information( 0, i18n( "<p>The Collection Scanner was unable to process these files:</p>" ) +
 104                                   "<i>" + m_crashedFiles.join( "<br>" ) + "</i>",
 105                                   i18n( "Collection Scan Report" ) );
 106     }
 107     else if( m_crashedFiles.size() >= MAX_RESTARTS ) {
 108         KMessageBox::error( 0, i18n( "<p>Sorry, the Collection Scan was aborted, since too many problems were encountered.</p>" ) +
 109                             "<p>Advice: A common source for this problem is a broken 'TagLib' package on your computer. Replacing this package may help fixing the issue.</p>"
 110                             "<p>The following files caused problems:</p>" +
 111                             "<i>" + m_crashedFiles.join( "<br>" ) + "</i>",
 112                             i18n( "Collection Scan Error" ) );
 113     }
 114
 115     m_scanner->kill();
 116     delete m_scanner;
 117     delete m_reader;
 118     delete m_source;
 119     ScanController::setInstance( 0 );
 120 }
 121
 122
 123 // Cause the CollectionDB to emit fileDeleted() signals
 124 void
 125 ScanController::completeJob( void )
 126 {
 127     m_fileMapsMutex.lock();
 128
 129     QMap<QString,QString>::Iterator it;
 130     if( !m_incremental )
 131     {
 132         CollectionDB::instance()->emitFilesAdded( m_filesAdded );
 133     }
 134     else
 135     {
 136         for( it = m_filesAdded.begin(); it != m_filesAdded.end(); ++it )
 137         {
 138             if( m_filesDeleted.contains( it.key() ) )
 139                 m_filesDeleted.remove( it.key() );
 140         }
 141         for( it = m_filesAdded.begin(); it != m_filesAdded.end(); ++it )
 142             CollectionDB::instance()->emitFileAdded( it.value(), it.key() );
 143         for( it = m_filesDeleted.begin(); it != m_filesDeleted.end(); ++it )
 144             CollectionDB::instance()->emitFileDeleted( it.value(), it.key() );
 145     }
 146
 147     m_fileMapsMutex.unlock();
 148
 149     emit scanDone( !m_incremental || m_hasChanged );
 150
 151     ThreadManager::DependentJob::completeJob();
 152 }
 153
 154
 155 /**
 156  * The Incremental Scanner works as follows: Here we check the mtime of every directory in the "directories"
 157  * table and store all changed directories in m_folders.
 158  *
 159  * These directories are then scanned in CollectionReader::doJob(), with m_recursively set according to the
 160  * user's preference, so the user can add directories or whole directory trees, too. Since we don't want to
 161  * rescan unchanged subdirectories, CollectionReader::readDir() checks if we are scanning recursively and
 162  * prevents that.
 163  */
 164 void
 165 ScanController::initIncremental()
 166 {
 167     DEBUG_BLOCK
 168
 169     connect( CollectionDB::instance(),
 170              SIGNAL( fileMoved( const QString &, const QString & ) ),
 171              SLOT( slotFileMoved( const QString &, const QString & ) ) );
 172     connect( CollectionDB::instance(),
 173              SIGNAL( fileMoved( const QString &, const QString &, const QString & ) ),
 174              SLOT( slotFileMoved( const QString &, const QString & ) ) );
 175
 176     IdList list = MountPointManager::instance()->getMountedDeviceIds();
 177     QString deviceIds;
 178     foreach( int id, list )
 179     {
 180         if ( !deviceIds.isEmpty() ) deviceIds += ',';
 181         deviceIds += QString::number( id );
 182     }
 183
 184     const QStringList values = CollectionDB::instance()->query(
 185             QString( "SELECT deviceid, dir, changedate FROM directories WHERE deviceid IN (%1);" )
 186             .arg( deviceIds ) );
 187
 188     oldForeach( values )
 189     {
 190         int id = (*it).toInt();
 191         const QString folder = MountPointManager::instance()->getAbsolutePath( id, (*++it) );
 192         const QString mtime  = *++it;
 193
 194         const QFileInfo info( folder );
 195         if( info.exists() )
 196         {
 197             if( info.lastModified().toTime_t() != mtime.toUInt() )
 198             {
 199                 m_folders << folder;
 200                 debug() << "Collection dir changed: " << folder << endl;
 201             }
 202         }
 203         else
 204         {
 205             // this folder has been removed
 206             m_folders << folder;
 207             debug() << "Collection dir removed: " << folder << endl;
 208         }
 209
 210         kapp->processEvents(); // Don't block the GUI
 211     }
 212
 213     if ( !m_folders.isEmpty() )
 214     {
 215         debug() << "Collection was modified." << endl;
 216         m_hasChanged = true;
 217         Amarok::StatusBar::instance()->shortMessage( i18n( "Updating Collection..." ) );
 218
 219         // Start scanner process
 220         if( AmarokConfig::scanRecursively() ) *m_scanner << "-r";
 221         *m_scanner << "-i";
 222         *m_scanner << m_folders;
 223         m_scanner->start();
 224     }
 225 }
 226
 227
 228 bool
 229 ScanController::doJob()
 230 {
 231     DEBUG_BLOCK
 232
 233     if( !CollectionDB::instance()->isConnected() )
 234         return false;
 235     if( m_incremental && !m_hasChanged )
 236         return true;
 237
 238     CollectionDB::instance()->createTables( true );
 239     m_tablesCreated = true;
 240
 241     //For a full rescan, we might not have cleared tags table (for devices not plugged
 242     //in), so preserve the necessary other tables (eg artist)
 243     CollectionDB::instance()->prepareTempTables();
 244
 245     CollectionDB::instance()->invalidateArtistAlbumCache();
 246     setProgressTotalSteps( 100 );
 247
 248 main_loop:
 249     uint delayCount = 100;
 250
 251     bool sessionStarted = false;
 252
 253     /// Main Loop
 254     while( !isAborted() ) {
 255         if( m_xmlData.isNull() ) {
 256             if( !m_scanner->isRunning() )
 257                 delayCount--;
 258             // Wait a bit after process has exited, so that we have time to parse all data
 259             if( delayCount == 0 )
 260                 break;
 261             msleep( 15 );
 262         }
 263         else {
 264             m_dataMutex.lock();
 265
 266             QString data = m_xmlData;
 267             m_source->setData( data );
 268             m_xmlData.clear();
 269
 270             m_dataMutex.unlock();
 271             if ( !sessionStarted )
 272                 if ( m_reader->parse( m_source, true ) ) { //start a new session
 273                     sessionStarted = true;
 274                 }
 275                 else
 276                     debug() << "Incremental parsing failed: " << errorString() << endl << QString( data ) << endl;
 277             else if( !m_reader->parseContinue() ) {
 278                 debug() << "parseContinue() failed: " << errorString() << endl << QString( data ) << endl;
 279             }
 280         }
 281     }
 282
 283     if( !isAborted() ) {
 284         if( m_scanner->normalExit() && !m_scanner->signalled() ) {
 285             CollectionDB::instance()->sanitizeCompilations();
 286             if ( m_incremental ) {
 287                 m_foldersToRemove += m_folders;
 288                 foreach( QString str, m_foldersToRemove ) {
 289                     m_fileMapsMutex.lock();
 290                     CollectionDB::instance()->removeSongsInDir( str, &m_filesDeleted );
 291                     m_fileMapsMutex.unlock();
 292                     CollectionDB::instance()->removeDirFromCollection( str );
 293                 }
 294                 CollectionDB::instance()->removeOrphanedEmbeddedImages();
 295             }
 296             else
 297                 CollectionDB::instance()->clearTables( false ); // empty permanent tables
 298
 299             CollectionDB::instance()->copyTempTables(); // copy temp into permanent tables
 300
 301             //Clean up unused entries in the main tables (eg artist, composer)
 302             CollectionDB::instance()->deleteAllRedundant( "artist" );
 303             CollectionDB::instance()->deleteAllRedundant( "composer" );
 304             CollectionDB::instance()->deleteAllRedundant( "year" );
 305             CollectionDB::instance()->deleteAllRedundant( "genre" );
 306             CollectionDB::instance()->deleteAllRedundant( "album" );
 307
 308             //Remove free space and fragmentation in the DB. Used to run on shutdown, but
 309             //that took too long, sometimes causing Amarok to be killed.
 310             CollectionDB::instance()->vacuum();
 311
 312         }
 313         else {
 314             if( m_crashedFiles.size() <= MAX_RESTARTS ||
 315                     m_crashedFiles.size() <= (m_scanCount * MAX_FAILURE_PERCENTAGE) / 100 ) {
 316                 kapp->postEvent( this, new RestartEvent() );
 317                 sleep( 3 );
 318             }
 319             else
 320                 m_aborted = true;
 321
 322             goto main_loop;
 323         }
 324     }
 325
 326     if( CollectionDB::instance()->isConnected() )
 327     {
 328         m_tablesCreated = false;
 329         CollectionDB::instance()->dropTables( true ); // drop temp tables
 330     }
 331
 332     return !isAborted();
 333 }
 334
 335
 336 void
 337 ScanController::slotReadReady()
 338 {
 339     QString line;
 340
 341     m_dataMutex.lock();
 342
 343     while( m_scanner->readln( line, true, 0 ) != -1 ) {
 344         if( !line.startsWith( "exepath=" ) ) // skip binary location info from scanner
 345             m_xmlData += line;
 346     }
 347     m_dataMutex.unlock();
 348 }
 349
 350
 351 void
 352 ScanController::slotFileMoved( const QString &/*src*/, const QString &/*dest*/)
 353 {
 354     //why is this needed?  QBob, take a look at this
 355     /*
 356     if( m_incremental ) // pedantry
 357       {
 358         m_fileMapsMutex.lock();
 359         m_filesFound[ src ] = true;
 360         m_fileMapsMutex.unlock();
 361       }
 362     */
 363 }
 364
 365
 366 bool
 367 ScanController::startElement( const QString&, const QString& localName, const QString&, const QXmlAttributes& attrs )
 368 {
 369     // List of entity names:
 370     //
 371     // itemcount     Number of files overall
 372     // folder        Folder which is being processed
 373     // dud           Invalid audio file
 374     // tags          Valid audio file with metadata
 375     // playlist      Playlist file
 376     // image         Cover image
 377     // compilation   Folder to check for compilation
 378     // filesize      Size of the track in bytes
 379
 380     if( localName == "dud" || localName == "tags" || localName == "playlist" ) {
 381         incrementProgress();
 382     }
 383
 384     if( localName == "itemcount") {
 385         const int totalSteps = attrs.value( "count" ).toInt();
 386         debug() << "itemcount event: " << totalSteps << endl;
 387         setProgressTotalSteps( totalSteps );
 388     }
 389
 390     else if( localName == "tags") {
 391         MetaBundle bundle;
 392         bundle.setPath      ( attrs.value( "path" ) );
 393         bundle.setTitle     ( attrs.value( "title" ) );
 394         bundle.setArtist    ( attrs.value( "artist" ) );
 395         bundle.setComposer  ( attrs.value( "composer" ) );
 396         bundle.setAlbum     ( attrs.value( "album" ) );
 397         bundle.setComment   ( attrs.value( "comment" ) );
 398         bundle.setGenre     ( attrs.value( "genre" ) );
 399         bundle.setYear      ( attrs.value( "year" ).toInt() );
 400         bundle.setTrack     ( attrs.value( "track" ).toInt() );
 401         bundle.setDiscNumber( attrs.value( "discnumber" ).toInt() );
 402         bundle.setBpm       ( attrs.value( "bpm" ).toFloat() );
 403         bundle.setFileType( attrs.value( "filetype" ).toInt() );
 404         bundle.setUniqueId( attrs.value( "uniqueid" ) );
 405         bundle.setCompilation( attrs.value( "compilation" ).toInt() );
 406
 407         if( attrs.value( "audioproperties" ) == "true" ) {
 408             bundle.setBitrate   ( attrs.value( "bitrate" ).toInt() );
 409             bundle.setLength    ( attrs.value( "length" ).toInt() );
 410             bundle.setSampleRate( attrs.value( "samplerate" ).toInt() );
 411         }
 412
 413         if( !attrs.value( "filesize" ).isNull()
 414                 && !attrs.value( "filesize" ).isEmpty() )
 415         {
 416             bundle.setFilesize( attrs.value( "filesize" ).toInt() );
 417         }
 418
 419         CollectionDB::instance()->addSong( &bundle, m_incremental );
 420         if( !bundle.uniqueId().isEmpty() )
 421         {
 422             m_fileMapsMutex.lock();
 423             m_filesAdded[bundle.uniqueId()] = bundle.url().path();
 424             m_fileMapsMutex.unlock();
 425         }
 426
 427         m_scanCount++;
 428     }
 429
 430     else if( localName == "folder" ) {
 431         const QString folder = attrs.value( "path" );
 432         const QFileInfo info( folder );
 433
 434         // Update dir statistics for rescanning purposes
 435         if( info.exists() )
 436             CollectionDB::instance()->updateDirStats( folder, info.lastModified().toTime_t(), true);
 437
 438         if( m_incremental ) {
 439             m_foldersToRemove += folder;
 440         }
 441     }
 442
 443     else if( localName == "playlist" )
 444         QApplication::postEvent( PlaylistBrowser::instance(), new PlaylistFoundEvent( attrs.value( "path" ) ) );
 445
 446     else if( localName == "compilation" )
 447         CollectionDB::instance()->checkCompilations( attrs.value( "path" ), !m_incremental);
 448
 449     else if( localName == "image" ) {
 450         // Deserialize CoverBundle list
 451         QString data = attrs.value( "list" );
 452         QStringList list = data.split( "AMAROK_MAGIC" );
 453         QList< QPair<QString, QString> > covers;
 454
 455         for( int i = 0; i + 1 < list.count(); ) {
 456             covers += qMakePair( list[i], list[i + 1] );
 457             i += 2;
 458         }
 459
 460         CollectionDB::instance()->addImageToAlbum( attrs.value( "path" ), covers, CollectionDB::instance()->isConnected() );
 461     }
 462
 463     else if( localName == "embed" ) {
 464         CollectionDB::instance()->addEmbeddedImage( attrs.value( "path" ), attrs.value( "hash" ), attrs.value( "description" ) );
 465     }
 466
 467     return true;
 468 }
 469
 470
 471 void
 472 ScanController::customEvent( QCustomEvent* e )
 473 {
 474     if( e->type() == RestartEventType )
 475     {
 476         debug() << "RestartEvent received." << endl;
 477
 478         QFile log( Amarok::saveLocation( QString() ) + "collection_scan.log" );
 479         if ( !log.open( QIODevice::ReadOnly ) )
 480         ::warning() << "Failed opening log file " << log.fileName() << endl;
 481         else {
 482             QByteArray path = QByteArray(log.readAll());
 483             m_crashedFiles << QString::fromUtf8( path, path.length() );
 484
 485         }
 486
 487         m_dataMutex.lock();
 488         m_xmlData.clear();
 489         delete m_source;
 490         m_source = new QXmlInputSource();
 491         m_dataMutex.unlock();
 492
 493         delete m_reader;
 494         m_reader = new QXmlSimpleReader();
 495
 496         m_reader->setContentHandler( this );
 497         m_reader->parse( m_source, true );
 498
 499         delete m_scanner; // Reusing doesn't work, so we have to destroy and reinstantiate
 500         m_scanner = new Amarok::ProcIO();
 501         connect( m_scanner, SIGNAL( readReady( K3ProcIO* ) ), SLOT( slotReadReady() ) );
 502
 503         *m_scanner << "amarokcollectionscanner";
 504         *m_scanner << "--nocrashhandler"; // We want to be able to catch SIGSEGV
 505         if( m_incremental )
 506             *m_scanner << "-i";
 507
 508         *m_scanner << "-p";
 509         *m_scanner << "-s";
 510         m_scanner->start();
 511     }
 512     else
 513         ThreadManager::Job::customEvent( e );
 514 }
 515
 516
 517 #include "scancontroller.moc"