src/SharedFileList.cpp

   1 //
   2 // This file is part of the aMule Project.
   3 //
   4 // Copyright (c) 2003-2011 aMule Team ( admin@amule.org / http://www.amule.org )
   5 // Copyright (c) 2002-2011 Merkur ( devs@emule-project.net / http://www.emule-project.net )
   6 //
   7 // Any parts of this program derived from the xMule, lMule or eMule project,
   8 // or contributed by third-party developers are copyrighted by their
   9 // respective authors.
  10 //
  11 // This program is free software; you can redistribute it and/or modify
  12 // it under the terms of the GNU General Public License as published by
  13 // the Free Software Foundation; either version 2 of the License, or
  14 // (at your option) any later version.
  15 //
  16 // This program is distributed in the hope that it will be useful,
  17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 // GNU General Public License for more details.
  20 //
  21 // You should have received a copy of the GNU General Public License
  22 // along with this program; if not, write to the Free Software
  23 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
  24 //
  25
  26 #include "SharedFileList.h"     // Interface declarations  // Do_not_auto_remove
  27
  28 #include <protocol/Protocols.h>
  29 #include <protocol/kad/Constants.h>
  30 #include <tags/FileTags.h>
  31
  32 #include <wx/utils.h>
  33
  34 #include "Packet.h"             // Needed for CPacket
  35 #include "MemFile.h"            // Needed for CMemFile
  36 #include "ServerConnect.h"      // Needed for CServerConnect
  37 #include "KnownFileList.h"      // Needed for CKnownFileList
  38 #include "ThreadTasks.h"        // Needed for CThreadScheduler and CHasherTask
  39 #include "Preferences.h"        // Needed for thePrefs
  40 #include "DownloadQueue.h"      // Needed for CDownloadQueue
  41 #include "amule.h"              // Needed for theApp
  42 #include "PartFile.h"           // Needed for PartFile
  43 #include "Server.h"             // Needed for CServer
  44 #include "Statistics.h"         // Needed for theStats
  45 #include "Logger.h"
  46 #include <common/Format.h>
  47 #include <common/FileFunctions.h>
  48 #include "GuiEvents.h"          // Needed for Notify_*
  49 #include "SHAHashSet.h"         // Needed for CAICHHash
  50
  51
  52 #include "kademlia/kademlia/Kademlia.h"
  53 #include "kademlia/kademlia/Search.h"
  54 #include "ClientList.h"
  55
  56 typedef std::deque<CKnownFile*> KnownFileArray;
  57
  58 ///////////////////////////////////////////////////////////////////////////////
  59 // CPublishKeyword
  60
  61 class CPublishKeyword
  62 {
  63 public:
  64         CPublishKeyword(const wxString& rstrKeyword)
  65         {
  66                 m_strKeyword = rstrKeyword;
  67                 // min. keyword char is allowed to be < 3 in some cases (see also 'CSearchManager::getWords')
  68                 //ASSERT( rstrKeyword.GetLength() >= 3 );
  69                 wxASSERT( !rstrKeyword.IsEmpty() );
  70                 KadGetKeywordHash(rstrKeyword, &m_nKadID);
  71                 SetNextPublishTime(0);
  72                 SetPublishedCount(0);
  73         }
  74
  75         const Kademlia::CUInt128& GetKadID() const { return m_nKadID; }
  76         const wxString& GetKeyword() const { return m_strKeyword; }
  77         int GetRefCount() const { return m_aFiles.size(); }
  78         const KnownFileArray& GetReferences() const { return m_aFiles; }
  79
  80         uint32 GetNextPublishTime() const { return m_tNextPublishTime; }
  81         void SetNextPublishTime(uint32 tNextPublishTime) { m_tNextPublishTime = tNextPublishTime; }
  82
  83         uint32 GetPublishedCount() const { return m_uPublishedCount; }
  84         void SetPublishedCount(uint32 uPublishedCount) { m_uPublishedCount = uPublishedCount; }
  85         void IncPublishedCount() { m_uPublishedCount++; }
  86
  87         bool AddRef(CKnownFile* pFile) {
  88                 if (std::find(m_aFiles.begin(), m_aFiles.end(), pFile) != m_aFiles.end()) {
  89                         wxFAIL;
  90                         return false;
  91                 }
  92                 m_aFiles.push_back(pFile);
  93                 return true;
  94         }
  95
  96         int RemoveRef(CKnownFile* pFile) {
  97                 KnownFileArray::iterator it = std::find(m_aFiles.begin(), m_aFiles.end(), pFile);
  98                 if (it != m_aFiles.end()) {
  99                         m_aFiles.erase(it);
 100                 }
 101                 return m_aFiles.size();
 102         }
 103
 104         void RemoveAllReferences() {
 105                 m_aFiles.clear();
 106         }
 107
 108         void RotateReferences(unsigned iRotateSize) {
 109                 wxCHECK_RET(m_aFiles.size(), wxT("RotateReferences: Rotating empty array"));
 110
 111                 unsigned shift = (iRotateSize % m_aFiles.size());
 112                 std::rotate(m_aFiles.begin(), m_aFiles.begin() + shift, m_aFiles.end());
 113         }
 114
 115 protected:
 116         wxString m_strKeyword;
 117         Kademlia::CUInt128 m_nKadID;
 118         uint32 m_tNextPublishTime;
 119         uint32 m_uPublishedCount;
 120         KnownFileArray m_aFiles;
 121 };
 122
 123
 124 ///////////////////////////////////////////////////////////////////////////////
 125 // CPublishKeywordList
 126
 127 class CPublishKeywordList
 128 {
 129 public:
 130         CPublishKeywordList();
 131         ~CPublishKeywordList();
 132
 133         void AddKeyword(const wxString& keyword, CKnownFile *file);
 134         void AddKeywords(CKnownFile* pFile);
 135         void RemoveKeyword(const wxString& keyword, CKnownFile *file);
 136         void RemoveKeywords(CKnownFile* pFile);
 137         void RemoveAllKeywords();
 138
 139         void RemoveAllKeywordReferences();
 140         void PurgeUnreferencedKeywords();
 141
 142         int GetCount() const { return m_lstKeywords.size(); }
 143
 144         CPublishKeyword* GetNextKeyword();
 145         void ResetNextKeyword();
 146
 147         uint32 GetNextPublishTime() const { return m_tNextPublishKeywordTime; }
 148         void SetNextPublishTime(uint32 tNextPublishKeywordTime) { m_tNextPublishKeywordTime = tNextPublishKeywordTime; }
 149
 150 protected:
 151         // can't use a CMap - too many disadvantages in processing the 'list'
 152         //CTypedPtrMap<CMapStringToPtr, CString, CPublishKeyword*> m_lstKeywords;
 153         typedef std::list<CPublishKeyword*> CKeyWordList;
 154         CKeyWordList m_lstKeywords;
 155         CKeyWordList::iterator m_posNextKeyword;
 156         uint32 m_tNextPublishKeywordTime;
 157
 158         CPublishKeyword* FindKeyword(const wxString& rstrKeyword, CKeyWordList::iterator* ppos = NULL);
 159 };
 160
 161 CPublishKeywordList::CPublishKeywordList()
 162 {
 163         ResetNextKeyword();
 164         SetNextPublishTime(0);
 165 }
 166
 167 CPublishKeywordList::~CPublishKeywordList()
 168 {
 169         RemoveAllKeywords();
 170 }
 171
 172 CPublishKeyword* CPublishKeywordList::GetNextKeyword()
 173 {
 174         if (m_posNextKeyword == m_lstKeywords.end()) {
 175                 m_posNextKeyword = m_lstKeywords.begin();
 176                 if (m_posNextKeyword == m_lstKeywords.end()) {
 177                         return NULL;
 178                 }
 179         }
 180         return *m_posNextKeyword++;
 181 }
 182
 183 void CPublishKeywordList::ResetNextKeyword()
 184 {
 185         m_posNextKeyword = m_lstKeywords.begin();
 186 }
 187
 188 CPublishKeyword* CPublishKeywordList::FindKeyword(const wxString& rstrKeyword, CKeyWordList::iterator* ppos)
 189 {
 190         CKeyWordList::iterator it = m_lstKeywords.begin();
 191         for (; it != m_lstKeywords.end(); ++it) {
 192                 CPublishKeyword* pPubKw = *it;
 193                 if (pPubKw->GetKeyword() == rstrKeyword) {
 194                         if (ppos) {
 195                                 (*ppos) = it;
 196                         }
 197
 198                         return pPubKw;
 199                 }
 200         }
 201
 202         return NULL;
 203 }
 204
 205 void CPublishKeywordList::AddKeyword(const wxString& keyword, CKnownFile *file)
 206 {
 207         CPublishKeyword* pubKw = FindKeyword(keyword);
 208         if (pubKw == NULL) {
 209                 pubKw = new CPublishKeyword(keyword);
 210                 m_lstKeywords.push_back(pubKw);
 211                 SetNextPublishTime(0);
 212         }
 213         pubKw->AddRef(file);
 214 }
 215
 216 void CPublishKeywordList::AddKeywords(CKnownFile* pFile)
 217 {
 218         const Kademlia::WordList& wordlist = pFile->GetKadKeywords();
 219
 220         Kademlia::WordList::const_iterator it;
 221         for (it = wordlist.begin(); it != wordlist.end(); ++it) {
 222                 AddKeyword(*it, pFile);
 223         }
 224 }
 225
 226 void CPublishKeywordList::RemoveKeyword(const wxString& keyword, CKnownFile *file)
 227 {
 228         CKeyWordList::iterator pos;
 229         CPublishKeyword* pubKw = FindKeyword(keyword, &pos);
 230         if (pubKw != NULL) {
 231                 if (pubKw->RemoveRef(file) == 0) {
 232                         if (pos == m_posNextKeyword) {
 233                                 ++m_posNextKeyword;
 234                         }
 235                         m_lstKeywords.erase(pos);
 236                         delete pubKw;
 237                         SetNextPublishTime(0);
 238                 }
 239         }
 240 }
 241
 242 void CPublishKeywordList::RemoveKeywords(CKnownFile* pFile)
 243 {
 244         const Kademlia::WordList& wordlist = pFile->GetKadKeywords();
 245         Kademlia::WordList::const_iterator it;
 246         for (it = wordlist.begin(); it != wordlist.end(); ++it) {
 247                 RemoveKeyword(*it, pFile);
 248         }
 249 }
 250
 251
 252 void CPublishKeywordList::RemoveAllKeywords()
 253 {
 254         DeleteContents(m_lstKeywords);
 255         ResetNextKeyword();
 256         SetNextPublishTime(0);
 257 }
 258
 259
 260 void CPublishKeywordList::RemoveAllKeywordReferences()
 261 {
 262         CKeyWordList::iterator it = m_lstKeywords.begin();
 263         for (; it != m_lstKeywords.end(); ++it) {
 264                 (*it)->RemoveAllReferences();
 265         }
 266 }
 267
 268
 269 void CPublishKeywordList::PurgeUnreferencedKeywords()
 270 {
 271         CKeyWordList::iterator it = m_lstKeywords.begin();
 272         while (it != m_lstKeywords.end()) {
 273                 CPublishKeyword* pPubKw = *it;
 274                 if (pPubKw->GetRefCount() == 0) {
 275                         if (it == m_posNextKeyword) {
 276                                 ++m_posNextKeyword;
 277                         }
 278                         m_lstKeywords.erase(it++);
 279                         delete pPubKw;
 280                         SetNextPublishTime(0);
 281                 } else {
 282                         ++it;
 283                 }
 284         }
 285 }
 286
 287
 288 CSharedFileList::CSharedFileList(CKnownFileList* in_filelist){
 289         filelist = in_filelist;
 290         reloading = false;
 291         m_lastPublishED2K = 0;
 292         m_lastPublishED2KFlag = true;
 293         /* Kad Stuff */
 294         m_keywords = new CPublishKeywordList;
 295         m_currFileSrc = 0;
 296         m_currFileNotes = 0;
 297         m_lastPublishKadSrc = 0;
 298         m_lastPublishKadNotes = 0;
 299         m_currFileKey = 0;
 300 }
 301
 302
 303 CSharedFileList::~CSharedFileList()
 304 {
 305         delete m_keywords;
 306 }
 307
 308
 309 void CSharedFileList::FindSharedFiles()
 310 {
 311         /* Abort loading if we are shutting down. */
 312         if(theApp->IsOnShutDown()) {
 313                 return;
 314         }
 315
 316         // Clear statistics.
 317         theStats::ClearSharedFilesInfo();
 318
 319         // Reload shareddir.dat
 320         theApp->glob_prefs->ReloadSharedFolders();
 321
 322         {
 323                 wxMutexLocker lock(list_mut);
 324                 m_Files_map.clear();
 325         }
 326
 327         // All part files are automatically shared.
 328         for ( uint32 i = 0; i < theApp->downloadqueue->GetFileCount(); ++i ) {
 329                 CPartFile* file = theApp->downloadqueue->GetFileByIndex( i );
 330
 331                 if ( file->GetStatus(true) == PS_READY ) {
 332                         AddLogLineNS(CFormat(_("Adding file %s to shares"))
 333                                 % file->GetFullName().GetPrintable());
 334                         AddFile(file);
 335                 }
 336         }
 337
 338         // Create a list of all shared paths and weed out duplicates.
 339         std::list<CPath> sharedPaths;
 340
 341         // Global incoming dir and all category incoming directories are automatically shared.
 342         sharedPaths.push_back(thePrefs::GetIncomingDir());
 343         for (unsigned int i = 1;i < theApp->glob_prefs->GetCatCount(); ++i) {
 344                 sharedPaths.push_back(theApp->glob_prefs->GetCatPath(i));
 345         }
 346
 347         const thePrefs::PathList& shared = theApp->glob_prefs->shareddir_list;
 348         sharedPaths.insert(sharedPaths.end(), shared.begin(), shared.end());
 349
 350         sharedPaths.sort();
 351         sharedPaths.unique();
 352
 353         filelist->PrepareIndex();
 354         // Gathering is done in the foreground and can be slowed down severely by parallel background hashing.
 355         // So just store the hashing tasks for now.
 356         TaskList hashTasks;
 357         for (std::list<CPath>::iterator it = sharedPaths.begin(); it != sharedPaths.end(); ++it) {
 358                 AddFilesFromDirectory(*it, hashTasks);
 359         }
 360         filelist->ReleaseIndex();
 361
 362         // Now that the shared files are gathered feed the hashing tasks to the scheduler to start hashing.
 363         unsigned addedFiles = 0;
 364         for (TaskList::iterator it = hashTasks.begin(); it != hashTasks.end(); ++it) {
 365                 if (CThreadScheduler::AddTask(*it)) {
 366                         addedFiles++;
 367                 }
 368         }
 369
 370         if (addedFiles == 0) {
 371                 AddLogLineN(CFormat(wxPLURAL("Found %i known shared file", "Found %i known shared files", GetCount())) % GetCount());
 372
 373                 // Make sure the AICH-hashes are up to date.
 374                 CThreadScheduler::AddTask(new CAICHSyncTask());
 375         } else {
 376                 // New files, AICH thread will be run at the end of the hashing thread.
 377                 AddLogLineN(CFormat(wxPLURAL("Found %i known shared file, %i unknown", "Found %i known shared files, %i unknown", GetCount())) % GetCount() % addedFiles);
 378         }
 379 }
 380
 381
 382 // Checks if the dir a is the same as b. If they are, then logs the message and returns true.
 383 static bool CheckDirectory(const wxString& a, const CPath& b)
 384 {
 385         if (CPath(a).IsSameDir(b)) {
 386                 AddLogLineC(CFormat( _("ERROR: Attempted to share %s") ) % a);
 387
 388                 return true;
 389         }
 390
 391         return false;
 392 }
 393
 394
 395 unsigned CSharedFileList::AddFilesFromDirectory(const CPath& directory, TaskList & hashTasks)
 396 {
 397         // Do not allow these folders to be shared:
 398         //  - The .aMule folder
 399         //  - The Temp folder
 400         //  - The users home-dir
 401         if (CheckDirectory(wxGetHomeDir(), directory)) {
 402                 return 0;
 403         } else if (CheckDirectory(thePrefs::GetConfigDir(), directory)) {
 404                 return 0;
 405         } else if (CheckDirectory(thePrefs::GetTempDir().GetRaw(), directory)) {
 406                 return 0;
 407         }
 408
 409         if (!directory.DirExists()) {
 410                 AddLogLineNS(CFormat(_("Shared directory not found, skipping: %s"))
 411                         % directory.GetPrintable());
 412
 413                 return 0;
 414         }
 415
 416         CDirIterator::FileType searchFor = CDirIterator::FileNoHidden;
 417         if (thePrefs::ShareHiddenFiles()) {
 418                  searchFor = CDirIterator::File;
 419         }
 420
 421         unsigned knownFiles = 0;
 422         unsigned addedFiles = 0;
 423
 424         CDirIterator SharedDir(directory);
 425
 426         for (CPath fname = SharedDir.GetFirstFile(searchFor); fname.IsOk(); fname = SharedDir.GetNextFile()) {
 427                 CPath fullPath = directory.JoinPaths(fname);
 428
 429                 if (!fullPath.FileExists()) {
 430                         AddDebugLogLineN(logKnownFiles,
 431                                 CFormat(wxT("Shared file does not exist (possibly a broken link): %s")) % fullPath);
 432                         continue;
 433                 }
 434
 435                 AddDebugLogLineN(logKnownFiles,
 436                         CFormat(wxT("Found shared file: %s")) % fullPath);
 437
 438                 time_t fdate = CPath::GetModificationTime(fullPath);
 439                 sint64 fsize = fullPath.GetFileSize();
 440
 441                 // This will also catch files with too strict permissions.
 442                 if ((fdate == (time_t)-1) || (fsize == wxInvalidOffset)) {
 443                         AddDebugLogLineN(logKnownFiles,
 444                                 CFormat(wxT("Failed to retrieve modification time or size for '%s', skipping.")) % fullPath);
 445                         continue;
 446                 }
 447
 448                 if (fsize == 0) {
 449                         AddDebugLogLineN(logKnownFiles,
 450                                 CFormat(wxT("Skip zero size file '%s'")) % fullPath);
 451                         continue;
 452                 }
 453
 454
 455                 CKnownFile* toadd = filelist->FindKnownFile(fname, fdate, fsize);
 456                 if (toadd) {
 457                         knownFiles++;
 458                         if (AddFile(toadd)) {
 459                                 AddDebugLogLineN(logKnownFiles,
 460                                         CFormat(wxT("Added known file '%s' to shares"))
 461                                                 % fname);
 462
 463                                 toadd->SetFilePath(directory);
 464                         } else {
 465                                 AddDebugLogLineN(logKnownFiles,
 466                                         CFormat(wxT("File already shared, skipping: %s"))
 467                                                 % fname);
 468                         }
 469                 } else {
 470                         //not in knownfilelist - start adding thread to hash file
 471                         AddDebugLogLineN(logKnownFiles,
 472                                 CFormat(wxT("Hashing new unknown shared file '%s'")) % fname);
 473
 474                         hashTasks.push_back(new CHashingTask(directory, fname));
 475                         addedFiles++;
 476                 }
 477         }
 478
 479         if ((addedFiles == 0) && (knownFiles == 0)) {
 480                 AddLogLineN(CFormat(_("No shareable files found in directory: %s"))
 481                         % directory.GetPrintable());
 482         }
 483
 484         return addedFiles;
 485 }
 486
 487
 488 bool CSharedFileList::AddFile(CKnownFile* pFile)
 489 {
 490         wxASSERT(pFile->GetHashCount() == pFile->GetED2KPartHashCount());
 491
 492         wxMutexLocker lock(list_mut);
 493
 494         CKnownFileMap::value_type entry(pFile->GetFileHash(), pFile);
 495         if (m_Files_map.insert(entry).second) {
 496                 /* Keywords to publish on Kad */
 497                 m_keywords->AddKeywords(pFile);
 498                 theStats::AddSharedFile(pFile->GetFileSize());
 499                 return true;
 500         }
 501         return false;
 502 }
 503
 504
 505 void CSharedFileList::SafeAddKFile(CKnownFile* toadd, bool bOnlyAdd)
 506 {
 507         // TODO: Check if the file is already known - only with another date
 508
 509         if (AddFile(toadd)) {
 510                 Notify_SharedFilesShowFile(toadd);
 511         }
 512
 513         if (!bOnlyAdd && theApp->IsConnectedED2K()) {
 514                 // Publishing of files is not anymore handled here.
 515                 // Instead, the timer does it by itself.
 516                 m_lastPublishED2KFlag = true;
 517         }
 518 }
 519
 520
 521 // removes first occurrence of 'toremove' in 'list'
 522 void CSharedFileList::RemoveFile(CKnownFile* toremove){
 523         Notify_SharedFilesRemoveFile(toremove);
 524         wxMutexLocker lock(list_mut);
 525         if (m_Files_map.erase(toremove->GetFileHash()) > 0) {
 526                 theStats::RemoveSharedFile(toremove->GetFileSize());
 527         }
 528         /* This file keywords must not be published to kad anymore */
 529         m_keywords->RemoveKeywords(toremove);
 530 }
 531
 532
 533 void CSharedFileList::Reload()
 534 {
 535         // Madcat - Disable reloading if reloading already in progress.
 536         // Kry - Fixed to let non-english language users use the 'Reload' button :P
 537         // deltaHF - removed the old ugly button and changed the code to use the new small one
 538         // Kry - bah, let's use a var.
 539         if (!reloading) {
 540                 AddDebugLogLineN(logKnownFiles, wxT("Reload shared files"));
 541                 reloading = true;
 542                 Notify_SharedFilesRemoveAllItems();
 543
 544                 /* All Kad keywords must be removed */
 545                 m_keywords->RemoveAllKeywordReferences();
 546
 547                 /* Public identifiers must be erased as they might be invalid now */
 548                 m_PublicSharedDirNames.clear();
 549
 550                 FindSharedFiles();
 551
 552                 /* And now the unreferenced keywords must be removed also */
 553                 m_keywords->PurgeUnreferencedKeywords();
 554
 555                 Notify_SharedFilesShowFileList();
 556
 557                 reloading = false;
 558         }
 559 }
 560
 561
 562 const CKnownFile *CSharedFileList::GetFileByIndex(unsigned int index) const
 563 {
 564         wxMutexLocker lock(list_mut);
 565         if ( index >= m_Files_map.size() ) {
 566                 return NULL;
 567         }
 568         CKnownFileMap::const_iterator pos = m_Files_map.begin();
 569         std::advance(pos, index);
 570         return pos->second;
 571 }
 572
 573
 574 CKnownFile*     CSharedFileList::GetFileByID(const CMD4Hash& filehash)
 575 {
 576         wxMutexLocker lock(list_mut);
 577         CKnownFileMap::iterator it = m_Files_map.find(filehash);
 578
 579         if ( it != m_Files_map.end() ) {
 580                 return it->second;
 581         } else {
 582                 return NULL;
 583         }
 584 }
 585
 586 short CSharedFileList::GetFilePriorityByID(const CMD4Hash& filehash)
 587 {
 588         CKnownFile* tocheck = GetFileByID(filehash);
 589         if (tocheck)
 590                 return tocheck->GetUpPriority();
 591         else
 592                 return -10;     // file doesn't exist
 593 }
 594
 595
 596 void CSharedFileList::CopyFileList(std::vector<CKnownFile*>& out_list) const
 597 {
 598         wxMutexLocker lock(list_mut);
 599
 600         out_list.reserve(m_Files_map.size());
 601         for (
 602                 CKnownFileMap::const_iterator it = m_Files_map.begin();
 603                 it != m_Files_map.end();
 604                 ++it
 605                 ) {
 606                 out_list.push_back(it->second);
 607         }
 608 }
 609
 610
 611
 612 void CSharedFileList::UpdateItem(CKnownFile* toupdate)
 613 {
 614         Notify_SharedFilesUpdateItem(toupdate);
 615 }
 616
 617
 618 void CSharedFileList::GetSharedFilesByDirectory(const wxString& directory,
 619                                 CKnownFilePtrList& list)
 620 {
 621         wxMutexLocker lock(list_mut);
 622
 623         const CPath dir = CPath(directory);
 624         for (CKnownFileMap::iterator pos = m_Files_map.begin();
 625              pos != m_Files_map.end(); ++pos ) {
 626                 CKnownFile *cur_file = pos->second;
 627
 628                 if (dir.IsSameDir(cur_file->GetFilePath())) {
 629                         list.push_back(cur_file);
 630                 }
 631         }
 632 }
 633
 634 /* ---------------- Network ----------------- */
 635
 636 void CSharedFileList::ClearED2KPublishInfo(){
 637         CKnownFile* cur_file;
 638         m_lastPublishED2KFlag = true;
 639         wxMutexLocker lock(list_mut);
 640         for (CKnownFileMap::iterator pos = m_Files_map.begin(); pos != m_Files_map.end(); ++pos ) {
 641                 cur_file = pos->second;
 642                 cur_file->SetPublishedED2K(false);
 643         }
 644 }
 645
 646 void CSharedFileList::ClearKadSourcePublishInfo()
 647 {
 648         wxMutexLocker lock(list_mut);
 649         CKnownFile* cur_file;
 650         for (CKnownFileMap::iterator pos = m_Files_map.begin(); pos != m_Files_map.end(); ++pos ) {
 651                 cur_file = pos->second;
 652                 cur_file->SetLastPublishTimeKadSrc(0,0);
 653         }
 654 }
 655
 656 void CSharedFileList::RepublishFile(CKnownFile* pFile)
 657 {
 658         CServer* server = theApp->serverconnect->GetCurrentServer();
 659         if (server && (server->GetTCPFlags() & SRV_TCPFLG_COMPRESSION)) {
 660                 m_lastPublishED2KFlag = true;
 661                 pFile->SetPublishedED2K(false); // FIXME: this creates a wrong 'No' for the ed2k shared info in the listview until the file is shared again.
 662         }
 663 }
 664
 665 static uint8 GetRealPrio(uint8 in)
 666 {
 667         switch(in) {
 668                 case 4 : return 0;
 669                 case 0 : return 1;
 670                 case 1 : return 2;
 671                 case 2 : return 3;
 672                 case 3 : return 4;
 673         }
 674         return 0;
 675 }
 676
 677 static bool SortFunc( const CKnownFile* fileA, const CKnownFile* fileB )
 678 {
 679     return GetRealPrio(fileA->GetUpPriority()) < GetRealPrio(fileB->GetUpPriority());
 680 }
 681
 682 void CSharedFileList::SendListToServer(){
 683         std::vector<CKnownFile*> SortedList;
 684
 685         {
 686                 wxMutexLocker lock(list_mut);
 687
 688                 if (m_Files_map.empty() || !theApp->IsConnectedED2K() ) {
 689                         return;
 690                 }
 691
 692                 // Getting a sorted list of the non-published files.
 693                 SortedList.reserve( m_Files_map.size() );
 694
 695                 CKnownFileMap::iterator it = m_Files_map.begin();
 696                 for ( ; it != m_Files_map.end(); ++it ) {
 697                         if (!it->second->GetPublishedED2K()) {
 698                                 SortedList.push_back( it->second );
 699                         }
 700                 }
 701         }
 702
 703         std::sort( SortedList.begin(), SortedList.end(), SortFunc );
 704
 705         // Limits for the server.
 706
 707         CServer* server = theApp->serverconnect->GetCurrentServer();
 708
 709         uint32 limit = server ? server->GetSoftFiles() : 0;
 710         if( limit == 0 || limit > 200 ) {
 711                 limit = 200;
 712         }
 713
 714         if( (uint32)SortedList.size() < limit ) {
 715                 limit = SortedList.size();
 716                 if (limit == 0) {
 717                         m_lastPublishED2KFlag = false;
 718                         return;
 719                 }
 720         }
 721
 722         CMemFile files;
 723
 724         // Files sent.
 725         files.WriteUInt32(limit);
 726
 727         uint16 count = 0;
 728         // Add to packet
 729         std::vector<CKnownFile*>::iterator sorted_it = SortedList.begin();
 730         for ( ; (sorted_it != SortedList.end()) && (count < limit); ++sorted_it ) {
 731                 CKnownFile* file = *sorted_it;
 732                 if (!file->IsLargeFile() || (server && server->SupportsLargeFilesTCP())) {
 733                         file->CreateOfferedFilePacket(&files, server, NULL);
 734                 }
 735                 file->SetPublishedED2K(true);
 736                 ++count;
 737         }
 738
 739         wxASSERT(count == limit);
 740
 741         CPacket* packet = new CPacket(files, OP_EDONKEYPROT, OP_OFFERFILES);
 742         // compress packet
 743         //   - this kind of data is highly compressable (N * (1 MD4 and at least 3 string meta data tags and 1 integer meta data tag))
 744         //   - the min. amount of data needed for one published file is ~100 bytes
 745         //   - this function is called once when connecting to a server and when a file becomes shareable - so, it's called rarely.
 746         //   - if the compressed size is still >= the original size, we send the uncompressed packet
 747         // therefor we always try to compress the packet
 748         if (server->GetTCPFlags() & SRV_TCPFLG_COMPRESSION){
 749                 packet->PackPacket();
 750         }
 751
 752         theStats::AddUpOverheadServer(packet->GetPacketSize());
 753         theApp->serverconnect->SendPacket(packet,true);
 754 }
 755
 756
 757 void CSharedFileList::Process()
 758 {
 759         Publish();
 760         if( !m_lastPublishED2KFlag || ( ::GetTickCount() - m_lastPublishED2K < ED2KREPUBLISHTIME ) ) {
 761                 return;
 762         }
 763         SendListToServer();
 764         m_lastPublishED2K = ::GetTickCount();
 765 }
 766
 767 void CSharedFileList::Publish()
 768 {
 769         // Variables to save cpu.
 770         unsigned int tNow = time(NULL);
 771         bool IsFirewalled = theApp->IsFirewalled();
 772
 773         if( Kademlia::CKademlia::IsConnected() && ( !IsFirewalled || ( IsFirewalled && theApp->clientlist->GetBuddyStatus() == Connected)) && GetCount() && Kademlia::CKademlia::GetPublish()) {
 774                 //We are connected to Kad. We are either open or have a buddy. And Kad is ready to start publishing.
 775
 776                 if( Kademlia::CKademlia::GetTotalStoreKey() < KADEMLIATOTALSTOREKEY) {
 777
 778                         //We are not at the max simultaneous keyword publishes
 779                         if (tNow >= m_keywords->GetNextPublishTime()) {
 780
 781                                 //Enough time has passed since last keyword publish
 782
 783                                 //Get the next keyword which has to be (re)-published
 784                                 CPublishKeyword* pPubKw = m_keywords->GetNextKeyword();
 785                                 if (pPubKw) {
 786
 787                                         //We have the next keyword to check if it can be published
 788
 789                                         //Debug check to make sure things are going well.
 790                                         wxASSERT( pPubKw->GetRefCount() != 0 );
 791
 792                                         if (tNow >= pPubKw->GetNextPublishTime()) {
 793                                                 //This keyword can be published.
 794                                                 Kademlia::CSearch* pSearch = Kademlia::CSearchManager::PrepareLookup(Kademlia::CSearch::STOREKEYWORD, false, pPubKw->GetKadID());
 795                                                 if (pSearch) {
 796                                                         //pSearch was created. Which means no search was already being done with this HashID.
 797                                                         //This also means that it was checked to see if network load wasn't a factor.
 798
 799                                                         //This sets the filename into the search object so we can show it in the gui.
 800                                                         pSearch->SetFileName(pPubKw->GetKeyword());
 801
 802                                                         //Add all file IDs which relate to the current keyword to be published
 803                                                         const KnownFileArray& aFiles = pPubKw->GetReferences();
 804                                                         uint32 count = 0;
 805                                                         for (unsigned int f = 0; f < aFiles.size(); ++f) {
 806
 807                                                                 //Only publish complete files as someone else should have the full file to publish these keywords.
 808                                                                 //As a side effect, this may help reduce people finding incomplete files in the network.
 809                                                                 if( !aFiles[f]->IsPartFile() ) {
 810                                                                         count++;
 811                                                                         pSearch->AddFileID(Kademlia::CUInt128(aFiles[f]->GetFileHash().GetHash()));
 812                                                                         if( count > 150 ) {
 813                                                                                 //We only publish up to 150 files per keyword publish then rotate the list.
 814                                                                                 pPubKw->RotateReferences(f);
 815                                                                                 break;
 816                                                                         }
 817                                                                 }
 818                                                         }
 819
 820                                                         if( count ) {
 821                                                                 //Start our keyword publish
 822                                                                 pPubKw->SetNextPublishTime(tNow+(KADEMLIAREPUBLISHTIMEK));
 823                                                                 pPubKw->IncPublishedCount();
 824                                                                 Kademlia::CSearchManager::StartSearch(pSearch);
 825                                                         } else {
 826                                                                 //There were no valid files to publish with this keyword.
 827                                                                 delete pSearch;
 828                                                         }
 829                                                 }
 830                                         }
 831                                 }
 832                                 m_keywords->SetNextPublishTime(KADEMLIAPUBLISHTIME+tNow);
 833                         }
 834                 }
 835
 836                 if( Kademlia::CKademlia::GetTotalStoreSrc() < KADEMLIATOTALSTORESRC) {
 837                         if(tNow >= m_lastPublishKadSrc) {
 838                                 if(m_currFileSrc > GetCount()) {
 839                                         m_currFileSrc = 0;
 840                                 }
 841                                 CKnownFile* pCurKnownFile = const_cast<CKnownFile*>(GetFileByIndex(m_currFileSrc));
 842                                 if(pCurKnownFile) {
 843                                         if(pCurKnownFile->PublishSrc()) {
 844                                                 Kademlia::CUInt128 kadFileID;
 845                                                 kadFileID.SetValueBE(pCurKnownFile->GetFileHash().GetHash());
 846                                                 if(Kademlia::CSearchManager::PrepareLookup(Kademlia::CSearch::STOREFILE, true, kadFileID )==NULL) {
 847                                                         pCurKnownFile->SetLastPublishTimeKadSrc(0,0);
 848                                                 }
 849                                         }
 850                                 }
 851                                 m_currFileSrc++;
 852
 853                                 // even if we did not publish a source, reset the timer so that this list is processed
 854                                 // only every KADEMLIAPUBLISHTIME seconds.
 855                                 m_lastPublishKadSrc = KADEMLIAPUBLISHTIME+tNow;
 856                         }
 857                 }
 858
 859                 if( Kademlia::CKademlia::GetTotalStoreNotes() < KADEMLIATOTALSTORENOTES) {
 860                         if(tNow >= m_lastPublishKadNotes) {
 861                                 if(m_currFileNotes > GetCount()) {
 862                                         m_currFileNotes = 0;
 863                                 }
 864                                 CKnownFile* pCurKnownFile = const_cast<CKnownFile*>(GetFileByIndex(m_currFileNotes));
 865                                 if(pCurKnownFile) {
 866                                         if(pCurKnownFile->PublishNotes()) {
 867                                                 Kademlia::CUInt128 kadFileID;
 868                                                 kadFileID.SetValueBE(pCurKnownFile->GetFileHash().GetHash());
 869                                                 if(Kademlia::CSearchManager::PrepareLookup(Kademlia::CSearch::STORENOTES, true, kadFileID )==NULL)
 870                                                         pCurKnownFile->SetLastPublishTimeKadNotes(0);
 871                                         }
 872                                 }
 873                                 m_currFileNotes++;
 874
 875                                 // even if we did not publish a source, reset the timer so that this list is processed
 876                                 // only every KADEMLIAPUBLISHTIME seconds.
 877                                 m_lastPublishKadNotes = KADEMLIAPUBLISHTIME+tNow;
 878                         }
 879                 }
 880         }
 881 }
 882
 883
 884 void CSharedFileList::AddKeywords(CKnownFile* pFile)
 885 {
 886         m_keywords->AddKeywords(pFile);
 887 }
 888
 889
 890 void CSharedFileList::RemoveKeywords(CKnownFile* pFile)
 891 {
 892         m_keywords->RemoveKeywords(pFile);
 893 }
 894
 895
 896 bool CSharedFileList::RenameFile(CKnownFile* file, const CPath& newName)
 897 {
 898         if (file->IsPartFile()) {
 899                 CPartFile* pfile = dynamic_cast<CPartFile*>(file);
 900
 901                 if (file->GetStatus() != PS_COMPLETING) {
 902                         pfile->SetFileName(newName);
 903                         pfile->SavePartFile();
 904
 905                         Notify_SharedFilesUpdateItem(file);
 906                         Notify_DownloadCtrlUpdateItem(file);
 907
 908                         return true;
 909                 }
 910         } else {
 911                 CPath oldPath = file->GetFilePath().JoinPaths(file->GetFileName());
 912                 CPath newPath = file->GetFilePath().JoinPaths(newName);
 913
 914                 if (CPath::RenameFile(oldPath, newPath)) {
 915                         // Must create a copy of the word list because:
 916                         // 1) it will be reset on SetFileName()
 917                         // 2) we will want to edit it
 918                         Kademlia::WordList oldwords = file->GetKadKeywords();
 919                         file->SetFileName(newName);
 920                         theApp->knownfiles->Save();
 921                         UpdateItem(file);
 922                         RepublishFile(file);
 923
 924                         const Kademlia::WordList& newwords = file->GetKadKeywords();
 925                         Kademlia::WordList::iterator itold;
 926                         Kademlia::WordList::const_iterator itnew;
 927                         // compare keywords in old and new names
 928                         for (itnew = newwords.begin(); itnew != newwords.end(); ++itnew) {
 929                                 for (itold = oldwords.begin(); itold != oldwords.end(); ++itold) {
 930                                         if (*itold == *itnew) {
 931                                                 break;
 932                                         }
 933                                 }
 934                                 if (itold != oldwords.end()) {
 935                                         // Remove keyword from old name which also exist in new name
 936                                         oldwords.erase(itold);
 937                                 } else {
 938                                         // This is a new keyword not present in the old name
 939                                         m_keywords->AddKeyword(*itnew, file);
 940                                 }
 941                         }
 942                         // Remove all remaining old keywords not present in the new name
 943                         for (itold = oldwords.begin(); itold != oldwords.end(); ++itold) {
 944                                 m_keywords->RemoveKeyword(*itold, file);
 945                         }
 946
 947                         Notify_DownloadCtrlUpdateItem(file);
 948                         Notify_SharedFilesUpdateItem(file);
 949
 950                         return true;
 951                 }
 952         }
 953
 954         return false;
 955 }
 956
 957
 958 const CPath* CSharedFileList::GetDirForPublicSharedDirName(const wxString& strSharedDir) const
 959 {
 960         StringPathMap::const_iterator it = m_PublicSharedDirNames.find(strSharedDir);
 961
 962         if (it != m_PublicSharedDirNames.end()) {
 963                 return &(it->second);
 964         } else {
 965                 return NULL;
 966         }
 967 }
 968
 969
 970 wxString CSharedFileList::GetPublicSharedDirName(const CPath& dir)
 971 {
 972         // safety check: is the directory supposed to be shared after all?
 973         if (!IsShared(dir))     {
 974                 wxFAIL;
 975                 return wxT("");
 976         }
 977         // check if the public name for the directory is cached in our Map
 978         StringPathMap::const_iterator it;
 979         for (it = m_PublicSharedDirNames.begin(); it != m_PublicSharedDirNames.end(); ++it) {
 980                 if (it->second.IsSameDir(dir)) {
 981                         // public name for directory was determined earlier
 982                         return it->first;
 983                 }
 984         }
 985
 986         // we store the path separator (forward or back slash) for quick access
 987         wxChar cPathSepa = wxFileName::GetPathSeparator();
 988
 989         // determine and cache the public name for "dir" ...
 990         // We need to use the 'raw' filename, so the receiving client can recognize it.
 991         wxString strDirectoryTmp = dir.GetRaw();
 992         if (strDirectoryTmp.EndsWith(&cPathSepa)) {
 993                 strDirectoryTmp.RemoveLast();
 994         }
 995
 996         wxString strPublicName;
 997         int iPos;
 998         // check all the subdirectories in the path for being shared
 999         // the public name will consist of these concatenated
1000         while ((iPos = strDirectoryTmp.Find( cPathSepa, true )) != wxNOT_FOUND) {
1001                 strPublicName = strDirectoryTmp.Right(strDirectoryTmp.Length() - iPos) + strPublicName;
1002                 strDirectoryTmp.Truncate(iPos);
1003                 if (!IsShared(CPath(strDirectoryTmp)))
1004                         break;
1005         }
1006         if (!strPublicName.IsEmpty()) {
1007                 // remove first path separator ???
1008                 wxASSERT( strPublicName.GetChar(0) == cPathSepa );
1009                 strPublicName = strPublicName.Right(strPublicName.Length() - 1);
1010         } else {
1011                 // must be a rootdirectory on Windos
1012                 wxASSERT( strDirectoryTmp.Length() == 2 );
1013                 strPublicName = strDirectoryTmp;
1014         }
1015         // we have the name, make sure it is unique by appending an index if necessary
1016         if (m_PublicSharedDirNames.find(strPublicName) != m_PublicSharedDirNames.end()) {
1017                 wxString strUniquePublicName;
1018                 for (iPos = 2; ; ++iPos) {
1019                         strUniquePublicName = CFormat(wxT("%s_%i")) % strPublicName % iPos;
1020
1021                         if (m_PublicSharedDirNames.find(strUniquePublicName) == m_PublicSharedDirNames.end()) {
1022                                 AddDebugLogLineN(logClient, CFormat(wxT("Using public name '%s' for directory '%s'"))
1023                                                             % strUniquePublicName
1024                                                             % dir.GetPrintable());
1025                                 m_PublicSharedDirNames.insert(std::pair<wxString, CPath> (strUniquePublicName, dir));
1026                                 return strUniquePublicName;
1027                         }
1028                         // This is from eMule and it checks if there are more than 200 shared folders with the same public name.
1029                         // The condition can be true if many shared subfolders with the same name exist in folders that are not
1030                         // shared. So they get the names of each shared subfolders concatenated. But those might all be the same!
1031                         // It's here for safety reasons so we should not run out of memory.
1032                         else if (iPos > 200)  // Only 200 identical names are indexed.
1033                         {
1034                                 wxASSERT( false );
1035                                 return wxT("");
1036                         }
1037                 }
1038         } else {
1039                 AddDebugLogLineN(logClient, CFormat(wxT("Using public name '%s' for directory '%s'")) % strPublicName % dir.GetPrintable());
1040                 m_PublicSharedDirNames.insert(std::pair<wxString, CPath> (strPublicName, dir));
1041                 return strPublicName;
1042         }
1043 }
1044
1045
1046 bool CSharedFileList::IsShared(const CPath& path) const
1047 {
1048         if( path.IsDir(CPath::exists) ) {
1049                 // check if it's a shared folder
1050                 const unsigned folderCount = theApp->glob_prefs->shareddir_list.size();
1051                 for (unsigned i = 0; i < folderCount; ++i) {
1052                         if (path.IsSameDir(theApp->glob_prefs->shareddir_list[i])) {
1053                                 return true;
1054                         }
1055                 }
1056
1057                 // check if it's one of the categories folders (category 0 = incoming)
1058                 for (unsigned i = 0; i < theApp->glob_prefs->GetCatCount(); ++i) {
1059                         if (path.IsSameDir(theApp->glob_prefs->GetCategory(i)->path)) {
1060                                 return true;
1061                         }
1062                 }
1063         }
1064
1065         return false;
1066 }
1067
1068
1069 void CSharedFileList::CheckAICHHashes(const std::list<CAICHHash>& hashes)
1070 {
1071         wxMutexLocker locker(list_mut);
1072
1073         // Now we check that all files which are in the sharedfilelist have a
1074         // corresponding hash in our list. Those how don't are queued for hashing.
1075         CKnownFileMap::iterator it = m_Files_map.begin();
1076         for (; it != m_Files_map.end(); ++it) {
1077                 const CKnownFile* file = it->second;
1078
1079                 if (file->IsPartFile() == false) {
1080                         CAICHHashSet* hashset = file->GetAICHHashset();
1081
1082                         if (hashset->GetStatus() == AICH_HASHSETCOMPLETE) {
1083                                 if (std::find(hashes.begin(), hashes.end(), hashset->GetMasterHash()) != hashes.end()) {
1084                                         continue;
1085                                 }
1086                         }
1087
1088                         hashset->SetStatus(AICH_ERROR);
1089
1090                         CThreadScheduler::AddTask(new CHashingTask(file));
1091                 }
1092         }
1093
1094 }
1095
1096 // File_checked_for_headers