src/KnownFileList.cpp

   1 //
   2 // This file is part of the aMule Project.
   3 //
   4 // Copyright (c) 2003-2011 aMule Team ( admin@amule.org / http://www.amule.org )
   5 // Copyright (c) 2002-2011 Merkur ( devs@emule-project.net / http://www.emule-project.net )
   6 //
   7 // Any parts of this program derived from the xMule, lMule or eMule project,
   8 // or contributed by third-party developers are copyrighted by their
   9 // respective authors.
  10 //
  11 // This program is free software; you can redistribute it and/or modify
  12 // it under the terms of the GNU General Public License as published by
  13 // the Free Software Foundation; either version 2 of the License, or
  14 // (at your option) any later version.
  15 //
  16 // This program is distributed in the hope that it will be useful,
  17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 // GNU General Public License for more details.
  20 //
  21 // You should have received a copy of the GNU General Public License
  22 // along with this program; if not, write to the Free Software
  23 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA
  24 //
  25
  26
  27 #include "KnownFileList.h"      // Interface declarations
  28
  29 #include <common/DataFileVersion.h>
  30
  31 #include <memory>               // Do_not_auto_remove (lionel's Mac, 10.3)
  32 #include "PartFile.h"           // Needed for CPartFile
  33 #include "amule.h"
  34 #include "Logger.h"
  35 #include "MemFile.h"
  36 #include "ScopedPtr.h"
  37 #include "SearchList.h"         // Needed for UpdateSearchFileByHash
  38 #include <common/Format.h>
  39 #include "Preferences.h"        // Needed for thePrefs
  40
  41
  42 // This function is inlined for performance
  43 inline bool CKnownFileList::KnownFileMatches(
  44         CKnownFile *knownFile,
  45         const CPath& filename,
  46         uint32 in_date,
  47         uint64 in_size) const
  48 {
  49         return
  50                 (knownFile->GetLastChangeDatetime() == (time_t)in_date) &&
  51                 (knownFile->GetFileSize() == in_size) &&
  52                 (knownFile->GetFileName() == filename);
  53 }
  54
  55
  56 CKnownFileList::CKnownFileList()
  57 {
  58         accepted = 0;
  59         requested = 0;
  60         transferred = 0;
  61         m_filename = wxT("known.met");
  62         m_knownSizeMap = NULL;
  63         m_duplicateSizeMap = NULL;
  64         Init();
  65 }
  66
  67
  68 CKnownFileList::~CKnownFileList()
  69 {
  70         Clear();
  71 }
  72
  73
  74 bool CKnownFileList::Init()
  75 {
  76         CFile file;
  77
  78         CPath fullpath = CPath(thePrefs::GetConfigDir() + m_filename);
  79         if (!fullpath.FileExists()) {
  80                 // This is perfectly normal. The file was probably either
  81                 // deleted, or this is the first time running aMule.
  82                 return false;
  83         }
  84
  85         if (!file.Open(fullpath)) {
  86                 AddLogLineC(CFormat(_("WARNING: %s cannot be opened.")) % m_filename);
  87                 return false;
  88         }
  89
  90         try {
  91                 uint8 version = file.ReadUInt8();
  92                 if ((version != MET_HEADER) && (version != MET_HEADER_WITH_LARGEFILES)) {
  93                         AddLogLineC(_("WARNING: Known file list corrupted, contains invalid header."));
  94                         return false;
  95                 }
  96
  97                 wxMutexLocker sLock(list_mut);
  98                 uint32 RecordsNumber = file.ReadUInt32();
  99                 AddDebugLogLineN(logKnownFiles, CFormat(wxT("Reading %i known files from file format 0x%2.2x."))
 100                         % RecordsNumber % version);
 101                 for (uint32 i = 0; i < RecordsNumber; i++) {
 102                         CScopedPtr<CKnownFile> record;
 103                         if (record->LoadFromFile(&file)) {
 104                                 AddDebugLogLineN(logKnownFiles,
 105                                         CFormat(wxT("Known file read: %s")) % record->GetFileName());
 106                                 Append(record.release());
 107                         } else {
 108                                 AddLogLineC(_("Failed to load entry in known file list, file may be corrupt"));
 109                         }
 110                 }
 111                 AddDebugLogLineN(logKnownFiles, wxT("Finished reading known files"));
 112
 113                 return true;
 114         } catch (const CInvalidPacket& e) {
 115                 AddLogLineC(_("Invalid entry in known file list, file may be corrupt: ") + e.what());
 116         } catch (const CSafeIOException& e) {
 117                 AddLogLineC(CFormat(_("IO error while reading %s file: %s")) % m_filename % e.what());
 118         }
 119
 120         return false;
 121 }
 122
 123
 124 void CKnownFileList::Save()
 125 {
 126         CFile file(thePrefs::GetConfigDir() + m_filename, CFile::write_safe);
 127         if (!file.IsOpened()) {
 128                 return;
 129         }
 130
 131         wxMutexLocker sLock(list_mut);
 132         AddDebugLogLineN(logKnownFiles, CFormat(wxT("start saving %s")) % m_filename);
 133
 134         try {
 135                 // Kry - This is the version, but we don't know it till
 136                 // we know if any largefile is saved. This allows the list
 137                 // to be compatible with previous versions.
 138                 bool bContainsAnyLargeFiles = false;
 139                 file.WriteUInt8(0);
 140
 141                 file.WriteUInt32(m_knownFileMap.size() + m_duplicateFileList.size());
 142
 143                 // Duplicates handling. Duplicates needs to be saved first,
 144                 // since it is the last entry that gets used.
 145                 KnownFileList::iterator itDup = m_duplicateFileList.begin();
 146                 for ( ; itDup != m_duplicateFileList.end(); ++itDup ) {
 147                         (*itDup)->WriteToFile(&file);
 148                         if ((*itDup)->IsLargeFile()) {
 149                                 bContainsAnyLargeFiles = true;
 150                         }
 151                 }
 152
 153                 CKnownFileMap::iterator it = m_knownFileMap.begin();
 154                 for (; it != m_knownFileMap.end(); ++it) {
 155                         it->second->WriteToFile(&file);
 156                         if (it->second->IsLargeFile()) {
 157                                 bContainsAnyLargeFiles = true;
 158                         }
 159                 }
 160
 161                 file.Seek(0);
 162                 file.WriteUInt8(bContainsAnyLargeFiles ? MET_HEADER_WITH_LARGEFILES : MET_HEADER);
 163                 file.Close();
 164         } catch (const CIOFailureException& e) {
 165                 AddLogLineC(CFormat(_("Error while saving %s file: %s")) % m_filename % e.what());
 166         }
 167         AddDebugLogLineN(logKnownFiles, CFormat(wxT("finished saving %s")) % m_filename);
 168 }
 169
 170
 171 void CKnownFileList::Clear()
 172 {
 173         wxMutexLocker sLock(list_mut);
 174
 175         DeleteContents(m_knownFileMap);
 176         DeleteContents(m_duplicateFileList);
 177         ReleaseIndex();
 178 }
 179
 180
 181 CKnownFile* CKnownFileList::FindKnownFile(
 182         const CPath& filename,
 183         time_t in_date,
 184         uint64 in_size)
 185 {
 186         wxMutexLocker sLock(list_mut);
 187
 188         if (m_knownSizeMap) {
 189                 std::pair<KnownFileSizeMap::const_iterator, KnownFileSizeMap::const_iterator> p;
 190                 p = m_knownSizeMap->equal_range((uint32) in_size);
 191                 for (KnownFileSizeMap::const_iterator it = p.first; it != p.second; ++it) {
 192                         CKnownFile *cur_file = it->second;
 193                         if (KnownFileMatches(cur_file, filename, in_date, in_size)) {
 194                                 return cur_file;
 195                         }
 196                 }
 197         } else {
 198                 for (CKnownFileMap::const_iterator it = m_knownFileMap.begin();
 199                          it != m_knownFileMap.end(); ++it) {
 200                         CKnownFile *cur_file = it->second;
 201                         if (KnownFileMatches(cur_file, filename, in_date, in_size)) {
 202                                 return cur_file;
 203                         }
 204                 }
 205         }
 206
 207         return IsOnDuplicates(filename, in_date, in_size);
 208 }
 209
 210
 211 CKnownFile *CKnownFileList::IsOnDuplicates(
 212         const CPath& filename,
 213         uint32 in_date,
 214         uint64 in_size) const
 215 {
 216         if (m_duplicateSizeMap) {
 217                 std::pair<KnownFileSizeMap::const_iterator, KnownFileSizeMap::const_iterator> p;
 218                 p = m_duplicateSizeMap->equal_range((uint32) in_size);
 219                 for (KnownFileSizeMap::const_iterator it = p.first; it != p.second; ++it) {
 220                         CKnownFile *cur_file = it->second;
 221                         if (KnownFileMatches(cur_file, filename, in_date, in_size)) {
 222                                 return cur_file;
 223                         }
 224                 }
 225         } else {
 226                 for (KnownFileList::const_iterator it = m_duplicateFileList.begin();
 227                          it != m_duplicateFileList.end(); ++it) {
 228                         CKnownFile *cur_file = *it;
 229                         if (KnownFileMatches(cur_file, filename, in_date, in_size)) {
 230                                 return cur_file;
 231                         }
 232                 }
 233         }
 234         return NULL;
 235 }
 236
 237
 238 CKnownFile* CKnownFileList::FindKnownFileByID(const CMD4Hash& hash)
 239 {
 240         wxMutexLocker sLock(list_mut);
 241
 242         if (!hash.IsEmpty()) {
 243                 if (m_knownFileMap.find(hash) != m_knownFileMap.end()) {
 244                         return m_knownFileMap[hash];
 245                 } else {
 246                         return NULL;
 247                 }
 248         }
 249         return NULL;
 250
 251 }
 252
 253
 254 bool CKnownFileList::SafeAddKFile(CKnownFile* toadd, bool afterHashing)
 255 {
 256         bool ret;
 257         {
 258                 wxMutexLocker sLock(list_mut);
 259                 ret = Append(toadd, afterHashing);
 260         }
 261         if (ret) {
 262                 theApp->searchlist->UpdateSearchFileByHash(toadd->GetFileHash());
 263         }
 264         return ret;
 265 }
 266
 267
 268 bool CKnownFileList::Append(CKnownFile *Record, bool afterHashing)
 269 {
 270         if (Record->GetFileSize() > 0) {
 271                 // sanity check if the number of part hashes is correct here
 272                 if (Record->GetHashCount() != Record->GetED2KPartHashCount()) {
 273                         AddDebugLogLineC(logKnownFiles, CFormat(wxT("%s with size %d should have %d part hashes, but only %d are available"))
 274                                 % Record->GetFileName().GetPrintable() % Record->GetFileSize() % Record->GetED2KPartHashCount() % Record->GetHashCount());
 275                         return false;
 276                 }
 277                 const CMD4Hash& tkey = Record->GetFileHash();
 278                 CKnownFileMap::iterator it = m_knownFileMap.find(tkey);
 279                 if (it == m_knownFileMap.end()) {
 280                         m_knownFileMap[tkey] = Record;
 281                         return true;
 282                 } else {
 283                         CKnownFile *existing = it->second;
 284                         if (KnownFileMatches(Record, existing->GetFileName(), existing->GetLastChangeDatetime(), existing->GetFileSize())) {
 285                                 // The file is already on the list, ignore it.
 286                                 AddDebugLogLineN(logKnownFiles, CFormat(wxT("%s is already on the list")) % Record->GetFileName().GetPrintable());
 287                                 return false;
 288                         } else if (IsOnDuplicates(Record->GetFileName(), Record->GetLastChangeDatetime(), Record->GetFileSize())) {
 289                                 // The file is on the duplicates list, ignore it.
 290                                 // Should not happen, at least not after hashing. Or why did it get hashed in the first place then?
 291                                 AddDebugLogLineN(logKnownFiles, CFormat(wxT("%s is on the duplicates list")) % Record->GetFileName().GetPrintable());
 292                                 return false;
 293                         } else {
 294                                 if (afterHashing && existing->GetFileSize() == Record->GetFileSize()) {
 295                                         // We just hashed a "new" shared file and find it's already known under a different name or date.
 296                                         // Guess what - it was probably renamed or touched.
 297                                         // So copy over all properties from the existing known file and just keep name/date.
 298                                         time_t newDate = Record->GetLastChangeDatetime();
 299                                         CPath newName = Record->GetFileName();
 300                                         CMemFile f;
 301                                         existing->WriteToFile(&f);
 302                                         f.Reset();
 303                                         if (!Record->LoadFromFile(&f)) {
 304                                                 // this also shouldn't happen
 305                                                 AddDebugLogLineC(logKnownFiles, CFormat(wxT("error copying known file: existing: %s %d %d %d  Record: %s %d %d %d"))
 306                                                         % existing->GetFileName().GetPrintable() % existing->GetFileSize() % existing->GetED2KPartHashCount() % existing->GetHashCount()
 307                                                         % Record->GetFileName().GetPrintable() % Record->GetFileSize() % Record->GetED2KPartHashCount() % Record->GetHashCount());
 308                                                 return false;
 309                                         }
 310                                         Record->SetLastChangeDatetime(newDate);
 311                                         Record->SetFileName(newName);
 312                                 }
 313                                 // The file is a duplicated hash. Add THE OLD ONE to the duplicates list.
 314                                 // (This is used when reading the known file list where the duplicates are stored in front.)
 315                                 m_duplicateFileList.push_back(existing);
 316                                 if (theApp->sharedfiles) {
 317                                         // Removing the old kad keywords created with the old filename
 318                                         theApp->sharedfiles->RemoveKeywords(existing);
 319                                 }
 320                                 m_knownFileMap[tkey] = Record;
 321                                 return true;
 322                         }
 323                 }
 324         } else {
 325                 AddDebugLogLineN(logGeneral,
 326                         CFormat(wxT("%s is 0-size, not added")) %
 327                         Record->GetFileName());
 328
 329                 return false;
 330         }
 331 }
 332
 333 // Make an index by size to speed up FindKnownFile
 334 // Size modulo 2^32 is enough here
 335 void CKnownFileList::PrepareIndex()
 336 {
 337         ReleaseIndex();
 338         m_knownSizeMap = new KnownFileSizeMap;
 339         for (CKnownFileMap::const_iterator it = m_knownFileMap.begin(); it != m_knownFileMap.end(); ++it) {
 340                 m_knownSizeMap->insert(std::pair<uint32, CKnownFile*>((uint32) it->second->GetFileSize(), it->second));
 341         }
 342         m_duplicateSizeMap = new KnownFileSizeMap;
 343         for (KnownFileList::const_iterator it = m_duplicateFileList.begin(); it != m_duplicateFileList.end(); ++it) {
 344                 m_duplicateSizeMap->insert(std::pair<uint32, CKnownFile*>((uint32) (*it)->GetFileSize(), *it));
 345         }
 346 }
 347
 348
 349 void CKnownFileList::ReleaseIndex()
 350 {
 351         delete m_knownSizeMap;
 352         delete m_duplicateSizeMap;
 353         m_knownSizeMap = NULL;
 354         m_duplicateSizeMap = NULL;
 355 }
 356
 357 // File_checked_for_headers