2 // This file is part of the aMule Project.
4 // Copyright (c) 2006-2011 Mikkel Schubert ( xaignar@amule.org / http:://www.amule.org )
5 // Copyright (c) 2003-2011 aMule Team ( admin@amule.org / http://www.amule.org )
6 // Copyright (c) 2002-2011 Merkur ( devs@emule-project.net / http://www.emule-project.net )
8 // Any parts of this program derived from the xMule, lMule or eMule project,
9 // or contributed by third-party developers are copyrighted by their
10 // respective authors.
12 // This program is free software; you can redistribute it and/or modify
13 // it under the terms of the GNU General Public License as published by
14 // the Free Software Foundation; either version 2 of the License, or
15 // (at your option) any later version.
17 // This program is distributed in the hope that it will be useful,
18 // but WITHOUT ANY WARRANTY; without even the implied warranty of
19 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 // GNU General Public License for more details.
22 // You should have received a copy of the GNU General Public License
23 // along with this program; if not, write to the Free Software
24 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
28 #include <wx/app.h> // Needed for wxTheApp
30 #include "ThreadTasks.h" // Interface declarations
31 #include "PartFile.h" // Needed for CPartFile
32 #include "Logger.h" // Needed for Add(Debug)LogLine{C,N}
33 #include <common/Format.h> // Needed for CFormat
34 #include "amule.h" // Needed for theApp
35 #include "KnownFileList.h" // Needed for theApp->knownfiles
36 #include "Preferences.h" // Needed for thePrefs
37 #include "ScopedPtr.h" // Needed for CScopedPtr and CScopedArray
38 #include "PlatformSpecific.h" // Needed for CanFSHandleSpecialChars
41 //! This hash represents the value for an empty MD4 hashing
42 const uint8_t g_emptyMD4Hash
[16] = {
43 0x31, 0xD6, 0xCF, 0xE0, 0xD1, 0x6A, 0xE9, 0x31,
44 0xB7, 0x3C, 0x59, 0xD7, 0xE0, 0xC0, 0x89, 0xC0 };
47 ////////////////////////////////////////////////////////////
50 CHashingTask::CHashingTask(const CPath
& path
, const CPath
& filename
, const CPartFile
* part
)
51 // GetPrintable is used to improve the readability of the log.
52 : CThreadTask(wxT("Hashing"), path
.JoinPaths(filename
).GetPrintable(), (part
? ETP_High
: ETP_Normal
)),
55 m_toHash((EHashes
)(EH_MD4
| EH_AICH
)),
58 // We can only create the AICH hashset if the file is a knownfile or
59 // if the partfile is complete, since the MD4 hashset is checked first,
60 // so that the AICH hashset only gets assigned if the MD4 hashset
61 // matches what we expected. Due to the rareity of post-completion
62 // corruptions, this gives us a nice speedup in most cases.
63 if (part
&& !part
->GetGapList().empty()) {
69 CHashingTask::CHashingTask(const CKnownFile
* toAICHHash
)
70 // GetPrintable is used to improve the readability of the log.
71 : CThreadTask(wxT("AICH Hashing"), toAICHHash
->GetFilePath().JoinPaths(toAICHHash
->GetFileName()).GetPrintable(), ETP_Low
),
72 m_path(toAICHHash
->GetFilePath()),
73 m_filename(toAICHHash
->GetFileName()),
80 void CHashingTask::Entry()
84 CPath fullPath
= m_path
.JoinPaths(m_filename
);
85 if (!file
.Open(fullPath
, CFile::read
)) {
86 AddDebugLogLineC(logHasher
,
87 CFormat(wxT("Warning, failed to open file, skipping: %s")) % fullPath
);
91 uint64 fileLength
= 0;
93 fileLength
= file
.GetLength();
94 } catch (const CIOFailureException
&) {
95 AddDebugLogLineC(logHasher
,
96 CFormat(wxT("Warning, failed to retrieve file-length, skipping: %s")) % fullPath
);
100 if (fileLength
> MAX_FILE_SIZE
) {
101 AddDebugLogLineC(logHasher
,
102 CFormat(wxT("Warning, file is larger than supported size, skipping: %s")) % fullPath
);
104 } else if (fileLength
== 0) {
106 // It makes no sense to try to hash empty partfiles ...
109 // Zero-size partfiles should be hashed, but not zero-sized shared-files.
110 AddDebugLogLineC(logHasher
,
111 CFormat(wxT("Warning, 0-size file, skipping: %s")) % fullPath
);
117 // For thread-safety, results are passed via a temporary file object.
118 CScopedPtr
<CKnownFile
> knownfile
;
119 knownfile
->m_filePath
= m_path
;
120 knownfile
->SetFileName(m_filename
);
121 knownfile
->SetFileSize(fileLength
);
122 knownfile
->m_lastDateChanged
= CPath::GetModificationTime(fullPath
);
123 knownfile
->m_AvailPartFrequency
.insert(
124 knownfile
->m_AvailPartFrequency
.begin(),
125 knownfile
->GetPartCount(), 0);
127 if ((m_toHash
& EH_MD4
) && (m_toHash
& EH_AICH
)) {
128 knownfile
->GetAICHHashset()->FreeHashSet();
129 AddDebugLogLineN( logHasher
, CFormat(
130 wxT("Starting to create MD4 and AICH hash for file: %s")) %
132 } else if ((m_toHash
& EH_MD4
)) {
133 AddDebugLogLineN( logHasher
, CFormat(
134 wxT("Starting to create MD4 hash for file: %s")) % m_filename
);
135 } else if ((m_toHash
& EH_AICH
)) {
136 knownfile
->GetAICHHashset()->FreeHashSet();
137 AddDebugLogLineN( logHasher
, CFormat(
138 wxT("Starting to create AICH hash for file: %s")) % m_filename
);
140 wxCHECK_RET(0, (CFormat(wxT("No hashes requested for file, skipping: %s"))
141 % m_filename
).GetString());
144 // This loops creates the part-hashes, loop-de-loop.
146 for (uint16 part
= 0; part
< knownfile
->GetPartCount() && !TestDestroy(); part
++) {
147 SetHashingProgress(part
+ 1);
148 if (CreateNextPartHash(file
, part
, knownfile
.get(), m_toHash
) == false) {
149 AddDebugLogLineC(logHasher
,
150 CFormat(wxT("Error while hashing file, skipping: %s"))
153 SetHashingProgress(0);
157 } catch (const CSafeIOException
& e
) {
158 AddDebugLogLineC(logHasher
, wxT("IO exception while hashing file: ") + e
.what());
159 SetHashingProgress(0);
162 SetHashingProgress(0);
164 if ((m_toHash
& EH_MD4
) && !TestDestroy()) {
165 // If the file is < PARTSIZE, then the filehash is that one hash,
166 // otherwise, the filehash is the hash of the parthashes
167 if ( knownfile
->m_hashlist
.size() == 1 ) {
168 knownfile
->m_abyFileHash
= knownfile
->m_hashlist
[0];
169 knownfile
->m_hashlist
.clear();
170 } else if ( knownfile
->m_hashlist
.size() ) {
172 knownfile
->CreateHashFromHashlist(knownfile
->m_hashlist
, &hash
);
173 knownfile
->m_abyFileHash
= hash
;
175 // This should not happen!
180 // Did we create a AICH hashset?
181 if ((m_toHash
& EH_AICH
) && !TestDestroy()) {
182 CAICHHashSet
* AICHHashSet
= knownfile
->GetAICHHashset();
184 AICHHashSet
->ReCalculateHash(false);
185 if (AICHHashSet
->VerifyHashTree(true) ) {
186 AICHHashSet
->SetStatus(AICH_HASHSETCOMPLETE
);
187 if (!AICHHashSet
->SaveHashSet()) {
188 AddDebugLogLineC( logHasher
,
189 CFormat(wxT("Warning, failed to save AICH hashset for file: %s"))
192 // delete hashset now to free memory
193 AICHHashSet
->FreeHashSet();
197 if ((m_toHash
== EH_AICH
) && !TestDestroy()) {
198 CHashingEvent
evt(MULE_EVT_AICH_HASHING
, knownfile
.release(), m_owner
);
200 wxPostEvent(wxTheApp
, evt
);
201 } else if (!TestDestroy()) {
202 CHashingEvent
evt(MULE_EVT_HASHING
, knownfile
.release(), m_owner
);
204 wxPostEvent(wxTheApp
, evt
);
209 void CHashingTask::SetHashingProgress(uint16 part
)
212 m_owner
->SetHashingProgress(part
);
217 bool CHashingTask::CreateNextPartHash(CFileAutoClose
& file
, uint16 part
, CKnownFile
* owner
, EHashes toHash
)
219 wxCHECK_MSG(!file
.Eof(), false, wxT("Unexpected EOF in CreateNextPartHash"));
221 const uint64 offset
= part
* PARTSIZE
;
222 // We'll read at most PARTSIZE bytes per cycle
223 const uint64 partLength
= owner
->GetPartSize(part
);
226 CMD4Hash
* md4Hash
= ((toHash
& EH_MD4
) ? &hash
: NULL
);
227 CAICHHashTree
* aichHash
= NULL
;
229 // Setup for AICH hashing
230 if (toHash
& EH_AICH
) {
231 aichHash
= owner
->GetAICHHashset()->m_pHashTree
.FindHash(offset
, partLength
);
234 owner
->CreateHashFromFile(file
, offset
, partLength
, md4Hash
, aichHash
);
236 if (toHash
& EH_MD4
) {
237 // Store the md4 hash
238 owner
->m_hashlist
.push_back(hash
);
240 // This is because of the ed2k implementation for parts. A 2 * PARTSIZE
241 // file i.e. will have 3 parts (see CKnownFile::SetFileSize for comments).
242 // So we have to create the hash for the 0-size data, which will be the default
243 // md4 hash for null data: 31D6CFE0D16AE931B73C59D7E0C089C0
244 if ((partLength
== PARTSIZE
) && file
.Eof()) {
245 owner
->m_hashlist
.push_back(CMD4Hash(g_emptyMD4Hash
));
253 void CHashingTask::OnLastTask()
255 if (GetType() == wxT("Hashing")) {
256 // To prevent rehashing in case of crashes, we
257 // explicity save the list of hashed files here.
258 theApp
->knownfiles
->Save();
260 // Make sure the AICH-hashes are up to date.
261 CThreadScheduler::AddTask(new CAICHSyncTask());
266 ////////////////////////////////////////////////////////////
269 CAICHSyncTask::CAICHSyncTask()
270 : CThreadTask(wxT("AICH Syncronizing"), wxEmptyString
, ETP_Low
)
275 void CAICHSyncTask::Entry()
277 ConvertToKnown2ToKnown264();
279 AddDebugLogLineN( logAICHThread
, wxT("Syncronization thread started.") );
281 // We collect all masterhashs which we find in the known2.met and store them in a list
282 std::list
<CAICHHash
> hashlist
;
283 const CPath fullpath
= CPath(thePrefs::GetConfigDir() + KNOWN2_MET_FILENAME
);
286 if (!fullpath
.FileExists()) {
287 // File does not exist. Try to create it to see if it can be created at all (and don't start hashing otherwise).
288 if (!file
.Open(fullpath
, CFile::write
)) {
289 AddDebugLogLineC( logAICHThread
, wxT("Error, failed to open 'known2_64.met' file!") );
293 file
.WriteUInt8(KNOWN2_MET_VERSION
);
294 } catch (const CIOFailureException
& e
) {
295 AddDebugLogLineC(logAICHThread
, wxT("IO failure while creating hashlist (Aborting): ") + e
.what());
299 if (!file
.Open(fullpath
, CFile::read
)) {
300 AddDebugLogLineC( logAICHThread
, wxT("Error, failed to open 'known2_64.met' file!") );
304 uint32 nLastVerifiedPos
= 0;
306 if (file
.ReadUInt8() != KNOWN2_MET_VERSION
) {
307 throw CEOFException(wxT("Invalid met-file header found, removing file."));
310 uint64 nExistingSize
= file
.GetLength();
311 while (file
.GetPosition() < nExistingSize
) {
312 // Read the next hash
313 hashlist
.push_back(CAICHHash(&file
));
315 uint32 nHashCount
= file
.ReadUInt32();
316 if (file
.GetPosition() + nHashCount
* CAICHHash::GetHashSize() > nExistingSize
){
317 throw CEOFException(wxT("Hashlist ends past end of file."));
320 // skip the rest of this hashset
321 nLastVerifiedPos
= file
.Seek(nHashCount
* HASHSIZE
, wxFromCurrent
);
323 } catch (const CEOFException
&) {
324 AddDebugLogLineC(logAICHThread
, wxT("Hashlist corrupted, truncating file."));
326 file
.Reopen(CFile::read_write
);
327 file
.SetLength(nLastVerifiedPos
);
328 } catch (const CIOFailureException
& e
) {
329 AddDebugLogLineC(logAICHThread
, wxT("IO failure while reading hashlist (Aborting): ") + e
.what());
334 AddDebugLogLineN( logAICHThread
, wxT("Masterhashes of known files have been loaded.") );
337 // Now we check that all files which are in the sharedfilelist have a
338 // corresponding hash in our list. Those how don't are queued for hashing.
339 theApp
->sharedfiles
->CheckAICHHashes(hashlist
);
343 bool CAICHSyncTask::ConvertToKnown2ToKnown264()
345 // converting known2.met to known2_64.met to support large files
346 // changing hashcount from uint16 to uint32
348 const CPath oldfullpath
= CPath(thePrefs::GetConfigDir() + OLD_KNOWN2_MET_FILENAME
);
349 const CPath newfullpath
= CPath(thePrefs::GetConfigDir() + KNOWN2_MET_FILENAME
);
351 if (newfullpath
.FileExists() || !oldfullpath
.FileExists()) {
352 // In this case, there is nothing that we need to do.
359 if (!oldfile
.Open(oldfullpath
, CFile::read
)) {
360 AddDebugLogLineC(logAICHThread
, wxT("Failed to open 'known2.met' file."));
362 // else -> known2.met also doesn't exists, so nothing to convert
367 if (!newfile
.Open(newfullpath
, CFile::write_excl
)) {
368 AddDebugLogLineC(logAICHThread
, wxT("Failed to create 'known2_64.met' file."));
373 AddLogLineN(CFormat(_("Converting old AICH hashsets in '%s' to 64b in '%s'."))
374 % OLD_KNOWN2_MET_FILENAME
% KNOWN2_MET_FILENAME
);
377 newfile
.WriteUInt8(KNOWN2_MET_VERSION
);
379 while (newfile
.GetPosition() < oldfile
.GetLength()) {
380 CAICHHash
aichHash(&oldfile
);
381 uint32 nHashCount
= oldfile
.ReadUInt16();
383 CScopedArray
<uint8_t> buffer(nHashCount
* CAICHHash::GetHashSize());
385 oldfile
.Read(buffer
.get(), nHashCount
* CAICHHash::GetHashSize());
386 newfile
.Write(aichHash
.GetRawHash(), CAICHHash::GetHashSize());
387 newfile
.WriteUInt32(nHashCount
);
388 newfile
.Write(buffer
.get(), nHashCount
* CAICHHash::GetHashSize());
391 } catch (const CEOFException
& e
) {
392 AddDebugLogLineC(logAICHThread
, wxT("Error reading old 'known2.met' file.") + e
.what());
394 } catch (const CIOFailureException
& e
) {
395 AddDebugLogLineC(logAICHThread
, wxT("IO error while converting 'known2.met' file: ") + e
.what());
399 // FIXME LARGE FILES (uncomment)
400 //DeleteFile(oldfullpath);
407 ////////////////////////////////////////////////////////////
411 CCompletionTask::CCompletionTask(const CPartFile
* file
)
412 // GetPrintable is used to improve the readability of the log.
413 : CThreadTask(wxT("Completing"), file
->GetFullName().GetPrintable(), ETP_High
),
414 m_filename(file
->GetFileName()),
415 m_metPath(file
->GetFullName()),
416 m_category(file
->GetCategory()),
420 wxASSERT(m_filename
.IsOk());
421 wxASSERT(m_metPath
.IsOk());
426 void CCompletionTask::Entry()
432 // Prevent the preference values from changing underneeth us.
433 wxMutexGuiLocker guiLock
;
435 //#warning Thread-safety needed
438 targetPath
= theApp
->glob_prefs
->GetCategory(m_category
)->path
;
439 if (!targetPath
.DirExists()) {
440 targetPath
= thePrefs::GetIncomingDir();
444 CPath dstName
= m_filename
.Cleanup(true, !PlatformSpecific::CanFSHandleSpecialChars(targetPath
));
446 // Avoid empty filenames ...
447 if (!dstName
.IsOk()) {
448 dstName
= CPath(wxT("Unknown"));
451 if (m_filename
!= dstName
) {
452 AddLogLineC(CFormat(_("WARNING: The filename '%s' is invalid and has been renamed to '%s'.")) % m_filename
% dstName
);
455 // Avoid saving to an already existing filename
456 CPath newName
= targetPath
.JoinPaths(dstName
);
457 for (unsigned count
= 0; newName
.FileExists(); ++count
) {
458 wxString postfix
= CFormat(wxT("(%u)")) % count
;
460 newName
= targetPath
.JoinPaths(dstName
.AddPostfix(postfix
));
463 if (newName
!= targetPath
.JoinPaths(dstName
)) {
464 AddLogLineC(CFormat(_("WARNING: The file '%s' already exists, new file renamed to '%s'.")) % dstName
% newName
.GetFullName());
467 // Move will handle dirs on the same partition, otherwise copy is needed.
468 CPath partfilename
= m_metPath
.RemoveExt();
469 if (!CPath::RenameFile(partfilename
, newName
)) {
470 if (!CPath::CloneFile(partfilename
, newName
, true)) {
475 if (!CPath::RemoveFile(partfilename
)) {
476 AddDebugLogLineC(logPartFile
, CFormat(wxT("WARNING: Could not remove original '%s' after creating backup")) % partfilename
);
480 // Removes the various other data-files
481 const wxChar
* otherMetExt
[] = { wxT(""), PARTMET_BAK_EXT
, wxT(".seeds"), NULL
};
482 for (size_t i
= 0; otherMetExt
[i
]; ++i
) {
483 CPath toRemove
= m_metPath
.AppendExt(otherMetExt
[i
]);
485 if (toRemove
.FileExists()) {
486 if (!CPath::RemoveFile(toRemove
)) {
487 AddDebugLogLineC(logPartFile
, CFormat(wxT("WARNING: Failed to delete %s")) % toRemove
);
496 void CCompletionTask::OnExit()
498 // Notify the app that the completion has finished for this file.
499 CCompletionEvent
evt(m_error
, m_owner
, m_newName
);
501 wxPostEvent(wxTheApp
, evt
);
506 ////////////////////////////////////////////////////////////
509 #ifdef HAVE_FALLOCATE
516 # include <linux/falloc.h>
517 #elif defined HAVE_SYS_FALLOCATE
518 # include <sys/syscall.h>
519 # include <sys/types.h>
521 #elif defined HAVE_POSIX_FALLOCATE
522 # define _XOPEN_SOURCE 600
531 CAllocateFileTask::CAllocateFileTask(CPartFile
*file
, bool pause
)
532 // GetPrintable is used to improve the readability of the log.
533 : CThreadTask(wxT("Allocating"), file
->GetFullName().RemoveExt().GetPrintable(), ETP_High
),
534 m_file(file
), m_pause(pause
), m_result(ENOSYS
)
536 wxASSERT(file
!= NULL
);
539 void CAllocateFileTask::Entry()
541 if (m_file
->GetFileSize() == 0) {
546 uint64_t minFree
= thePrefs::IsCheckDiskspaceEnabled() ? thePrefs::GetMinFreeDiskSpace() : 0;
547 int64_t freeSpace
= CPath::GetFreeSpaceAt(thePrefs::GetTempDir());
549 // Don't even try to allocate, if there's no space to complete the operation.
550 if (freeSpace
!= wxInvalidOffset
) {
551 if ((uint64_t)freeSpace
< m_file
->GetFileSize() + minFree
) {
558 file
.Open(m_file
->GetFullName().RemoveExt(), CFile::read_write
);
562 // File is already created as non-sparse, so we only need to set the length.
563 // This will fail to allocate the file e.g. under wine on linux/ext3,
564 // but works with NTFS and FAT32.
565 file
.Seek(m_file
->GetFileSize() - 1, wxFromStart
);
569 } catch (const CSafeIOException
&) {
573 // Use kernel level routines if possible
574 # ifdef HAVE_FALLOCATE
575 m_result
= fallocate(file
.fd(), 0, 0, m_file
->GetFileSize());
576 # elif defined HAVE_SYS_FALLOCATE
577 m_result
= syscall(SYS_fallocate
, file
.fd(), 0, (loff_t
)0, (loff_t
)m_file
->GetFileSize());
578 if (m_result
== -1) {
581 # elif defined HAVE_POSIX_FALLOCATE
582 // otherwise use glibc implementation, if available
583 m_result
= posix_fallocate(file
.fd(), 0, m_file
->GetFileSize());
586 if (m_result
!= 0 && m_result
!= ENOSPC
) {
587 // If everything else fails, use slow-and-dirty method of allocating the file: write the whole file with zeroes.
588 # define BLOCK_SIZE 1048576 /* Write 1 MB blocks */
589 void *zero
= calloc(1, BLOCK_SIZE
);
592 uint64_t size
= m_file
->GetFileSize();
593 for (; size
>= BLOCK_SIZE
; size
-= BLOCK_SIZE
) {
594 file
.Write(zero
, BLOCK_SIZE
);
597 file
.Write(zero
, size
);
601 } catch (const CSafeIOException
&) {
611 if (file
.IsOpened()) {
616 void CAllocateFileTask::OnExit()
618 // Notify the app that the preallocation has finished for this file.
619 CAllocFinishedEvent
evt(m_file
, m_pause
, m_result
);
621 wxPostEvent(wxTheApp
, evt
);
626 ////////////////////////////////////////////////////////////
629 DEFINE_LOCAL_EVENT_TYPE(MULE_EVT_HASHING
)
630 DEFINE_LOCAL_EVENT_TYPE(MULE_EVT_AICH_HASHING
)
632 CHashingEvent::CHashingEvent(wxEventType type
, CKnownFile
* result
, const CKnownFile
* owner
)
640 wxEvent
* CHashingEvent::Clone() const
642 return new CHashingEvent(GetEventType(), m_result
, m_owner
);
646 const CKnownFile
* CHashingEvent::GetOwner() const
652 CKnownFile
* CHashingEvent::GetResult() const
660 ////////////////////////////////////////////////////////////
663 DEFINE_LOCAL_EVENT_TYPE(MULE_EVT_FILE_COMPLETED
)
666 CCompletionEvent::CCompletionEvent(bool errorOccured
, const CPartFile
* owner
, const CPath
& fullPath
)
667 : wxEvent(-1, MULE_EVT_FILE_COMPLETED
),
668 m_fullPath(fullPath
),
670 m_error(errorOccured
)
675 wxEvent
* CCompletionEvent::Clone() const
677 return new CCompletionEvent(m_error
, m_owner
, m_fullPath
);
681 bool CCompletionEvent::ErrorOccured() const
687 const CPartFile
* CCompletionEvent::GetOwner() const
693 const CPath
& CCompletionEvent::GetFullPath() const
699 ////////////////////////////////////////////////////////////
700 // CAllocFinishedEvent
702 DEFINE_LOCAL_EVENT_TYPE(MULE_EVT_ALLOC_FINISHED
)
704 wxEvent
*CAllocFinishedEvent::Clone() const
706 return new CAllocFinishedEvent(m_file
, m_pause
, m_result
);
709 // File_checked_for_headers