1 /* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2 /* vim:set ts=4 sw=4 sts=4 cin et: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #ifndef _nsDiskCacheMap_h_
8 #define _nsDiskCacheMap_h_
10 #include "mozilla/MemoryReporting.h"
19 #include "nsDiskCache.h"
20 #include "nsDiskCacheBlockFile.h"
23 class nsDiskCacheBinding
;
24 struct nsDiskCacheEntry
;
26 /******************************************************************************
29 * Cache Location Format
31 * 1000 0000 0000 0000 0000 0000 0000 0000 : initialized bit
33 * 0011 0000 0000 0000 0000 0000 0000 0000 : File Selector (0 = separate file)
34 * 0000 0011 0000 0000 0000 0000 0000 0000 : number of extra contiguous blocks 1-4
35 * 0100 1100 0000 0000 0000 0000 0000 0000 : reserved bits
36 * 0000 0000 1111 1111 1111 1111 1111 1111 : block# 0-16777216 (2^24)
38 * 0000 0000 1111 1111 1111 1111 0000 0000 : eFileSizeMask (size of file in k: see note)
39 * 0000 0000 0000 0000 0000 0000 1111 1111 : eFileGenerationMask
42 * 0 = separate file on disk
43 * 1 = 256 byte block file
47 * eFileSizeMask note: Files larger than 65535 KiB have this limit stored in
48 * the location. The file itself must be examined to
49 * determine its actual size if necessary.
51 *****************************************************************************/
54 We have 3 block files with roughly the same max size (32MB)
55 1 - block size 256B, number of blocks 131072
56 2 - block size 1kB, number of blocks 32768
57 3 - block size 4kB, number of blocks 8192
59 #define kNumBlockFiles 3
60 #define SIZE_SHIFT(idx) (2 * ((idx) - 1))
61 #define BLOCK_SIZE_FOR_INDEX(idx) ((idx) ? (256 << SIZE_SHIFT(idx)) : 0)
62 #define BITMAP_SIZE_FOR_INDEX(idx) ((idx) ? (131072 >> SIZE_SHIFT(idx)) : 0)
64 // Min and max values for the number of records in the DiskCachemap
65 #define kMinRecordCount 512
67 #define kSeparateFile 0
68 #define kBuckets (1 << 5) // must be a power of 2!
70 // Maximum size in K which can be stored in the location (see eFileSizeMask).
71 // Both data and metadata can be larger, but only up to kMaxDataSizeK can be
72 // counted into total cache size. I.e. if there are entries where either data or
73 // metadata is larger than kMaxDataSizeK, the total cache size will be
74 // inaccurate (smaller) than the actual cache size. The alternative is to stat
75 // the files to find the real size, which was decided against for performance
76 // reasons. See bug #651100 comment #21.
77 #define kMaxDataSizeK 0xFFFF
79 // preallocate up to 1MB of separate cache file
80 #define kPreallocateLimit 1 * 1024 * 1024
82 // The minimum amount of milliseconds to wait before re-attempting to
83 // revalidate the cache.
84 #define kRevalidateCacheTimeout 3000
85 #define kRevalidateCacheTimeoutTolerance 10
86 #define kRevalidateCacheErrorTimeout 1000
88 class nsDiskCacheRecord
{
92 uint32_t mEvictionRank
;
93 uint32_t mDataLocation
;
94 uint32_t mMetaLocation
;
97 eLocationInitializedMask
= 0x80000000,
99 eLocationSelectorMask
= 0x30000000,
100 eLocationSelectorOffset
= 28,
102 eExtraBlocksMask
= 0x03000000,
103 eExtraBlocksOffset
= 24,
105 eReservedMask
= 0x4C000000,
107 eBlockNumberMask
= 0x00FFFFFF,
109 eFileSizeMask
= 0x00FFFF00,
111 eFileGenerationMask
= 0x000000FF,
112 eFileReservedMask
= 0x4F000000
118 : mHashNumber(0), mEvictionRank(0), mDataLocation(0), mMetaLocation(0)
124 if ((mDataLocation
& eReservedMask
) || (mMetaLocation
& eReservedMask
))
129 // HashNumber accessors
130 uint32_t HashNumber() const { return mHashNumber
; }
131 void SetHashNumber( uint32_t hashNumber
) { mHashNumber
= hashNumber
; }
133 // EvictionRank accessors
134 uint32_t EvictionRank() const { return mEvictionRank
; }
135 void SetEvictionRank( uint32_t rank
) { mEvictionRank
= rank
? rank
: 1; }
137 // DataLocation accessors
138 bool DataLocationInitialized() const { return 0 != (mDataLocation
& eLocationInitializedMask
); }
139 void ClearDataLocation() { mDataLocation
= 0; }
141 uint32_t DataFile() const
143 return (uint32_t)(mDataLocation
& eLocationSelectorMask
) >> eLocationSelectorOffset
;
146 void SetDataBlocks( uint32_t index
, uint32_t startBlock
, uint32_t blockCount
)
152 NS_ASSERTION( index
< (kNumBlockFiles
+ 1), "invalid location index");
153 NS_ASSERTION( index
> 0,"invalid location index");
154 mDataLocation
|= (index
<< eLocationSelectorOffset
) & eLocationSelectorMask
;
157 NS_ASSERTION(startBlock
== (startBlock
& eBlockNumberMask
), "invalid block number");
158 mDataLocation
|= startBlock
& eBlockNumberMask
;
161 NS_ASSERTION( (blockCount
>=1) && (blockCount
<=4),"invalid block count");
163 mDataLocation
|= (blockCount
<< eExtraBlocksOffset
) & eExtraBlocksMask
;
165 mDataLocation
|= eLocationInitializedMask
;
168 uint32_t DataBlockCount() const
170 return (uint32_t)((mDataLocation
& eExtraBlocksMask
) >> eExtraBlocksOffset
) + 1;
173 uint32_t DataStartBlock() const
175 return (mDataLocation
& eBlockNumberMask
);
178 uint32_t DataBlockSize() const
180 return BLOCK_SIZE_FOR_INDEX(DataFile());
183 uint32_t DataFileSize() const { return (mDataLocation
& eFileSizeMask
) >> eFileSizeOffset
; }
184 void SetDataFileSize(uint32_t size
)
186 NS_ASSERTION((mDataLocation
& eFileReservedMask
) == 0, "bad location");
187 mDataLocation
&= ~eFileSizeMask
; // clear eFileSizeMask
188 mDataLocation
|= (size
<< eFileSizeOffset
) & eFileSizeMask
;
191 uint8_t DataFileGeneration() const
193 return (mDataLocation
& eFileGenerationMask
);
196 void SetDataFileGeneration( uint8_t generation
)
198 // clear everything, (separate file index = 0)
200 mDataLocation
|= generation
& eFileGenerationMask
;
201 mDataLocation
|= eLocationInitializedMask
;
204 // MetaLocation accessors
205 bool MetaLocationInitialized() const { return 0 != (mMetaLocation
& eLocationInitializedMask
); }
206 void ClearMetaLocation() { mMetaLocation
= 0; }
207 uint32_t MetaLocation() const { return mMetaLocation
; }
209 uint32_t MetaFile() const
211 return (uint32_t)(mMetaLocation
& eLocationSelectorMask
) >> eLocationSelectorOffset
;
214 void SetMetaBlocks( uint32_t index
, uint32_t startBlock
, uint32_t blockCount
)
220 NS_ASSERTION( index
< (kNumBlockFiles
+ 1), "invalid location index");
221 NS_ASSERTION( index
> 0, "invalid location index");
222 mMetaLocation
|= (index
<< eLocationSelectorOffset
) & eLocationSelectorMask
;
225 NS_ASSERTION(startBlock
== (startBlock
& eBlockNumberMask
), "invalid block number");
226 mMetaLocation
|= startBlock
& eBlockNumberMask
;
229 NS_ASSERTION( (blockCount
>=1) && (blockCount
<=4),"invalid block count");
231 mMetaLocation
|= (blockCount
<< eExtraBlocksOffset
) & eExtraBlocksMask
;
233 mMetaLocation
|= eLocationInitializedMask
;
236 uint32_t MetaBlockCount() const
238 return (uint32_t)((mMetaLocation
& eExtraBlocksMask
) >> eExtraBlocksOffset
) + 1;
241 uint32_t MetaStartBlock() const
243 return (mMetaLocation
& eBlockNumberMask
);
246 uint32_t MetaBlockSize() const
248 return BLOCK_SIZE_FOR_INDEX(MetaFile());
251 uint32_t MetaFileSize() const { return (mMetaLocation
& eFileSizeMask
) >> eFileSizeOffset
; }
252 void SetMetaFileSize(uint32_t size
)
254 mMetaLocation
&= ~eFileSizeMask
; // clear eFileSizeMask
255 mMetaLocation
|= (size
<< eFileSizeOffset
) & eFileSizeMask
;
258 uint8_t MetaFileGeneration() const
260 return (mMetaLocation
& eFileGenerationMask
);
263 void SetMetaFileGeneration( uint8_t generation
)
265 // clear everything, (separate file index = 0)
267 mMetaLocation
|= generation
& eFileGenerationMask
;
268 mMetaLocation
|= eLocationInitializedMask
;
271 uint8_t Generation() const
273 if ((mDataLocation
& eLocationInitializedMask
) &&
275 return DataFileGeneration();
277 if ((mMetaLocation
& eLocationInitializedMask
) &&
279 return MetaFileGeneration();
281 return 0; // no generation
284 #if defined(IS_LITTLE_ENDIAN)
287 mHashNumber
= htonl(mHashNumber
);
288 mEvictionRank
= htonl(mEvictionRank
);
289 mDataLocation
= htonl(mDataLocation
);
290 mMetaLocation
= htonl(mMetaLocation
);
294 #if defined(IS_LITTLE_ENDIAN)
297 mHashNumber
= ntohl(mHashNumber
);
298 mEvictionRank
= ntohl(mEvictionRank
);
299 mDataLocation
= ntohl(mDataLocation
);
300 mMetaLocation
= ntohl(mMetaLocation
);
307 /******************************************************************************
308 * nsDiskCacheRecordVisitor
309 *****************************************************************************/
311 enum { kDeleteRecordAndContinue
= -1,
312 kStopVisitingRecords
= 0,
316 class nsDiskCacheRecordVisitor
{
319 virtual int32_t VisitRecord( nsDiskCacheRecord
* mapRecord
) = 0;
323 /******************************************************************************
325 *****************************************************************************/
327 struct nsDiskCacheHeader
{
328 uint32_t mVersion
; // cache version.
329 uint32_t mDataSize
; // size of cache in units of 1024bytes.
330 int32_t mEntryCount
; // number of entries stored in cache.
331 uint32_t mIsDirty
; // dirty flag.
332 int32_t mRecordCount
; // Number of records
333 uint32_t mEvictionRank
[kBuckets
]; // Highest EvictionRank of the bucket
334 uint32_t mBucketUsage
[kBuckets
]; // Number of used entries in the bucket
337 : mVersion(nsDiskCache::kCurrentVersion
)
346 #if defined(IS_LITTLE_ENDIAN)
347 mVersion
= htonl(mVersion
);
348 mDataSize
= htonl(mDataSize
);
349 mEntryCount
= htonl(mEntryCount
);
350 mIsDirty
= htonl(mIsDirty
);
351 mRecordCount
= htonl(mRecordCount
);
353 for (uint32_t i
= 0; i
< kBuckets
; i
++) {
354 mEvictionRank
[i
] = htonl(mEvictionRank
[i
]);
355 mBucketUsage
[i
] = htonl(mBucketUsage
[i
]);
362 #if defined(IS_LITTLE_ENDIAN)
363 mVersion
= ntohl(mVersion
);
364 mDataSize
= ntohl(mDataSize
);
365 mEntryCount
= ntohl(mEntryCount
);
366 mIsDirty
= ntohl(mIsDirty
);
367 mRecordCount
= ntohl(mRecordCount
);
369 for (uint32_t i
= 0; i
< kBuckets
; i
++) {
370 mEvictionRank
[i
] = ntohl(mEvictionRank
[i
]);
371 mBucketUsage
[i
] = ntohl(mBucketUsage
[i
]);
378 /******************************************************************************
380 *****************************************************************************/
382 class nsDiskCacheMap
{
386 mCacheDirectory(nullptr),
389 mRecordArray(nullptr),
392 mMaxRecordCount(16384), // this default value won't matter
393 mIsDirtyCacheFlushed(false),
394 mLastInvalidateTime(0)
407 * Creates a new cache map file if one doesn't exist.
408 * Returns error if it detects change in format or cache wasn't closed.
410 nsresult
Open( nsIFile
* cacheDirectory
,
411 nsDiskCache::CorruptCacheInfo
* corruptInfo
,
412 bool reportCacheCleanTelemetryData
);
413 nsresult
Close(bool flush
);
416 nsresult
FlushHeader();
417 nsresult
FlushRecords( bool unswap
);
419 void NotifyCapacityChange(uint32_t capacity
);
424 nsresult
AddRecord( nsDiskCacheRecord
* mapRecord
, nsDiskCacheRecord
* oldRecord
);
425 nsresult
UpdateRecord( nsDiskCacheRecord
* mapRecord
);
426 nsresult
FindRecord( uint32_t hashNumber
, nsDiskCacheRecord
* mapRecord
);
427 nsresult
DeleteRecord( nsDiskCacheRecord
* mapRecord
);
428 nsresult
VisitRecords( nsDiskCacheRecordVisitor
* visitor
);
429 nsresult
EvictRecords( nsDiskCacheRecordVisitor
* visitor
);
432 * Disk Entry operations
434 nsresult
DeleteStorage( nsDiskCacheRecord
* record
);
436 nsresult
GetFileForDiskCacheRecord( nsDiskCacheRecord
* record
,
441 nsresult
GetLocalFileForDiskCacheRecord( nsDiskCacheRecord
* record
,
446 // On success, this returns the buffer owned by nsDiskCacheMap,
447 // so it must not be deleted by the caller.
448 nsDiskCacheEntry
* ReadDiskCacheEntry( nsDiskCacheRecord
* record
);
450 nsresult
WriteDiskCacheEntry( nsDiskCacheBinding
* binding
);
452 nsresult
ReadDataCacheBlocks(nsDiskCacheBinding
* binding
, char * buffer
, uint32_t size
);
453 nsresult
WriteDataCacheBlocks(nsDiskCacheBinding
* binding
, char * buffer
, uint32_t size
);
454 nsresult
DeleteStorage( nsDiskCacheRecord
* record
, bool metaData
);
457 * Statistical Operations
459 void IncrementTotalSize( uint32_t delta
)
461 mHeader
.mDataSize
+= delta
;
462 mHeader
.mIsDirty
= true;
465 void DecrementTotalSize( uint32_t delta
)
467 NS_ASSERTION(mHeader
.mDataSize
>= delta
, "disk cache size negative?");
468 mHeader
.mDataSize
= mHeader
.mDataSize
> delta
? mHeader
.mDataSize
- delta
: 0;
469 mHeader
.mIsDirty
= true;
472 inline void IncrementTotalSize( uint32_t blocks
, uint32_t blockSize
)
474 // Round up to nearest K
475 IncrementTotalSize(((blocks
*blockSize
) + 0x03FF) >> 10);
478 inline void DecrementTotalSize( uint32_t blocks
, uint32_t blockSize
)
480 // Round up to nearest K
481 DecrementTotalSize(((blocks
*blockSize
) + 0x03FF) >> 10);
484 uint32_t TotalSize() { return mHeader
.mDataSize
; }
486 int32_t EntryCount() { return mHeader
.mEntryCount
; }
488 size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf
);
496 nsresult
OpenBlockFiles(nsDiskCache::CorruptCacheInfo
* corruptInfo
);
497 nsresult
CloseBlockFiles(bool flush
);
498 bool CacheFilesExist();
500 nsresult
CreateCacheSubDirectories();
502 uint32_t CalculateFileIndex(uint32_t size
);
504 nsresult
GetBlockFileForIndex( uint32_t index
, nsIFile
** result
);
505 uint32_t GetBlockSizeForIndex( uint32_t index
) const {
506 return BLOCK_SIZE_FOR_INDEX(index
);
508 uint32_t GetBitMapSizeForIndex( uint32_t index
) const {
509 return BITMAP_SIZE_FOR_INDEX(index
);
512 // returns the bucket number
513 uint32_t GetBucketIndex( uint32_t hashNumber
) const {
514 return (hashNumber
& (kBuckets
- 1));
517 // Gets the size of the bucket (in number of records)
518 uint32_t GetRecordsPerBucket() const {
519 return mHeader
.mRecordCount
/ kBuckets
;
522 // Gets the first record in the bucket
523 nsDiskCacheRecord
*GetFirstRecordInBucket(uint32_t bucket
) const {
524 return mRecordArray
+ bucket
* GetRecordsPerBucket();
527 uint32_t GetBucketRank(uint32_t bucketIndex
, uint32_t targetRank
);
529 int32_t VisitEachRecord(uint32_t bucketIndex
,
530 nsDiskCacheRecordVisitor
* visitor
,
531 uint32_t evictionRank
);
533 nsresult
GrowRecords();
534 nsresult
ShrinkRecords();
536 nsresult
EnsureBuffer(uint32_t bufSize
);
538 // The returned structure will point to the buffer owned by nsDiskCacheMap,
539 // so it must not be deleted by the caller.
540 nsDiskCacheEntry
* CreateDiskCacheEntry(nsDiskCacheBinding
* binding
,
543 // Initializes the _CACHE_CLEAN_ related functionality
544 nsresult
InitCacheClean(nsIFile
* cacheDirectory
,
545 nsDiskCache::CorruptCacheInfo
* corruptInfo
,
546 bool reportCacheCleanTelemetryData
);
547 // Writes out a value of '0' or '1' in the _CACHE_CLEAN_ file
548 nsresult
WriteCacheClean(bool clean
);
549 // Resets the timout for revalidating the cache
550 nsresult
ResetCacheTimer(int32_t timeout
= kRevalidateCacheTimeout
);
551 // Invalidates the cache, calls WriteCacheClean and ResetCacheTimer
552 nsresult
InvalidateCache();
553 // Determines if the cache is in a safe state
554 bool IsCacheInSafeState();
555 // Revalidates the cache by writting out the header, records, and finally
556 // by calling WriteCacheClean(true).
557 nsresult
RevalidateCache();
558 // Timer which revalidates the cache
559 static void RevalidateTimerCallback(nsITimer
*aTimer
, void *arg
);
565 nsCOMPtr
<nsITimer
> mCleanCacheTimer
;
566 nsCOMPtr
<nsIFile
> mCacheDirectory
;
568 PRFileDesc
* mCleanFD
;
569 nsDiskCacheRecord
* mRecordArray
;
570 nsDiskCacheBlockFile mBlockFile
[kNumBlockFiles
];
571 uint32_t mBufferSize
;
573 nsDiskCacheHeader mHeader
;
574 int32_t mMaxRecordCount
;
575 bool mIsDirtyCacheFlushed
;
576 PRIntervalTime mLastInvalidateTime
;
579 #endif // _nsDiskCacheMap_h_