net/disk_cache/simple/simple_index_file.cc

   1 // Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include "net/disk_cache/simple/simple_index_file.h"
   6
   7 #include <vector>
   8
   9 #include "base/file_util.h"
  10 #include "base/files/file_enumerator.h"
  11 #include "base/hash.h"
  12 #include "base/logging.h"
  13 #include "base/metrics/histogram.h"
  14 #include "base/pickle.h"
  15 #include "base/single_thread_task_runner.h"
  16 #include "base/threading/thread_restrictions.h"
  17 #include "net/disk_cache/simple/simple_entry_format.h"
  18 #include "net/disk_cache/simple/simple_index.h"
  19 #include "net/disk_cache/simple/simple_synchronous_entry.h"
  20 #include "net/disk_cache/simple/simple_util.h"
  21 #include "third_party/zlib/zlib.h"
  22
  23
  24 namespace {
  25
  26 const uint64 kMaxEntiresInIndex = 100000000;
  27
  28 uint32 CalculatePickleCRC(const Pickle& pickle) {
  29   return crc32(crc32(0, Z_NULL, 0),
  30                reinterpret_cast<const Bytef*>(pickle.payload()),
  31                pickle.payload_size());
  32 }
  33
  34 void DoomEntrySetReply(scoped_ptr<int> result,
  35                        const base::Callback<void(int)>& reply_callback) {
  36   reply_callback.Run(*result.get());
  37 }
  38
  39 void WriteToDiskInternal(const base::FilePath& index_filename,
  40                          scoped_ptr<Pickle> pickle,
  41                          const base::TimeTicks& start_time,
  42                          bool app_on_background) {
  43   const base::FilePath temp_filename =
  44       index_filename.DirName().AppendASCII("index_temp");
  45   int bytes_written = file_util::WriteFile(
  46       temp_filename,
  47       reinterpret_cast<const char*>(pickle->data()),
  48       pickle->size());
  49   DCHECK_EQ(bytes_written, implicit_cast<int>(pickle->size()));
  50   if (bytes_written != static_cast<int>(pickle->size())) {
  51     // TODO(felipeg): Add better error handling.
  52     LOG(ERROR) << "Could not write Simple Cache index to temporary file: "
  53                << temp_filename.value();
  54     base::Delete(temp_filename, /* recursive = */ false);
  55   } else {
  56     // Swap temp and index_file.
  57     bool result = base::ReplaceFile(temp_filename, index_filename, NULL);
  58     DCHECK(result);
  59   }
  60   if (app_on_background) {
  61     UMA_HISTOGRAM_TIMES("SimpleCache.IndexWriteToDiskTime.Background",
  62                         (base::TimeTicks::Now() - start_time));
  63   } else {
  64     UMA_HISTOGRAM_TIMES("SimpleCache.IndexWriteToDiskTime.Foreground",
  65                         (base::TimeTicks::Now() - start_time));
  66   }
  67 }
  68
  69 }  // namespace
  70
  71 namespace disk_cache {
  72
  73 // static
  74 const char SimpleIndexFile::kIndexFileName[] = "the-real-index";
  75
  76 SimpleIndexFile::IndexMetadata::IndexMetadata() :
  77     magic_number_(kSimpleIndexMagicNumber),
  78     version_(kSimpleVersion),
  79     number_of_entries_(0),
  80     cache_size_(0) {}
  81
  82 SimpleIndexFile::IndexMetadata::IndexMetadata(
  83     uint64 number_of_entries, uint64 cache_size) :
  84     magic_number_(kSimpleIndexMagicNumber),
  85     version_(kSimpleVersion),
  86     number_of_entries_(number_of_entries),
  87     cache_size_(cache_size) {}
  88
  89 void SimpleIndexFile::IndexMetadata::Serialize(Pickle* pickle) const {
  90   DCHECK(pickle);
  91   pickle->WriteUInt64(magic_number_);
  92   pickle->WriteUInt32(version_);
  93   pickle->WriteUInt64(number_of_entries_);
  94   pickle->WriteUInt64(cache_size_);
  95 }
  96
  97 bool SimpleIndexFile::IndexMetadata::Deserialize(PickleIterator* it) {
  98   DCHECK(it);
  99   return it->ReadUInt64(&magic_number_) &&
 100       it->ReadUInt32(&version_) &&
 101       it->ReadUInt64(&number_of_entries_)&&
 102       it->ReadUInt64(&cache_size_);
 103 }
 104
 105 bool SimpleIndexFile::IndexMetadata::CheckIndexMetadata() {
 106   return number_of_entries_ <= kMaxEntiresInIndex &&
 107       magic_number_ == disk_cache::kSimpleIndexMagicNumber &&
 108       version_ == disk_cache::kSimpleVersion;
 109 }
 110
 111 SimpleIndexFile::SimpleIndexFile(
 112     base::SingleThreadTaskRunner* cache_thread,
 113     base::TaskRunner* worker_pool,
 114     const base::FilePath& index_file_directory)
 115     : cache_thread_(cache_thread),
 116       worker_pool_(worker_pool),
 117       index_file_path_(index_file_directory.AppendASCII(kIndexFileName)) {
 118 }
 119
 120 SimpleIndexFile::~SimpleIndexFile() {}
 121
 122 void SimpleIndexFile::LoadIndexEntries(
 123     scoped_refptr<base::SingleThreadTaskRunner> response_thread,
 124     const IndexCompletionCallback& completion_callback) {
 125   worker_pool_->PostTask(
 126       FROM_HERE,
 127       base::Bind(&SimpleIndexFile::SyncLoadIndexEntries,
 128                  index_file_path_, response_thread, completion_callback));
 129 }
 130
 131 void SimpleIndexFile::WriteToDisk(const SimpleIndex::EntrySet& entry_set,
 132                                   uint64 cache_size,
 133                                   const base::TimeTicks& start,
 134                                   bool app_on_background) {
 135   IndexMetadata index_metadata(entry_set.size(), cache_size);
 136   scoped_ptr<Pickle> pickle = Serialize(index_metadata, entry_set);
 137   cache_thread_->PostTask(FROM_HERE, base::Bind(
 138       &WriteToDiskInternal,
 139       index_file_path_,
 140       base::Passed(&pickle),
 141       base::TimeTicks::Now(),
 142       app_on_background));
 143 }
 144
 145 void SimpleIndexFile::DoomEntrySet(
 146     scoped_ptr<std::vector<uint64> > entry_hashes,
 147     const base::Callback<void(int)>& reply_callback) {
 148   scoped_ptr<int> result(new int());
 149   int* result_p(result.get());
 150
 151   worker_pool_->PostTaskAndReply(
 152       FROM_HERE,
 153       base::Bind(&SimpleSynchronousEntry::DoomEntrySet,
 154                  base::Passed(entry_hashes.Pass()), index_file_path_.DirName(),
 155                  result_p),
 156       base::Bind(&DoomEntrySetReply, base::Passed(result.Pass()),
 157                  reply_callback));
 158 }
 159
 160 // static
 161 void SimpleIndexFile::SyncLoadIndexEntries(
 162     const base::FilePath& index_file_path,
 163     scoped_refptr<base::SingleThreadTaskRunner> response_thread,
 164     const IndexCompletionCallback& completion_callback) {
 165   // TODO(felipeg): probably could load a stale index and use it for something.
 166   scoped_ptr<SimpleIndex::EntrySet> index_file_entries;
 167
 168   const bool index_file_exists = file_util::PathExists(index_file_path);
 169
 170   // Only load if the index is not stale.
 171   const bool index_stale = IsIndexFileStale(index_file_path);
 172   if (!index_stale) {
 173     const base::TimeTicks start = base::TimeTicks::Now();
 174     index_file_entries = SyncLoadFromDisk(index_file_path);
 175     UMA_HISTOGRAM_TIMES("SimpleCache.IndexLoadTime",
 176                         base::TimeTicks::Now() - start);
 177     UMA_HISTOGRAM_COUNTS("SimpleCache.IndexEntriesLoaded",
 178                          index_file_entries ? index_file_entries->size() : 0);
 179   }
 180
 181   UMA_HISTOGRAM_BOOLEAN("SimpleCache.IndexStale", index_stale);
 182
 183   bool force_index_flush = false;
 184   if (!index_file_entries) {
 185     const base::TimeTicks start = base::TimeTicks::Now();
 186     index_file_entries = SyncRestoreFromDisk(index_file_path);
 187     UMA_HISTOGRAM_MEDIUM_TIMES("SimpleCache.IndexRestoreTime",
 188                         base::TimeTicks::Now() - start);
 189     UMA_HISTOGRAM_COUNTS("SimpleCache.IndexEntriesRestored",
 190                          index_file_entries->size());
 191
 192     // When we restore from disk we write the merged index file to disk right
 193     // away, this might save us from having to restore again next time.
 194     force_index_flush = true;
 195   }
 196   UMA_HISTOGRAM_BOOLEAN("SimpleCache.IndexCorrupt",
 197                         (!index_stale && force_index_flush));
 198
 199   // Used in histograms. Please only add new values at the end.
 200   enum {
 201     INITIALIZE_METHOD_RECOVERED = 0,
 202     INITIALIZE_METHOD_LOADED = 1,
 203     INITIALIZE_METHOD_NEWCACHE = 2,
 204     INITIALIZE_METHOD_MAX = 3,
 205   };
 206   int initialize_method;
 207   if (index_file_exists) {
 208     if (force_index_flush)
 209       initialize_method = INITIALIZE_METHOD_RECOVERED;
 210     else
 211       initialize_method = INITIALIZE_METHOD_LOADED;
 212   } else {
 213     UMA_HISTOGRAM_COUNTS("SimpleCache.IndexCreatedEntryCount",
 214                          index_file_entries->size());
 215     initialize_method = INITIALIZE_METHOD_NEWCACHE;
 216   }
 217
 218   UMA_HISTOGRAM_ENUMERATION("SimpleCache.IndexInitializeMethod",
 219                             initialize_method, INITIALIZE_METHOD_MAX);
 220   response_thread->PostTask(FROM_HERE,
 221                             base::Bind(completion_callback,
 222                                        base::Passed(&index_file_entries),
 223                                        force_index_flush));
 224 }
 225
 226 // static
 227 scoped_ptr<SimpleIndex::EntrySet> SimpleIndexFile::SyncLoadFromDisk(
 228     const base::FilePath& index_filename) {
 229   std::string contents;
 230   if (!file_util::ReadFileToString(index_filename, &contents)) {
 231     LOG(WARNING) << "Could not read Simple Index file.";
 232     base::Delete(index_filename, false);
 233     return scoped_ptr<SimpleIndex::EntrySet>();
 234   }
 235
 236   scoped_ptr<SimpleIndex::EntrySet> entries =
 237       SimpleIndexFile::Deserialize(contents.data(), contents.size());
 238   if (!entries) {
 239     base::Delete(index_filename, false);
 240     return scoped_ptr<SimpleIndex::EntrySet>();
 241   }
 242
 243   return entries.Pass();
 244 }
 245
 246 // static
 247 scoped_ptr<Pickle> SimpleIndexFile::Serialize(
 248     const SimpleIndexFile::IndexMetadata& index_metadata,
 249     const SimpleIndex::EntrySet& entries) {
 250   scoped_ptr<Pickle> pickle(new Pickle(sizeof(SimpleIndexFile::PickleHeader)));
 251
 252   index_metadata.Serialize(pickle.get());
 253   for (SimpleIndex::EntrySet::const_iterator it = entries.begin();
 254        it != entries.end(); ++it) {
 255     pickle->WriteUInt64(it->first);
 256     it->second.Serialize(pickle.get());
 257   }
 258   SimpleIndexFile::PickleHeader* header_p =
 259       pickle->headerT<SimpleIndexFile::PickleHeader>();
 260   header_p->crc = CalculatePickleCRC(*pickle);
 261   return pickle.Pass();
 262 }
 263
 264 // static
 265 scoped_ptr<SimpleIndex::EntrySet> SimpleIndexFile::Deserialize(const char* data,
 266                                                                int data_len) {
 267   DCHECK(data);
 268   Pickle pickle(data, data_len);
 269   if (!pickle.data()) {
 270     LOG(WARNING) << "Corrupt Simple Index File.";
 271     return scoped_ptr<SimpleIndex::EntrySet>();
 272   }
 273
 274   PickleIterator pickle_it(pickle);
 275
 276   SimpleIndexFile::PickleHeader* header_p =
 277       pickle.headerT<SimpleIndexFile::PickleHeader>();
 278   const uint32 crc_read = header_p->crc;
 279   const uint32 crc_calculated = CalculatePickleCRC(pickle);
 280
 281   if (crc_read != crc_calculated) {
 282     LOG(WARNING) << "Invalid CRC in Simple Index file.";
 283     return scoped_ptr<SimpleIndex::EntrySet>();
 284   }
 285
 286   SimpleIndexFile::IndexMetadata index_metadata;
 287   if (!index_metadata.Deserialize(&pickle_it)) {
 288     LOG(ERROR) << "Invalid index_metadata on Simple Cache Index.";
 289     return scoped_ptr<SimpleIndex::EntrySet>();
 290   }
 291
 292   if (!index_metadata.CheckIndexMetadata()) {
 293     LOG(ERROR) << "Invalid index_metadata on Simple Cache Index.";
 294     return scoped_ptr<SimpleIndex::EntrySet>();
 295   }
 296
 297   scoped_ptr<SimpleIndex::EntrySet> index_file_entries(
 298       new SimpleIndex::EntrySet());
 299   while (index_file_entries->size() < index_metadata.GetNumberOfEntries()) {
 300     uint64 hash_key;
 301     EntryMetadata entry_metadata;
 302     if (!pickle_it.ReadUInt64(&hash_key) ||
 303         !entry_metadata.Deserialize(&pickle_it)) {
 304       LOG(WARNING) << "Invalid EntryMetadata in Simple Index file.";
 305       return scoped_ptr<SimpleIndex::EntrySet>();
 306     }
 307     SimpleIndex::InsertInEntrySet(
 308         hash_key, entry_metadata, index_file_entries.get());
 309   }
 310
 311   return index_file_entries.Pass();
 312 }
 313
 314 // static
 315 scoped_ptr<SimpleIndex::EntrySet> SimpleIndexFile::SyncRestoreFromDisk(
 316     const base::FilePath& index_file_path) {
 317   LOG(INFO) << "Simple Cache Index is being restored from disk.";
 318
 319   base::Delete(index_file_path, /* recursive = */ false);
 320   scoped_ptr<SimpleIndex::EntrySet> index_file_entries(
 321       new SimpleIndex::EntrySet());
 322
 323   // TODO(felipeg,gavinp): Fix this once we have a one-file per entry format.
 324   COMPILE_ASSERT(kSimpleEntryFileCount == 3,
 325                  file_pattern_must_match_file_count);
 326
 327   const int kFileSuffixLength = sizeof("_0") - 1;
 328   const base::FilePath::StringType file_pattern = FILE_PATH_LITERAL("*_[0-2]");
 329   base::FileEnumerator enumerator(index_file_path.DirName(),
 330                                   false /* recursive */,
 331                                   base::FileEnumerator::FILES,
 332                                   file_pattern);
 333   for (base::FilePath file_path = enumerator.Next(); !file_path.empty();
 334        file_path = enumerator.Next()) {
 335     const base::FilePath::StringType base_name = file_path.BaseName().value();
 336     // Converting to std::string is OK since we never use UTF8 wide chars in our
 337     // file names.
 338     const std::string hash_key_string(base_name.begin(),
 339                                       base_name.end() - kFileSuffixLength);
 340     uint64 hash_key = 0;
 341     if (!simple_util::GetEntryHashKeyFromHexString(
 342             hash_key_string, &hash_key)) {
 343       LOG(WARNING) << "Invalid Entry Hash Key filename while restoring "
 344                    << "Simple Index from disk: " << base_name;
 345       // TODO(felipeg): Should we delete the invalid file here ?
 346       continue;
 347     }
 348
 349     base::FileEnumerator::FileInfo info = enumerator.GetInfo();
 350     base::Time last_used_time;
 351 #if defined(OS_POSIX)
 352     // For POSIX systems, a last access time is available. However, it's not
 353     // guaranteed to be more accurate than mtime. It is no worse though.
 354     last_used_time = base::Time::FromTimeT(info.stat().st_atime);
 355 #endif
 356     if (last_used_time.is_null())
 357       last_used_time = info.GetLastModifiedTime();
 358
 359     int64 file_size = info.GetSize();
 360     SimpleIndex::EntrySet::iterator it = index_file_entries->find(hash_key);
 361     if (it == index_file_entries->end()) {
 362       SimpleIndex::InsertInEntrySet(
 363           hash_key,
 364           EntryMetadata(last_used_time, file_size),
 365           index_file_entries.get());
 366     } else {
 367       // Summing up the total size of the entry through all the *_[0-2] files
 368       it->second.SetEntrySize(it->second.GetEntrySize() + file_size);
 369     }
 370   }
 371   return index_file_entries.Pass();
 372 }
 373
 374 // static
 375 bool SimpleIndexFile::IsIndexFileStale(const base::FilePath& index_filename) {
 376   base::Time index_mtime;
 377   base::Time dir_mtime;
 378   if (!simple_util::GetMTime(index_filename.DirName(), &dir_mtime))
 379     return true;
 380   if (!simple_util::GetMTime(index_filename, &index_mtime))
 381     return true;
 382   // Index file last_modified must be equal to the directory last_modified since
 383   // the last operation we do is ReplaceFile in the
 384   // SimpleIndexFile::WriteToDisk().
 385   // If not true, we need to restore the index.
 386   return index_mtime < dir_mtime;
 387 }
 388
 389 }  // namespace disk_cache