1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "chrome/browser/safe_browsing/safe_browsing_database.h"
10 #include "base/bind.h"
11 #include "base/file_util.h"
12 #include "base/message_loop/message_loop.h"
13 #include "base/metrics/histogram.h"
14 #include "base/metrics/stats_counters.h"
15 #include "base/process/process.h"
16 #include "base/process/process_metrics.h"
17 #include "base/sha1.h"
18 #include "base/strings/string_number_conversions.h"
19 #include "base/strings/stringprintf.h"
20 #include "base/time/time.h"
21 #include "chrome/browser/safe_browsing/prefix_set.h"
22 #include "chrome/browser/safe_browsing/safe_browsing_store_file.h"
23 #include "content/public/browser/browser_thread.h"
24 #include "crypto/sha2.h"
25 #include "net/base/net_util.h"
28 #if defined(OS_MACOSX)
29 #include "base/mac/mac_util.h"
32 using content::BrowserThread
;
36 // Filename suffix for the bloom filter.
37 const base::FilePath::CharType kBloomFilterFile
[] =
38 FILE_PATH_LITERAL(" Filter 2");
39 // Filename suffix for the prefix set.
40 const base::FilePath::CharType kPrefixSetFile
[] =
41 FILE_PATH_LITERAL(" Prefix Set");
42 // Filename suffix for download store.
43 const base::FilePath::CharType kDownloadDBFile
[] =
44 FILE_PATH_LITERAL(" Download");
45 // Filename suffix for client-side phishing detection whitelist store.
46 const base::FilePath::CharType kCsdWhitelistDBFile
[] =
47 FILE_PATH_LITERAL(" Csd Whitelist");
48 // Filename suffix for the download whitelist store.
49 const base::FilePath::CharType kDownloadWhitelistDBFile
[] =
50 FILE_PATH_LITERAL(" Download Whitelist");
51 // Filename suffix for the extension blacklist store.
52 const base::FilePath::CharType kExtensionBlacklistDBFile
[] =
53 FILE_PATH_LITERAL(" Extension Blacklist");
54 // Filename suffix for the side-effect free whitelist store.
55 const base::FilePath::CharType kSideEffectFreeWhitelistDBFile
[] =
56 FILE_PATH_LITERAL(" Side-Effect Free Whitelist");
57 // Filename suffix for the csd malware IP blacklist store.
58 const base::FilePath::CharType kIPBlacklistDBFile
[] =
59 FILE_PATH_LITERAL(" IP Blacklist");
61 // Filename suffix for browse store.
62 // TODO(shess): "Safe Browsing Bloom Prefix Set" is full of win.
63 // Unfortunately, to change the name implies lots of transition code
64 // for little benefit. If/when file formats change (say to put all
65 // the data in one file), that would be a convenient point to rectify
67 const base::FilePath::CharType kBrowseDBFile
[] = FILE_PATH_LITERAL(" Bloom");
69 // The maximum staleness for a cached entry.
70 const int kMaxStalenessMinutes
= 45;
72 // Maximum number of entries we allow in any of the whitelists.
73 // If a whitelist on disk contains more entries then all lookups to
74 // the whitelist will be considered a match.
75 const size_t kMaxWhitelistSize
= 5000;
77 // If the hash of this exact expression is on a whitelist then all
78 // lookups to this whitelist will be considered a match.
79 const char kWhitelistKillSwitchUrl
[] =
80 "sb-ssl.google.com/safebrowsing/csd/killswitch"; // Don't change this!
82 // If the hash of this exact expression is on a whitelist then the
83 // malware IP blacklisting feature will be disabled in csd.
85 const char kMalwareIPKillSwitchUrl
[] =
86 "sb-ssl.google.com/safebrowsing/csd/killswitch_malware";
88 const size_t kMaxIpPrefixSize
= 128;
89 const size_t kMinIpPrefixSize
= 1;
91 // To save space, the incoming |chunk_id| and |list_id| are combined
92 // into an |encoded_chunk_id| for storage by shifting the |list_id|
93 // into the low-order bits. These functions decode that information.
94 // TODO(lzheng): It was reasonable when database is saved in sqlite, but
95 // there should be better ways to save chunk_id and list_id after we use
96 // SafeBrowsingStoreFile.
97 int GetListIdBit(const int encoded_chunk_id
) {
98 return encoded_chunk_id
& 1;
100 int DecodeChunkId(int encoded_chunk_id
) {
101 return encoded_chunk_id
>> 1;
103 int EncodeChunkId(const int chunk
, const int list_id
) {
104 DCHECK_NE(list_id
, safe_browsing_util::INVALID
);
105 return chunk
<< 1 | list_id
% 2;
108 // Generate the set of full hashes to check for |url|. If
109 // |include_whitelist_hashes| is true we will generate additional path-prefixes
110 // to match against the csd whitelist. E.g., if the path-prefix /foo is on the
111 // whitelist it should also match /foo/bar which is not the case for all the
112 // other lists. We'll also always add a pattern for the empty path.
113 // TODO(shess): This function is almost the same as
114 // |CompareFullHashes()| in safe_browsing_util.cc, except that code
115 // does an early exit on match. Since match should be the infrequent
116 // case (phishing or malware found), consider combining this function
118 void BrowseFullHashesToCheck(const GURL
& url
,
119 bool include_whitelist_hashes
,
120 std::vector
<SBFullHash
>* full_hashes
) {
121 std::vector
<std::string
> hosts
;
122 if (url
.HostIsIPAddress()) {
123 hosts
.push_back(url
.host());
125 safe_browsing_util::GenerateHostsToCheck(url
, &hosts
);
128 std::vector
<std::string
> paths
;
129 safe_browsing_util::GeneratePathsToCheck(url
, &paths
);
131 for (size_t i
= 0; i
< hosts
.size(); ++i
) {
132 for (size_t j
= 0; j
< paths
.size(); ++j
) {
133 const std::string
& path
= paths
[j
];
134 full_hashes
->push_back(SBFullHashForString(hosts
[i
] + path
));
136 // We may have /foo as path-prefix in the whitelist which should
137 // also match with /foo/bar and /foo?bar. Hence, for every path
138 // that ends in '/' we also add the path without the slash.
139 if (include_whitelist_hashes
&&
141 path
[path
.size() - 1] == '/') {
142 full_hashes
->push_back(
143 SBFullHashForString(hosts
[i
] + path
.substr(0, path
.size() - 1)));
149 // Get the prefixes matching the download |urls|.
150 void GetDownloadUrlPrefixes(const std::vector
<GURL
>& urls
,
151 std::vector
<SBPrefix
>* prefixes
) {
152 std::vector
<SBFullHash
> full_hashes
;
153 for (size_t i
= 0; i
< urls
.size(); ++i
)
154 BrowseFullHashesToCheck(urls
[i
], false, &full_hashes
);
156 for (size_t i
= 0; i
< full_hashes
.size(); ++i
)
157 prefixes
->push_back(full_hashes
[i
].prefix
);
160 // Helper function to compare addprefixes in |store| with |prefixes|.
161 // The |list_bit| indicates which list (url or hash) to compare.
163 // Returns true if there is a match, |*prefix_hits| (if non-NULL) will contain
164 // the actual matching prefixes.
165 bool MatchAddPrefixes(SafeBrowsingStore
* store
,
167 const std::vector
<SBPrefix
>& prefixes
,
168 std::vector
<SBPrefix
>* prefix_hits
) {
169 prefix_hits
->clear();
170 bool found_match
= false;
172 SBAddPrefixes add_prefixes
;
173 store
->GetAddPrefixes(&add_prefixes
);
174 for (SBAddPrefixes::const_iterator iter
= add_prefixes
.begin();
175 iter
!= add_prefixes
.end(); ++iter
) {
176 for (size_t j
= 0; j
< prefixes
.size(); ++j
) {
177 const SBPrefix
& prefix
= prefixes
[j
];
178 if (prefix
== iter
->prefix
&&
179 GetListIdBit(iter
->chunk_id
) == list_bit
) {
180 prefix_hits
->push_back(prefix
);
188 // Find the entries in |full_hashes| with prefix in |prefix_hits|, and
189 // add them to |full_hits| if not expired. "Not expired" is when
190 // either |last_update| was recent enough, or the item has been
191 // received recently enough. Expired items are not deleted because a
192 // future update may make them acceptable again.
194 // For efficiency reasons the code walks |prefix_hits| and
195 // |full_hashes| in parallel, so they must be sorted by prefix.
196 void GetCachedFullHashesForBrowse(
197 const std::vector
<SBPrefix
>& prefix_hits
,
198 const std::vector
<SBFullHashCached
>& full_hashes
,
199 std::vector
<SBFullHashResult
>* full_hits
,
200 base::Time last_update
) {
201 const base::Time expire_time
=
202 base::Time::Now() - base::TimeDelta::FromMinutes(kMaxStalenessMinutes
);
204 std::vector
<SBPrefix
>::const_iterator piter
= prefix_hits
.begin();
205 std::vector
<SBFullHashCached
>::const_iterator hiter
= full_hashes
.begin();
207 while (piter
!= prefix_hits
.end() && hiter
!= full_hashes
.end()) {
208 if (*piter
< hiter
->hash
.prefix
) {
210 } else if (hiter
->hash
.prefix
< *piter
) {
213 if (expire_time
< last_update
||
214 expire_time
.ToTimeT() < hiter
->received
) {
215 SBFullHashResult result
;
216 result
.list_id
= hiter
->list_id
;
217 result
.hash
= hiter
->hash
;
218 full_hits
->push_back(result
);
221 // Only increment |hiter|, |piter| might have multiple hits.
227 // This function generates a chunk range string for |chunks|. It
228 // outputs one chunk range string per list and writes it to the
229 // |list_ranges| vector. We expect |list_ranges| to already be of the
230 // right size. E.g., if |chunks| contains chunks with two different
231 // list ids then |list_ranges| must contain two elements.
232 void GetChunkRanges(const std::vector
<int>& chunks
,
233 std::vector
<std::string
>* list_ranges
) {
234 // Since there are 2 possible list ids, there must be exactly two
235 // list ranges. Even if the chunk data should only contain one
236 // line, this code has to somehow handle corruption.
237 DCHECK_EQ(2U, list_ranges
->size());
239 std::vector
<std::vector
<int> > decoded_chunks(list_ranges
->size());
240 for (std::vector
<int>::const_iterator iter
= chunks
.begin();
241 iter
!= chunks
.end(); ++iter
) {
242 int mod_list_id
= GetListIdBit(*iter
);
243 DCHECK_GE(mod_list_id
, 0);
244 DCHECK_LT(static_cast<size_t>(mod_list_id
), decoded_chunks
.size());
245 decoded_chunks
[mod_list_id
].push_back(DecodeChunkId(*iter
));
247 for (size_t i
= 0; i
< decoded_chunks
.size(); ++i
) {
248 ChunksToRangeString(decoded_chunks
[i
], &((*list_ranges
)[i
]));
252 // Helper function to create chunk range lists for Browse related
254 void UpdateChunkRanges(SafeBrowsingStore
* store
,
255 const std::vector
<std::string
>& listnames
,
256 std::vector
<SBListChunkRanges
>* lists
) {
260 DCHECK_GT(listnames
.size(), 0U);
261 DCHECK_LE(listnames
.size(), 2U);
262 std::vector
<int> add_chunks
;
263 std::vector
<int> sub_chunks
;
264 store
->GetAddChunks(&add_chunks
);
265 store
->GetSubChunks(&sub_chunks
);
267 // Always decode 2 ranges, even if only the first one is expected.
268 // The loop below will only load as many into |lists| as |listnames|
270 std::vector
<std::string
> adds(2);
271 std::vector
<std::string
> subs(2);
272 GetChunkRanges(add_chunks
, &adds
);
273 GetChunkRanges(sub_chunks
, &subs
);
275 for (size_t i
= 0; i
< listnames
.size(); ++i
) {
276 const std::string
& listname
= listnames
[i
];
277 DCHECK_EQ(safe_browsing_util::GetListId(listname
) % 2,
278 static_cast<int>(i
% 2));
279 DCHECK_NE(safe_browsing_util::GetListId(listname
),
280 safe_browsing_util::INVALID
);
281 lists
->push_back(SBListChunkRanges(listname
));
282 lists
->back().adds
.swap(adds
[i
]);
283 lists
->back().subs
.swap(subs
[i
]);
287 void UpdateChunkRangesForLists(SafeBrowsingStore
* store
,
288 const std::string
& listname0
,
289 const std::string
& listname1
,
290 std::vector
<SBListChunkRanges
>* lists
) {
291 std::vector
<std::string
> listnames
;
292 listnames
.push_back(listname0
);
293 listnames
.push_back(listname1
);
294 UpdateChunkRanges(store
, listnames
, lists
);
297 void UpdateChunkRangesForList(SafeBrowsingStore
* store
,
298 const std::string
& listname
,
299 std::vector
<SBListChunkRanges
>* lists
) {
300 UpdateChunkRanges(store
, std::vector
<std::string
>(1, listname
), lists
);
303 // Order |SBFullHashCached| items on the prefix part.
304 bool SBFullHashCachedPrefixLess(const SBFullHashCached
& a
,
305 const SBFullHashCached
& b
) {
306 return a
.hash
.prefix
< b
.hash
.prefix
;
309 // This code always checks for non-zero file size. This helper makes
310 // that less verbose.
311 int64
GetFileSizeOrZero(const base::FilePath
& file_path
) {
313 if (!base::GetFileSize(file_path
, &size_64
))
318 // Used to order whitelist storage in memory.
319 bool SBFullHashLess(const SBFullHash
& a
, const SBFullHash
& b
) {
320 return memcmp(a
.full_hash
, b
.full_hash
, sizeof(a
.full_hash
)) < 0;
325 // The default SafeBrowsingDatabaseFactory.
326 class SafeBrowsingDatabaseFactoryImpl
: public SafeBrowsingDatabaseFactory
{
328 virtual SafeBrowsingDatabase
* CreateSafeBrowsingDatabase(
329 bool enable_download_protection
,
330 bool enable_client_side_whitelist
,
331 bool enable_download_whitelist
,
332 bool enable_extension_blacklist
,
333 bool enable_side_effect_free_whitelist
,
334 bool enable_ip_blacklist
) OVERRIDE
{
335 return new SafeBrowsingDatabaseNew(
336 new SafeBrowsingStoreFile
,
337 enable_download_protection
? new SafeBrowsingStoreFile
: NULL
,
338 enable_client_side_whitelist
? new SafeBrowsingStoreFile
: NULL
,
339 enable_download_whitelist
? new SafeBrowsingStoreFile
: NULL
,
340 enable_extension_blacklist
? new SafeBrowsingStoreFile
: NULL
,
341 enable_side_effect_free_whitelist
? new SafeBrowsingStoreFile
: NULL
,
342 enable_ip_blacklist
? new SafeBrowsingStoreFile
: NULL
);
345 SafeBrowsingDatabaseFactoryImpl() { }
348 DISALLOW_COPY_AND_ASSIGN(SafeBrowsingDatabaseFactoryImpl
);
352 SafeBrowsingDatabaseFactory
* SafeBrowsingDatabase::factory_
= NULL
;
354 // Factory method, non-thread safe. Caller has to make sure this s called
355 // on SafeBrowsing Thread.
356 // TODO(shess): There's no need for a factory any longer. Convert
357 // SafeBrowsingDatabaseNew to SafeBrowsingDatabase, and have Create()
358 // callers just construct things directly.
359 SafeBrowsingDatabase
* SafeBrowsingDatabase::Create(
360 bool enable_download_protection
,
361 bool enable_client_side_whitelist
,
362 bool enable_download_whitelist
,
363 bool enable_extension_blacklist
,
364 bool enable_side_effect_free_whitelist
,
365 bool enable_ip_blacklist
) {
367 factory_
= new SafeBrowsingDatabaseFactoryImpl();
368 return factory_
->CreateSafeBrowsingDatabase(
369 enable_download_protection
,
370 enable_client_side_whitelist
,
371 enable_download_whitelist
,
372 enable_extension_blacklist
,
373 enable_side_effect_free_whitelist
,
374 enable_ip_blacklist
);
377 SafeBrowsingDatabase::~SafeBrowsingDatabase() {
381 base::FilePath
SafeBrowsingDatabase::BrowseDBFilename(
382 const base::FilePath
& db_base_filename
) {
383 return base::FilePath(db_base_filename
.value() + kBrowseDBFile
);
387 base::FilePath
SafeBrowsingDatabase::DownloadDBFilename(
388 const base::FilePath
& db_base_filename
) {
389 return base::FilePath(db_base_filename
.value() + kDownloadDBFile
);
393 base::FilePath
SafeBrowsingDatabase::BloomFilterForFilename(
394 const base::FilePath
& db_filename
) {
395 return base::FilePath(db_filename
.value() + kBloomFilterFile
);
399 base::FilePath
SafeBrowsingDatabase::PrefixSetForFilename(
400 const base::FilePath
& db_filename
) {
401 return base::FilePath(db_filename
.value() + kPrefixSetFile
);
405 base::FilePath
SafeBrowsingDatabase::CsdWhitelistDBFilename(
406 const base::FilePath
& db_filename
) {
407 return base::FilePath(db_filename
.value() + kCsdWhitelistDBFile
);
411 base::FilePath
SafeBrowsingDatabase::DownloadWhitelistDBFilename(
412 const base::FilePath
& db_filename
) {
413 return base::FilePath(db_filename
.value() + kDownloadWhitelistDBFile
);
417 base::FilePath
SafeBrowsingDatabase::ExtensionBlacklistDBFilename(
418 const base::FilePath
& db_filename
) {
419 return base::FilePath(db_filename
.value() + kExtensionBlacklistDBFile
);
423 base::FilePath
SafeBrowsingDatabase::SideEffectFreeWhitelistDBFilename(
424 const base::FilePath
& db_filename
) {
425 return base::FilePath(db_filename
.value() + kSideEffectFreeWhitelistDBFile
);
429 base::FilePath
SafeBrowsingDatabase::IpBlacklistDBFilename(
430 const base::FilePath
& db_filename
) {
431 return base::FilePath(db_filename
.value() + kIPBlacklistDBFile
);
434 SafeBrowsingStore
* SafeBrowsingDatabaseNew::GetStore(const int list_id
) {
435 if (list_id
== safe_browsing_util::PHISH
||
436 list_id
== safe_browsing_util::MALWARE
) {
437 return browse_store_
.get();
438 } else if (list_id
== safe_browsing_util::BINURL
) {
439 return download_store_
.get();
440 } else if (list_id
== safe_browsing_util::CSDWHITELIST
) {
441 return csd_whitelist_store_
.get();
442 } else if (list_id
== safe_browsing_util::DOWNLOADWHITELIST
) {
443 return download_whitelist_store_
.get();
444 } else if (list_id
== safe_browsing_util::EXTENSIONBLACKLIST
) {
445 return extension_blacklist_store_
.get();
446 } else if (list_id
== safe_browsing_util::SIDEEFFECTFREEWHITELIST
) {
447 return side_effect_free_whitelist_store_
.get();
448 } else if (list_id
== safe_browsing_util::IPBLACKLIST
) {
449 return ip_blacklist_store_
.get();
455 void SafeBrowsingDatabase::RecordFailure(FailureType failure_type
) {
456 UMA_HISTOGRAM_ENUMERATION("SB2.DatabaseFailure", failure_type
,
457 FAILURE_DATABASE_MAX
);
460 SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew()
461 : creation_loop_(base::MessageLoop::current()),
462 browse_store_(new SafeBrowsingStoreFile
),
463 reset_factory_(this),
464 corruption_detected_(false),
465 change_detected_(false) {
466 DCHECK(browse_store_
.get());
467 DCHECK(!download_store_
.get());
468 DCHECK(!csd_whitelist_store_
.get());
469 DCHECK(!download_whitelist_store_
.get());
470 DCHECK(!extension_blacklist_store_
.get());
471 DCHECK(!side_effect_free_whitelist_store_
.get());
472 DCHECK(!ip_blacklist_store_
.get());
475 SafeBrowsingDatabaseNew::SafeBrowsingDatabaseNew(
476 SafeBrowsingStore
* browse_store
,
477 SafeBrowsingStore
* download_store
,
478 SafeBrowsingStore
* csd_whitelist_store
,
479 SafeBrowsingStore
* download_whitelist_store
,
480 SafeBrowsingStore
* extension_blacklist_store
,
481 SafeBrowsingStore
* side_effect_free_whitelist_store
,
482 SafeBrowsingStore
* ip_blacklist_store
)
483 : creation_loop_(base::MessageLoop::current()),
484 browse_store_(browse_store
),
485 download_store_(download_store
),
486 csd_whitelist_store_(csd_whitelist_store
),
487 download_whitelist_store_(download_whitelist_store
),
488 extension_blacklist_store_(extension_blacklist_store
),
489 side_effect_free_whitelist_store_(side_effect_free_whitelist_store
),
490 ip_blacklist_store_(ip_blacklist_store
),
491 reset_factory_(this),
492 corruption_detected_(false) {
493 DCHECK(browse_store_
.get());
496 SafeBrowsingDatabaseNew::~SafeBrowsingDatabaseNew() {
497 // The DCHECK is disabled due to crbug.com/338486 .
498 // DCHECK_EQ(creation_loop_, base::MessageLoop::current());
501 void SafeBrowsingDatabaseNew::Init(const base::FilePath
& filename_base
) {
502 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
503 // Ensure we haven't been run before.
504 DCHECK(browse_filename_
.empty());
505 DCHECK(download_filename_
.empty());
506 DCHECK(csd_whitelist_filename_
.empty());
507 DCHECK(download_whitelist_filename_
.empty());
508 DCHECK(extension_blacklist_filename_
.empty());
509 DCHECK(side_effect_free_whitelist_filename_
.empty());
510 DCHECK(ip_blacklist_filename_
.empty());
512 browse_filename_
= BrowseDBFilename(filename_base
);
513 browse_prefix_set_filename_
= PrefixSetForFilename(browse_filename_
);
517 base::Bind(&SafeBrowsingDatabaseNew::HandleCorruptDatabase
,
518 base::Unretained(this)));
519 DVLOG(1) << "Init browse store: " << browse_filename_
.value();
522 // NOTE: There is no need to grab the lock in this function, since
523 // until it returns, there are no pointers to this class on other
524 // threads. Then again, that means there is no possibility of
525 // contention on the lock...
526 base::AutoLock
locked(lookup_lock_
);
527 full_browse_hashes_
.clear();
528 cached_browse_hashes_
.clear();
532 if (download_store_
.get()) {
533 download_filename_
= DownloadDBFilename(filename_base
);
534 download_store_
->Init(
536 base::Bind(&SafeBrowsingDatabaseNew::HandleCorruptDatabase
,
537 base::Unretained(this)));
538 DVLOG(1) << "Init download store: " << download_filename_
.value();
541 if (csd_whitelist_store_
.get()) {
542 csd_whitelist_filename_
= CsdWhitelistDBFilename(filename_base
);
543 csd_whitelist_store_
->Init(
544 csd_whitelist_filename_
,
545 base::Bind(&SafeBrowsingDatabaseNew::HandleCorruptDatabase
,
546 base::Unretained(this)));
547 DVLOG(1) << "Init csd whitelist store: " << csd_whitelist_filename_
.value();
548 std::vector
<SBAddFullHash
> full_hashes
;
549 if (csd_whitelist_store_
->GetAddFullHashes(&full_hashes
)) {
550 LoadWhitelist(full_hashes
, &csd_whitelist_
);
552 WhitelistEverything(&csd_whitelist_
);
555 WhitelistEverything(&csd_whitelist_
); // Just to be safe.
558 if (download_whitelist_store_
.get()) {
559 download_whitelist_filename_
= DownloadWhitelistDBFilename(filename_base
);
560 download_whitelist_store_
->Init(
561 download_whitelist_filename_
,
562 base::Bind(&SafeBrowsingDatabaseNew::HandleCorruptDatabase
,
563 base::Unretained(this)));
564 DVLOG(1) << "Init download whitelist store: "
565 << download_whitelist_filename_
.value();
566 std::vector
<SBAddFullHash
> full_hashes
;
567 if (download_whitelist_store_
->GetAddFullHashes(&full_hashes
)) {
568 LoadWhitelist(full_hashes
, &download_whitelist_
);
570 WhitelistEverything(&download_whitelist_
);
573 WhitelistEverything(&download_whitelist_
); // Just to be safe.
576 if (extension_blacklist_store_
.get()) {
577 extension_blacklist_filename_
= ExtensionBlacklistDBFilename(filename_base
);
578 extension_blacklist_store_
->Init(
579 extension_blacklist_filename_
,
580 base::Bind(&SafeBrowsingDatabaseNew::HandleCorruptDatabase
,
581 base::Unretained(this)));
582 DVLOG(1) << "Init extension blacklist store: "
583 << extension_blacklist_filename_
.value();
586 if (side_effect_free_whitelist_store_
.get()) {
587 side_effect_free_whitelist_filename_
=
588 SideEffectFreeWhitelistDBFilename(filename_base
);
589 side_effect_free_whitelist_prefix_set_filename_
=
590 PrefixSetForFilename(side_effect_free_whitelist_filename_
);
591 side_effect_free_whitelist_store_
->Init(
592 side_effect_free_whitelist_filename_
,
593 base::Bind(&SafeBrowsingDatabaseNew::HandleCorruptDatabase
,
594 base::Unretained(this)));
595 DVLOG(1) << "Init side-effect free whitelist store: "
596 << side_effect_free_whitelist_filename_
.value();
598 // If there is no database, the filter cannot be used.
599 base::File::Info db_info
;
600 if (base::GetFileInfo(side_effect_free_whitelist_filename_
, &db_info
)
601 && db_info
.size
!= 0) {
602 const base::TimeTicks before
= base::TimeTicks::Now();
603 side_effect_free_whitelist_prefix_set_
=
604 safe_browsing::PrefixSet::LoadFile(
605 side_effect_free_whitelist_prefix_set_filename_
);
606 DVLOG(1) << "SafeBrowsingDatabaseNew read side-effect free whitelist "
608 << (base::TimeTicks::Now() - before
).InMilliseconds() << " ms";
609 UMA_HISTOGRAM_TIMES("SB2.SideEffectFreeWhitelistPrefixSetLoad",
610 base::TimeTicks::Now() - before
);
611 if (!side_effect_free_whitelist_prefix_set_
.get())
612 RecordFailure(FAILURE_SIDE_EFFECT_FREE_WHITELIST_PREFIX_SET_READ
);
615 // Delete any files of the side-effect free sidelist that may be around
616 // from when it was previously enabled.
617 SafeBrowsingStoreFile::DeleteStore(
618 SideEffectFreeWhitelistDBFilename(filename_base
));
621 if (ip_blacklist_store_
.get()) {
622 ip_blacklist_filename_
= IpBlacklistDBFilename(filename_base
);
623 ip_blacklist_store_
->Init(
624 ip_blacklist_filename_
,
625 base::Bind(&SafeBrowsingDatabaseNew::HandleCorruptDatabase
,
626 base::Unretained(this)));
627 DVLOG(1) << "SafeBrowsingDatabaseNew read ip blacklist: "
628 << ip_blacklist_filename_
.value();
629 std::vector
<SBAddFullHash
> full_hashes
;
630 if (ip_blacklist_store_
->GetAddFullHashes(&full_hashes
)) {
631 LoadIpBlacklist(full_hashes
);
633 DVLOG(1) << "Unable to load full hashes from the IP blacklist.";
634 LoadIpBlacklist(std::vector
<SBAddFullHash
>()); // Clear the list.
639 bool SafeBrowsingDatabaseNew::ResetDatabase() {
640 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
642 // Delete files on disk.
643 // TODO(shess): Hard to see where one might want to delete without a
644 // reset. Perhaps inline |Delete()|?
648 // Reset objects in memory.
650 base::AutoLock
locked(lookup_lock_
);
651 full_browse_hashes_
.clear();
652 cached_browse_hashes_
.clear();
653 prefix_miss_cache_
.clear();
654 browse_prefix_set_
.reset();
655 side_effect_free_whitelist_prefix_set_
.reset();
656 ip_blacklist_
.clear();
658 // Wants to acquire the lock itself.
659 WhitelistEverything(&csd_whitelist_
);
660 WhitelistEverything(&download_whitelist_
);
664 // TODO(lzheng): Remove matching_list, it is not used anywhere.
665 bool SafeBrowsingDatabaseNew::ContainsBrowseUrl(
667 std::string
* matching_list
,
668 std::vector
<SBPrefix
>* prefix_hits
,
669 std::vector
<SBFullHashResult
>* full_hits
,
670 base::Time last_update
) {
671 // Clear the results first.
672 matching_list
->clear();
673 prefix_hits
->clear();
676 std::vector
<SBFullHash
> full_hashes
;
677 BrowseFullHashesToCheck(url
, false, &full_hashes
);
678 if (full_hashes
.empty())
681 // This function is called on the I/O thread, prevent changes to
682 // filter and caches.
683 base::AutoLock
locked(lookup_lock_
);
685 // |browse_prefix_set_| is empty until it is either read from disk, or the
686 // first update populates it. Bail out without a hit if not yet
688 if (!browse_prefix_set_
.get())
691 size_t miss_count
= 0;
692 for (size_t i
= 0; i
< full_hashes
.size(); ++i
) {
693 const SBPrefix prefix
= full_hashes
[i
].prefix
;
694 if (browse_prefix_set_
->Exists(prefix
)) {
695 prefix_hits
->push_back(prefix
);
696 if (prefix_miss_cache_
.count(prefix
) > 0)
701 // If all the prefixes are cached as 'misses', don't issue a GetHash.
702 if (miss_count
== prefix_hits
->size())
705 // Find the matching full-hash results. |full_browse_hashes_| are from the
706 // database, |cached_browse_hashes_| are from GetHash requests between
708 std::sort(prefix_hits
->begin(), prefix_hits
->end());
710 GetCachedFullHashesForBrowse(*prefix_hits
, full_browse_hashes_
,
711 full_hits
, last_update
);
712 GetCachedFullHashesForBrowse(*prefix_hits
, cached_browse_hashes_
,
713 full_hits
, last_update
);
717 bool SafeBrowsingDatabaseNew::ContainsDownloadUrl(
718 const std::vector
<GURL
>& urls
,
719 std::vector
<SBPrefix
>* prefix_hits
) {
720 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
722 // Ignore this check when download checking is not enabled.
723 if (!download_store_
.get())
726 std::vector
<SBPrefix
> prefixes
;
727 GetDownloadUrlPrefixes(urls
, &prefixes
);
728 return MatchAddPrefixes(download_store_
.get(),
729 safe_browsing_util::BINURL
% 2,
734 bool SafeBrowsingDatabaseNew::ContainsCsdWhitelistedUrl(const GURL
& url
) {
735 // This method is theoretically thread-safe but we expect all calls to
736 // originate from the IO thread.
737 DCHECK(BrowserThread::CurrentlyOn(BrowserThread::IO
));
738 std::vector
<SBFullHash
> full_hashes
;
739 BrowseFullHashesToCheck(url
, true, &full_hashes
);
740 return ContainsWhitelistedHashes(csd_whitelist_
, full_hashes
);
743 bool SafeBrowsingDatabaseNew::ContainsDownloadWhitelistedUrl(const GURL
& url
) {
744 std::vector
<SBFullHash
> full_hashes
;
745 BrowseFullHashesToCheck(url
, true, &full_hashes
);
746 return ContainsWhitelistedHashes(download_whitelist_
, full_hashes
);
749 bool SafeBrowsingDatabaseNew::ContainsExtensionPrefixes(
750 const std::vector
<SBPrefix
>& prefixes
,
751 std::vector
<SBPrefix
>* prefix_hits
) {
752 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
753 if (!extension_blacklist_store_
)
756 return MatchAddPrefixes(extension_blacklist_store_
.get(),
757 safe_browsing_util::EXTENSIONBLACKLIST
% 2,
762 bool SafeBrowsingDatabaseNew::ContainsSideEffectFreeWhitelistUrl(
767 safe_browsing_util::CanonicalizeUrl(url
, &host
, &path
, &query
);
768 std::string url_to_check
= host
+ path
;
770 url_to_check
+= "?" + query
;
771 SBFullHash full_hash
= SBFullHashForString(url_to_check
);
773 // This function can be called on any thread, so lock against any changes
774 base::AutoLock
locked(lookup_lock_
);
776 // |side_effect_free_whitelist_prefix_set_| is empty until it is either read
777 // from disk, or the first update populates it. Bail out without a hit if
778 // not yet available.
779 if (!side_effect_free_whitelist_prefix_set_
.get())
782 return side_effect_free_whitelist_prefix_set_
->Exists(full_hash
.prefix
);
785 bool SafeBrowsingDatabaseNew::ContainsMalwareIP(const std::string
& ip_address
) {
786 net::IPAddressNumber ip_number
;
787 if (!net::ParseIPLiteralToNumber(ip_address
, &ip_number
)) {
788 DVLOG(2) << "Unable to parse IP address: '" << ip_address
<< "'";
791 if (ip_number
.size() == net::kIPv4AddressSize
) {
792 ip_number
= net::ConvertIPv4NumberToIPv6Number(ip_number
);
794 if (ip_number
.size() != net::kIPv6AddressSize
) {
795 DVLOG(2) << "Unable to convert IPv4 address to IPv6: '"
796 << ip_address
<< "'";
797 return false; // better safe than sorry.
799 // This function can be called from any thread.
800 base::AutoLock
locked(lookup_lock_
);
801 for (IPBlacklist::const_iterator it
= ip_blacklist_
.begin();
802 it
!= ip_blacklist_
.end();
804 const std::string
& mask
= it
->first
;
805 DCHECK_EQ(mask
.size(), ip_number
.size());
806 std::string
subnet(net::kIPv6AddressSize
, '\0');
807 for (size_t i
= 0; i
< net::kIPv6AddressSize
; ++i
) {
808 subnet
[i
] = ip_number
[i
] & mask
[i
];
810 const std::string hash
= base::SHA1HashString(subnet
);
811 DVLOG(2) << "Lookup Malware IP: "
812 << " ip:" << ip_address
813 << " mask:" << base::HexEncode(mask
.data(), mask
.size())
814 << " subnet:" << base::HexEncode(subnet
.data(), subnet
.size())
815 << " hash:" << base::HexEncode(hash
.data(), hash
.size());
816 if (it
->second
.count(hash
) > 0) {
823 bool SafeBrowsingDatabaseNew::ContainsDownloadWhitelistedString(
824 const std::string
& str
) {
825 std::vector
<SBFullHash
> hashes
;
826 hashes
.push_back(SBFullHashForString(str
));
827 return ContainsWhitelistedHashes(download_whitelist_
, hashes
);
830 bool SafeBrowsingDatabaseNew::ContainsWhitelistedHashes(
831 const SBWhitelist
& whitelist
,
832 const std::vector
<SBFullHash
>& hashes
) {
833 base::AutoLock
l(lookup_lock_
);
834 if (whitelist
.second
)
836 for (std::vector
<SBFullHash
>::const_iterator it
= hashes
.begin();
837 it
!= hashes
.end(); ++it
) {
838 if (std::binary_search(whitelist
.first
.begin(), whitelist
.first
.end(),
839 *it
, SBFullHashLess
)) {
846 // Helper to insert entries for all of the prefixes or full hashes in
847 // |entry| into the store.
848 void SafeBrowsingDatabaseNew::InsertAdd(int chunk_id
, SBPrefix host
,
849 const SBEntry
* entry
, int list_id
) {
850 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
852 SafeBrowsingStore
* store
= GetStore(list_id
);
855 STATS_COUNTER("SB.HostInsert", 1);
856 const int encoded_chunk_id
= EncodeChunkId(chunk_id
, list_id
);
857 const int count
= entry
->prefix_count();
859 DCHECK(!entry
->IsSub());
861 // No prefixes, use host instead.
862 STATS_COUNTER("SB.PrefixAdd", 1);
863 store
->WriteAddPrefix(encoded_chunk_id
, host
);
864 } else if (entry
->IsPrefix()) {
866 for (int i
= 0; i
< count
; i
++) {
867 const SBPrefix prefix
= entry
->PrefixAt(i
);
868 STATS_COUNTER("SB.PrefixAdd", 1);
869 store
->WriteAddPrefix(encoded_chunk_id
, prefix
);
872 // Prefixes and hashes.
873 const base::Time receive_time
= base::Time::Now();
874 for (int i
= 0; i
< count
; ++i
) {
875 const SBFullHash full_hash
= entry
->FullHashAt(i
);
876 const SBPrefix prefix
= full_hash
.prefix
;
878 STATS_COUNTER("SB.PrefixAdd", 1);
879 store
->WriteAddPrefix(encoded_chunk_id
, prefix
);
881 STATS_COUNTER("SB.PrefixAddFull", 1);
882 store
->WriteAddHash(encoded_chunk_id
, receive_time
, full_hash
);
887 // Helper to iterate over all the entries in the hosts in |chunks| and
888 // add them to the store.
889 void SafeBrowsingDatabaseNew::InsertAddChunks(
890 const safe_browsing_util::ListType list_id
,
891 const SBChunkList
& chunks
) {
892 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
894 SafeBrowsingStore
* store
= GetStore(list_id
);
897 for (SBChunkList::const_iterator citer
= chunks
.begin();
898 citer
!= chunks
.end(); ++citer
) {
899 const int chunk_id
= citer
->chunk_number
;
901 // The server can give us a chunk that we already have because
902 // it's part of a range. Don't add it again.
903 const int encoded_chunk_id
= EncodeChunkId(chunk_id
, list_id
);
904 if (store
->CheckAddChunk(encoded_chunk_id
))
907 store
->SetAddChunk(encoded_chunk_id
);
908 for (std::deque
<SBChunkHost
>::const_iterator hiter
= citer
->hosts
.begin();
909 hiter
!= citer
->hosts
.end(); ++hiter
) {
910 // NOTE: Could pass |encoded_chunk_id|, but then inserting add
911 // chunks would look different from inserting sub chunks.
912 InsertAdd(chunk_id
, hiter
->host
, hiter
->entry
, list_id
);
917 // Helper to insert entries for all of the prefixes or full hashes in
918 // |entry| into the store.
919 void SafeBrowsingDatabaseNew::InsertSub(int chunk_id
, SBPrefix host
,
920 const SBEntry
* entry
, int list_id
) {
921 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
923 SafeBrowsingStore
* store
= GetStore(list_id
);
926 STATS_COUNTER("SB.HostDelete", 1);
927 const int encoded_chunk_id
= EncodeChunkId(chunk_id
, list_id
);
928 const int count
= entry
->prefix_count();
930 DCHECK(entry
->IsSub());
932 // No prefixes, use host instead.
933 STATS_COUNTER("SB.PrefixSub", 1);
934 const int add_chunk_id
= EncodeChunkId(entry
->chunk_id(), list_id
);
935 store
->WriteSubPrefix(encoded_chunk_id
, add_chunk_id
, host
);
936 } else if (entry
->IsPrefix()) {
938 for (int i
= 0; i
< count
; i
++) {
939 const SBPrefix prefix
= entry
->PrefixAt(i
);
940 const int add_chunk_id
=
941 EncodeChunkId(entry
->ChunkIdAtPrefix(i
), list_id
);
943 STATS_COUNTER("SB.PrefixSub", 1);
944 store
->WriteSubPrefix(encoded_chunk_id
, add_chunk_id
, prefix
);
947 // Prefixes and hashes.
948 for (int i
= 0; i
< count
; ++i
) {
949 const SBFullHash full_hash
= entry
->FullHashAt(i
);
950 const int add_chunk_id
=
951 EncodeChunkId(entry
->ChunkIdAtPrefix(i
), list_id
);
953 STATS_COUNTER("SB.PrefixSub", 1);
954 store
->WriteSubPrefix(encoded_chunk_id
, add_chunk_id
, full_hash
.prefix
);
956 STATS_COUNTER("SB.PrefixSubFull", 1);
957 store
->WriteSubHash(encoded_chunk_id
, add_chunk_id
, full_hash
);
962 // Helper to iterate over all the entries in the hosts in |chunks| and
963 // add them to the store.
964 void SafeBrowsingDatabaseNew::InsertSubChunks(
965 safe_browsing_util::ListType list_id
,
966 const SBChunkList
& chunks
) {
967 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
969 SafeBrowsingStore
* store
= GetStore(list_id
);
972 for (SBChunkList::const_iterator citer
= chunks
.begin();
973 citer
!= chunks
.end(); ++citer
) {
974 const int chunk_id
= citer
->chunk_number
;
976 // The server can give us a chunk that we already have because
977 // it's part of a range. Don't add it again.
978 const int encoded_chunk_id
= EncodeChunkId(chunk_id
, list_id
);
979 if (store
->CheckSubChunk(encoded_chunk_id
))
982 store
->SetSubChunk(encoded_chunk_id
);
983 for (std::deque
<SBChunkHost
>::const_iterator hiter
= citer
->hosts
.begin();
984 hiter
!= citer
->hosts
.end(); ++hiter
) {
985 InsertSub(chunk_id
, hiter
->host
, hiter
->entry
, list_id
);
990 void SafeBrowsingDatabaseNew::InsertChunks(const std::string
& list_name
,
991 const SBChunkList
& chunks
) {
992 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
994 if (corruption_detected_
|| chunks
.empty())
997 const base::TimeTicks before
= base::TimeTicks::Now();
999 const safe_browsing_util::ListType list_id
=
1000 safe_browsing_util::GetListId(list_name
);
1001 DVLOG(2) << list_name
<< ": " << list_id
;
1003 SafeBrowsingStore
* store
= GetStore(list_id
);
1006 change_detected_
= true;
1008 store
->BeginChunk();
1009 if (chunks
.front().is_add
) {
1010 InsertAddChunks(list_id
, chunks
);
1012 InsertSubChunks(list_id
, chunks
);
1014 store
->FinishChunk();
1016 UMA_HISTOGRAM_TIMES("SB2.ChunkInsert", base::TimeTicks::Now() - before
);
1019 void SafeBrowsingDatabaseNew::DeleteChunks(
1020 const std::vector
<SBChunkDelete
>& chunk_deletes
) {
1021 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
1023 if (corruption_detected_
|| chunk_deletes
.empty())
1026 const std::string
& list_name
= chunk_deletes
.front().list_name
;
1027 const safe_browsing_util::ListType list_id
=
1028 safe_browsing_util::GetListId(list_name
);
1030 SafeBrowsingStore
* store
= GetStore(list_id
);
1033 change_detected_
= true;
1035 for (size_t i
= 0; i
< chunk_deletes
.size(); ++i
) {
1036 std::vector
<int> chunk_numbers
;
1037 RangesToChunks(chunk_deletes
[i
].chunk_del
, &chunk_numbers
);
1038 for (size_t j
= 0; j
< chunk_numbers
.size(); ++j
) {
1039 const int encoded_chunk_id
= EncodeChunkId(chunk_numbers
[j
], list_id
);
1040 if (chunk_deletes
[i
].is_sub_del
)
1041 store
->DeleteSubChunk(encoded_chunk_id
);
1043 store
->DeleteAddChunk(encoded_chunk_id
);
1048 void SafeBrowsingDatabaseNew::CacheHashResults(
1049 const std::vector
<SBPrefix
>& prefixes
,
1050 const std::vector
<SBFullHashResult
>& full_hits
) {
1051 // This is called on the I/O thread, lock against updates.
1052 base::AutoLock
locked(lookup_lock_
);
1054 if (full_hits
.empty()) {
1055 prefix_miss_cache_
.insert(prefixes
.begin(), prefixes
.end());
1059 const base::Time now
= base::Time::Now();
1060 const size_t orig_size
= cached_browse_hashes_
.size();
1061 for (std::vector
<SBFullHashResult
>::const_iterator iter
= full_hits
.begin();
1062 iter
!= full_hits
.end(); ++iter
) {
1063 if (iter
->list_id
== safe_browsing_util::MALWARE
||
1064 iter
->list_id
== safe_browsing_util::PHISH
) {
1065 SBFullHashCached cached_hash
;
1066 cached_hash
.hash
= iter
->hash
;
1067 cached_hash
.list_id
= iter
->list_id
;
1068 cached_hash
.received
= static_cast<int>(now
.ToTimeT());
1069 cached_browse_hashes_
.push_back(cached_hash
);
1073 // Sort new entries then merge with the previously-sorted entries.
1074 std::vector
<SBFullHashCached
>::iterator
1075 orig_end
= cached_browse_hashes_
.begin() + orig_size
;
1076 std::sort(orig_end
, cached_browse_hashes_
.end(), SBFullHashCachedPrefixLess
);
1077 std::inplace_merge(cached_browse_hashes_
.begin(),
1078 orig_end
, cached_browse_hashes_
.end(),
1079 SBFullHashCachedPrefixLess
);
1082 bool SafeBrowsingDatabaseNew::UpdateStarted(
1083 std::vector
<SBListChunkRanges
>* lists
) {
1084 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
1087 // If |BeginUpdate()| fails, reset the database.
1088 if (!browse_store_
->BeginUpdate()) {
1089 RecordFailure(FAILURE_BROWSE_DATABASE_UPDATE_BEGIN
);
1090 HandleCorruptDatabase();
1094 if (download_store_
.get() && !download_store_
->BeginUpdate()) {
1095 RecordFailure(FAILURE_DOWNLOAD_DATABASE_UPDATE_BEGIN
);
1096 HandleCorruptDatabase();
1100 if (csd_whitelist_store_
.get() && !csd_whitelist_store_
->BeginUpdate()) {
1101 RecordFailure(FAILURE_WHITELIST_DATABASE_UPDATE_BEGIN
);
1102 HandleCorruptDatabase();
1106 if (download_whitelist_store_
.get() &&
1107 !download_whitelist_store_
->BeginUpdate()) {
1108 RecordFailure(FAILURE_WHITELIST_DATABASE_UPDATE_BEGIN
);
1109 HandleCorruptDatabase();
1113 if (extension_blacklist_store_
&&
1114 !extension_blacklist_store_
->BeginUpdate()) {
1115 RecordFailure(FAILURE_EXTENSION_BLACKLIST_UPDATE_BEGIN
);
1116 HandleCorruptDatabase();
1120 if (side_effect_free_whitelist_store_
&&
1121 !side_effect_free_whitelist_store_
->BeginUpdate()) {
1122 RecordFailure(FAILURE_SIDE_EFFECT_FREE_WHITELIST_UPDATE_BEGIN
);
1123 HandleCorruptDatabase();
1127 if (ip_blacklist_store_
&& !ip_blacklist_store_
->BeginUpdate()) {
1128 RecordFailure(FAILURE_IP_BLACKLIST_UPDATE_BEGIN
);
1129 HandleCorruptDatabase();
1133 UpdateChunkRangesForLists(browse_store_
.get(),
1134 safe_browsing_util::kMalwareList
,
1135 safe_browsing_util::kPhishingList
,
1138 // NOTE(shess): |download_store_| used to contain kBinHashList, which has been
1139 // deprecated. Code to delete the list from the store shows ~15k hits/day as
1140 // of Feb 2014, so it has been removed. Everything _should_ be resilient to
1141 // extra data of that sort.
1142 UpdateChunkRangesForList(download_store_
.get(),
1143 safe_browsing_util::kBinUrlList
, lists
);
1145 UpdateChunkRangesForList(csd_whitelist_store_
.get(),
1146 safe_browsing_util::kCsdWhiteList
, lists
);
1148 UpdateChunkRangesForList(download_whitelist_store_
.get(),
1149 safe_browsing_util::kDownloadWhiteList
, lists
);
1151 UpdateChunkRangesForList(extension_blacklist_store_
.get(),
1152 safe_browsing_util::kExtensionBlacklist
, lists
);
1154 UpdateChunkRangesForList(side_effect_free_whitelist_store_
.get(),
1155 safe_browsing_util::kSideEffectFreeWhitelist
, lists
);
1157 UpdateChunkRangesForList(ip_blacklist_store_
.get(),
1158 safe_browsing_util::kIPBlacklist
, lists
);
1160 corruption_detected_
= false;
1161 change_detected_
= false;
1165 void SafeBrowsingDatabaseNew::UpdateFinished(bool update_succeeded
) {
1166 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
1168 // The update may have failed due to corrupt storage (for instance,
1169 // an excessive number of invalid add_chunks and sub_chunks).
1170 // Double-check that the databases are valid.
1171 // TODO(shess): Providing a checksum for the add_chunk and sub_chunk
1172 // sections would allow throwing a corruption error in
1174 if (!update_succeeded
) {
1175 if (!browse_store_
->CheckValidity())
1176 DLOG(ERROR
) << "Safe-browsing browse database corrupt.";
1178 if (download_store_
.get() && !download_store_
->CheckValidity())
1179 DLOG(ERROR
) << "Safe-browsing download database corrupt.";
1181 if (csd_whitelist_store_
.get() && !csd_whitelist_store_
->CheckValidity())
1182 DLOG(ERROR
) << "Safe-browsing csd whitelist database corrupt.";
1184 if (download_whitelist_store_
.get() &&
1185 !download_whitelist_store_
->CheckValidity()) {
1186 DLOG(ERROR
) << "Safe-browsing download whitelist database corrupt.";
1189 if (extension_blacklist_store_
&&
1190 !extension_blacklist_store_
->CheckValidity()) {
1191 DLOG(ERROR
) << "Safe-browsing extension blacklist database corrupt.";
1194 if (side_effect_free_whitelist_store_
&&
1195 !side_effect_free_whitelist_store_
->CheckValidity()) {
1196 DLOG(ERROR
) << "Safe-browsing side-effect free whitelist database "
1200 if (ip_blacklist_store_
&& !ip_blacklist_store_
->CheckValidity()) {
1201 DLOG(ERROR
) << "Safe-browsing IP blacklist database corrupt.";
1205 if (corruption_detected_
)
1208 // Unroll the transaction if there was a protocol error or if the
1209 // transaction was empty. This will leave the prefix set, the
1210 // pending hashes, and the prefix miss cache in place.
1211 if (!update_succeeded
|| !change_detected_
) {
1212 // Track empty updates to answer questions at http://crbug.com/72216 .
1213 if (update_succeeded
&& !change_detected_
)
1214 UMA_HISTOGRAM_COUNTS("SB2.DatabaseUpdateKilobytes", 0);
1215 browse_store_
->CancelUpdate();
1216 if (download_store_
.get())
1217 download_store_
->CancelUpdate();
1218 if (csd_whitelist_store_
.get())
1219 csd_whitelist_store_
->CancelUpdate();
1220 if (download_whitelist_store_
.get())
1221 download_whitelist_store_
->CancelUpdate();
1222 if (extension_blacklist_store_
)
1223 extension_blacklist_store_
->CancelUpdate();
1224 if (side_effect_free_whitelist_store_
)
1225 side_effect_free_whitelist_store_
->CancelUpdate();
1226 if (ip_blacklist_store_
)
1227 ip_blacklist_store_
->CancelUpdate();
1231 if (download_store_
) {
1232 int64 size_bytes
= UpdateHashPrefixStore(
1234 download_store_
.get(),
1235 FAILURE_DOWNLOAD_DATABASE_UPDATE_FINISH
);
1236 UMA_HISTOGRAM_COUNTS("SB2.DownloadDatabaseKilobytes",
1237 static_cast<int>(size_bytes
/ 1024));
1240 UpdateBrowseStore();
1241 UpdateWhitelistStore(csd_whitelist_filename_
,
1242 csd_whitelist_store_
.get(),
1244 UpdateWhitelistStore(download_whitelist_filename_
,
1245 download_whitelist_store_
.get(),
1246 &download_whitelist_
);
1248 if (extension_blacklist_store_
) {
1249 int64 size_bytes
= UpdateHashPrefixStore(
1250 extension_blacklist_filename_
,
1251 extension_blacklist_store_
.get(),
1252 FAILURE_EXTENSION_BLACKLIST_UPDATE_FINISH
);
1253 UMA_HISTOGRAM_COUNTS("SB2.ExtensionBlacklistKilobytes",
1254 static_cast<int>(size_bytes
/ 1024));
1257 if (side_effect_free_whitelist_store_
)
1258 UpdateSideEffectFreeWhitelistStore();
1260 if (ip_blacklist_store_
)
1261 UpdateIpBlacklistStore();
1264 void SafeBrowsingDatabaseNew::UpdateWhitelistStore(
1265 const base::FilePath
& store_filename
,
1266 SafeBrowsingStore
* store
,
1267 SBWhitelist
* whitelist
) {
1271 // Note: |builder| will not be empty. The current data store implementation
1272 // stores all full-length hashes as both full and prefix hashes.
1273 safe_browsing::PrefixSetBuilder builder
;
1274 std::vector
<SBAddFullHash
> full_hashes
;
1275 if (!store
->FinishUpdate(&builder
, &full_hashes
)) {
1276 RecordFailure(FAILURE_WHITELIST_DATABASE_UPDATE_FINISH
);
1277 WhitelistEverything(whitelist
);
1281 #if defined(OS_MACOSX)
1282 base::mac::SetFileBackupExclusion(store_filename
);
1285 LoadWhitelist(full_hashes
, whitelist
);
1288 int64
SafeBrowsingDatabaseNew::UpdateHashPrefixStore(
1289 const base::FilePath
& store_filename
,
1290 SafeBrowsingStore
* store
,
1291 FailureType failure_type
) {
1292 // These results are not used after this call. Simply ignore the
1293 // returned value after FinishUpdate(...).
1294 safe_browsing::PrefixSetBuilder builder
;
1295 std::vector
<SBAddFullHash
> add_full_hashes_result
;
1297 if (!store
->FinishUpdate(&builder
, &add_full_hashes_result
))
1298 RecordFailure(failure_type
);
1300 #if defined(OS_MACOSX)
1301 base::mac::SetFileBackupExclusion(store_filename
);
1304 return GetFileSizeOrZero(store_filename
);
1307 void SafeBrowsingDatabaseNew::UpdateBrowseStore() {
1308 // Measure the amount of IO during the filter build.
1309 base::IoCounters io_before
, io_after
;
1310 base::ProcessHandle handle
= base::Process::Current().handle();
1311 scoped_ptr
<base::ProcessMetrics
> metric(
1312 #if !defined(OS_MACOSX)
1313 base::ProcessMetrics::CreateProcessMetrics(handle
)
1315 // Getting stats only for the current process is enough, so NULL is fine.
1316 base::ProcessMetrics::CreateProcessMetrics(handle
, NULL
)
1320 // IoCounters are currently not supported on Mac, and may not be
1321 // available for Linux, so we check the result and only show IO
1322 // stats if they are available.
1323 const bool got_counters
= metric
->GetIOCounters(&io_before
);
1325 const base::TimeTicks before
= base::TimeTicks::Now();
1327 safe_browsing::PrefixSetBuilder builder
;
1328 std::vector
<SBAddFullHash
> add_full_hashes
;
1329 if (!browse_store_
->FinishUpdate(&builder
, &add_full_hashes
)) {
1330 RecordFailure(FAILURE_BROWSE_DATABASE_UPDATE_FINISH
);
1333 scoped_ptr
<safe_browsing::PrefixSet
> prefix_set(builder
.GetPrefixSet());
1335 std::vector
<SBFullHashCached
> full_hash_results
;
1336 for (size_t i
= 0; i
< add_full_hashes
.size(); ++i
) {
1337 SBFullHashCached result
;
1338 result
.hash
= add_full_hashes
[i
].full_hash
;
1339 result
.list_id
= GetListIdBit(add_full_hashes
[i
].chunk_id
);
1340 result
.received
= add_full_hashes
[i
].received
;
1341 full_hash_results
.push_back(result
);
1344 // This needs to be in sorted order by prefix for efficient access.
1345 std::sort(full_hash_results
.begin(), full_hash_results
.end(),
1346 SBFullHashCachedPrefixLess
);
1348 // Swap in the newly built filter and cache.
1350 base::AutoLock
locked(lookup_lock_
);
1351 full_browse_hashes_
.swap(full_hash_results
);
1353 // TODO(shess): If |CacheHashResults()| is posted between the
1354 // earlier lock and this clear, those pending hashes will be lost.
1355 // It could be fixed by only removing hashes which were collected
1356 // at the earlier point. I believe that is fail-safe as-is (the
1357 // hash will be fetched again).
1358 cached_browse_hashes_
.clear();
1359 prefix_miss_cache_
.clear();
1360 browse_prefix_set_
.swap(prefix_set
);
1363 DVLOG(1) << "SafeBrowsingDatabaseImpl built prefix set in "
1364 << (base::TimeTicks::Now() - before
).InMilliseconds()
1366 UMA_HISTOGRAM_LONG_TIMES("SB2.BuildFilter", base::TimeTicks::Now() - before
);
1368 // Persist the prefix set to disk. Since only this thread changes
1369 // |browse_prefix_set_|, there is no need to lock.
1372 // Gather statistics.
1373 if (got_counters
&& metric
->GetIOCounters(&io_after
)) {
1374 UMA_HISTOGRAM_COUNTS("SB2.BuildReadKilobytes",
1375 static_cast<int>(io_after
.ReadTransferCount
-
1376 io_before
.ReadTransferCount
) / 1024);
1377 UMA_HISTOGRAM_COUNTS("SB2.BuildWriteKilobytes",
1378 static_cast<int>(io_after
.WriteTransferCount
-
1379 io_before
.WriteTransferCount
) / 1024);
1380 UMA_HISTOGRAM_COUNTS("SB2.BuildReadOperations",
1381 static_cast<int>(io_after
.ReadOperationCount
-
1382 io_before
.ReadOperationCount
));
1383 UMA_HISTOGRAM_COUNTS("SB2.BuildWriteOperations",
1384 static_cast<int>(io_after
.WriteOperationCount
-
1385 io_before
.WriteOperationCount
));
1388 int64 file_size
= GetFileSizeOrZero(browse_prefix_set_filename_
);
1389 UMA_HISTOGRAM_COUNTS("SB2.PrefixSetKilobytes",
1390 static_cast<int>(file_size
/ 1024));
1391 file_size
= GetFileSizeOrZero(browse_filename_
);
1392 UMA_HISTOGRAM_COUNTS("SB2.BrowseDatabaseKilobytes",
1393 static_cast<int>(file_size
/ 1024));
1395 #if defined(OS_MACOSX)
1396 base::mac::SetFileBackupExclusion(browse_filename_
);
1400 void SafeBrowsingDatabaseNew::UpdateSideEffectFreeWhitelistStore() {
1401 safe_browsing::PrefixSetBuilder builder
;
1402 std::vector
<SBAddFullHash
> add_full_hashes_result
;
1404 if (!side_effect_free_whitelist_store_
->FinishUpdate(
1406 &add_full_hashes_result
)) {
1407 RecordFailure(FAILURE_SIDE_EFFECT_FREE_WHITELIST_UPDATE_FINISH
);
1410 scoped_ptr
<safe_browsing::PrefixSet
> prefix_set(builder
.GetPrefixSet());
1412 // Swap in the newly built prefix set.
1414 base::AutoLock
locked(lookup_lock_
);
1415 side_effect_free_whitelist_prefix_set_
.swap(prefix_set
);
1418 const base::TimeTicks before
= base::TimeTicks::Now();
1419 const bool write_ok
= side_effect_free_whitelist_prefix_set_
->WriteFile(
1420 side_effect_free_whitelist_prefix_set_filename_
);
1421 DVLOG(1) << "SafeBrowsingDatabaseNew wrote side-effect free whitelist prefix "
1422 << "set in " << (base::TimeTicks::Now() - before
).InMilliseconds()
1424 UMA_HISTOGRAM_TIMES("SB2.SideEffectFreePrefixSetWrite",
1425 base::TimeTicks::Now() - before
);
1428 RecordFailure(FAILURE_SIDE_EFFECT_FREE_WHITELIST_PREFIX_SET_WRITE
);
1430 // Gather statistics.
1431 int64 file_size
= GetFileSizeOrZero(
1432 side_effect_free_whitelist_prefix_set_filename_
);
1433 UMA_HISTOGRAM_COUNTS("SB2.SideEffectFreeWhitelistPrefixSetKilobytes",
1434 static_cast<int>(file_size
/ 1024));
1435 file_size
= GetFileSizeOrZero(side_effect_free_whitelist_filename_
);
1436 UMA_HISTOGRAM_COUNTS("SB2.SideEffectFreeWhitelistDatabaseKilobytes",
1437 static_cast<int>(file_size
/ 1024));
1439 #if defined(OS_MACOSX)
1440 base::mac::SetFileBackupExclusion(side_effect_free_whitelist_filename_
);
1441 base::mac::SetFileBackupExclusion(
1442 side_effect_free_whitelist_prefix_set_filename_
);
1446 void SafeBrowsingDatabaseNew::UpdateIpBlacklistStore() {
1447 // Note: prefixes will not be empty. The current data store implementation
1448 // stores all full-length hashes as both full and prefix hashes.
1449 safe_browsing::PrefixSetBuilder builder
;
1450 std::vector
<SBAddFullHash
> full_hashes
;
1451 if (!ip_blacklist_store_
->FinishUpdate(&builder
, &full_hashes
)) {
1452 RecordFailure(FAILURE_IP_BLACKLIST_UPDATE_FINISH
);
1453 LoadIpBlacklist(std::vector
<SBAddFullHash
>()); // Clear the list.
1457 #if defined(OS_MACOSX)
1458 base::mac::SetFileBackupExclusion(ip_blacklist_filename_
);
1461 LoadIpBlacklist(full_hashes
);
1464 void SafeBrowsingDatabaseNew::HandleCorruptDatabase() {
1465 // Reset the database after the current task has unwound (but only
1466 // reset once within the scope of a given task).
1467 if (!reset_factory_
.HasWeakPtrs()) {
1468 RecordFailure(FAILURE_DATABASE_CORRUPT
);
1469 base::MessageLoop::current()->PostTask(FROM_HERE
,
1470 base::Bind(&SafeBrowsingDatabaseNew::OnHandleCorruptDatabase
,
1471 reset_factory_
.GetWeakPtr()));
1475 void SafeBrowsingDatabaseNew::OnHandleCorruptDatabase() {
1476 RecordFailure(FAILURE_DATABASE_CORRUPT_HANDLER
);
1477 corruption_detected_
= true; // Stop updating the database.
1479 DLOG(FATAL
) << "SafeBrowsing database was corrupt and reset";
1482 // TODO(shess): I'm not clear why this code doesn't have any
1483 // real error-handling.
1484 void SafeBrowsingDatabaseNew::LoadPrefixSet() {
1485 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
1486 DCHECK(!browse_prefix_set_filename_
.empty());
1488 // If there is no database, the filter cannot be used.
1489 base::File::Info db_info
;
1490 if (!base::GetFileInfo(browse_filename_
, &db_info
) || db_info
.size
== 0)
1493 // Cleanup any stale bloom filter (no longer used).
1494 // TODO(shess): Track failure to delete?
1495 base::FilePath bloom_filter_filename
=
1496 BloomFilterForFilename(browse_filename_
);
1497 base::DeleteFile(bloom_filter_filename
, false);
1499 const base::TimeTicks before
= base::TimeTicks::Now();
1500 browse_prefix_set_
= safe_browsing::PrefixSet::LoadFile(
1501 browse_prefix_set_filename_
);
1502 DVLOG(1) << "SafeBrowsingDatabaseNew read prefix set in "
1503 << (base::TimeTicks::Now() - before
).InMilliseconds() << " ms";
1504 UMA_HISTOGRAM_TIMES("SB2.PrefixSetLoad", base::TimeTicks::Now() - before
);
1506 if (!browse_prefix_set_
.get())
1507 RecordFailure(FAILURE_BROWSE_PREFIX_SET_READ
);
1510 bool SafeBrowsingDatabaseNew::Delete() {
1511 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
1513 const bool r1
= browse_store_
->Delete();
1515 RecordFailure(FAILURE_DATABASE_STORE_DELETE
);
1517 const bool r2
= download_store_
.get() ? download_store_
->Delete() : true;
1519 RecordFailure(FAILURE_DATABASE_STORE_DELETE
);
1521 const bool r3
= csd_whitelist_store_
.get() ?
1522 csd_whitelist_store_
->Delete() : true;
1524 RecordFailure(FAILURE_DATABASE_STORE_DELETE
);
1526 const bool r4
= download_whitelist_store_
.get() ?
1527 download_whitelist_store_
->Delete() : true;
1529 RecordFailure(FAILURE_DATABASE_STORE_DELETE
);
1531 base::FilePath bloom_filter_filename
=
1532 BloomFilterForFilename(browse_filename_
);
1533 const bool r5
= base::DeleteFile(bloom_filter_filename
, false);
1535 RecordFailure(FAILURE_DATABASE_FILTER_DELETE
);
1537 const bool r6
= base::DeleteFile(browse_prefix_set_filename_
, false);
1539 RecordFailure(FAILURE_BROWSE_PREFIX_SET_DELETE
);
1541 const bool r7
= base::DeleteFile(extension_blacklist_filename_
, false);
1543 RecordFailure(FAILURE_EXTENSION_BLACKLIST_DELETE
);
1545 const bool r8
= base::DeleteFile(side_effect_free_whitelist_filename_
,
1548 RecordFailure(FAILURE_SIDE_EFFECT_FREE_WHITELIST_DELETE
);
1550 const bool r9
= base::DeleteFile(
1551 side_effect_free_whitelist_prefix_set_filename_
,
1554 RecordFailure(FAILURE_SIDE_EFFECT_FREE_WHITELIST_PREFIX_SET_DELETE
);
1556 const bool r10
= base::DeleteFile(ip_blacklist_filename_
, false);
1558 RecordFailure(FAILURE_IP_BLACKLIST_DELETE
);
1560 return r1
&& r2
&& r3
&& r4
&& r5
&& r6
&& r7
&& r8
&& r9
&& r10
;
1563 void SafeBrowsingDatabaseNew::WritePrefixSet() {
1564 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
1566 if (!browse_prefix_set_
.get())
1569 const base::TimeTicks before
= base::TimeTicks::Now();
1570 const bool write_ok
= browse_prefix_set_
->WriteFile(
1571 browse_prefix_set_filename_
);
1572 DVLOG(1) << "SafeBrowsingDatabaseNew wrote prefix set in "
1573 << (base::TimeTicks::Now() - before
).InMilliseconds() << " ms";
1574 UMA_HISTOGRAM_TIMES("SB2.PrefixSetWrite", base::TimeTicks::Now() - before
);
1577 RecordFailure(FAILURE_BROWSE_PREFIX_SET_WRITE
);
1579 #if defined(OS_MACOSX)
1580 base::mac::SetFileBackupExclusion(browse_prefix_set_filename_
);
1584 void SafeBrowsingDatabaseNew::WhitelistEverything(SBWhitelist
* whitelist
) {
1585 base::AutoLock
locked(lookup_lock_
);
1586 whitelist
->second
= true;
1587 whitelist
->first
.clear();
1590 void SafeBrowsingDatabaseNew::LoadWhitelist(
1591 const std::vector
<SBAddFullHash
>& full_hashes
,
1592 SBWhitelist
* whitelist
) {
1593 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
1594 if (full_hashes
.size() > kMaxWhitelistSize
) {
1595 WhitelistEverything(whitelist
);
1599 std::vector
<SBFullHash
> new_whitelist
;
1600 new_whitelist
.reserve(full_hashes
.size());
1601 for (std::vector
<SBAddFullHash
>::const_iterator it
= full_hashes
.begin();
1602 it
!= full_hashes
.end(); ++it
) {
1603 new_whitelist
.push_back(it
->full_hash
);
1605 std::sort(new_whitelist
.begin(), new_whitelist
.end(), SBFullHashLess
);
1607 SBFullHash kill_switch
= SBFullHashForString(kWhitelistKillSwitchUrl
);
1608 if (std::binary_search(new_whitelist
.begin(), new_whitelist
.end(),
1609 kill_switch
, SBFullHashLess
)) {
1610 // The kill switch is whitelisted hence we whitelist all URLs.
1611 WhitelistEverything(whitelist
);
1613 base::AutoLock
locked(lookup_lock_
);
1614 whitelist
->second
= false;
1615 whitelist
->first
.swap(new_whitelist
);
1619 void SafeBrowsingDatabaseNew::LoadIpBlacklist(
1620 const std::vector
<SBAddFullHash
>& full_hashes
) {
1621 DCHECK_EQ(creation_loop_
, base::MessageLoop::current());
1622 IPBlacklist new_blacklist
;
1623 DVLOG(2) << "Writing IP blacklist of size: " << full_hashes
.size();
1624 for (std::vector
<SBAddFullHash
>::const_iterator it
= full_hashes
.begin();
1625 it
!= full_hashes
.end();
1627 const char* full_hash
= it
->full_hash
.full_hash
;
1628 DCHECK_EQ(crypto::kSHA256Length
, arraysize(it
->full_hash
.full_hash
));
1629 // The format of the IP blacklist is:
1630 // SHA-1(IPv6 prefix) + uint8(prefix size) + 11 unused bytes.
1631 std::string
hashed_ip_prefix(full_hash
, base::kSHA1Length
);
1632 size_t prefix_size
= static_cast<uint8
>(full_hash
[base::kSHA1Length
]);
1633 if (prefix_size
> kMaxIpPrefixSize
|| prefix_size
< kMinIpPrefixSize
) {
1634 DVLOG(2) << "Invalid IP prefix size in IP blacklist: " << prefix_size
;
1635 RecordFailure(FAILURE_IP_BLACKLIST_UPDATE_INVALID
);
1636 new_blacklist
.clear(); // Load empty blacklist.
1640 // We precompute the mask for the given subnet size to speed up lookups.
1641 // Basically we need to create a 16B long string which has the highest
1642 // |size| bits sets to one.
1643 std::string
mask(net::kIPv6AddressSize
, '\0');
1644 mask
.replace(0, prefix_size
/ 8, prefix_size
/ 8, '\xFF');
1645 if ((prefix_size
% 8) != 0) {
1646 mask
[prefix_size
/ 8] = 0xFF << (8 - (prefix_size
% 8));
1648 DVLOG(2) << "Inserting malicious IP: "
1649 << " raw:" << base::HexEncode(full_hash
, crypto::kSHA256Length
)
1650 << " mask:" << base::HexEncode(mask
.data(), mask
.size())
1651 << " prefix_size:" << prefix_size
1652 << " hashed_ip:" << base::HexEncode(hashed_ip_prefix
.data(),
1653 hashed_ip_prefix
.size());
1654 new_blacklist
[mask
].insert(hashed_ip_prefix
);
1657 base::AutoLock
locked(lookup_lock_
);
1658 ip_blacklist_
.swap(new_blacklist
);
1661 bool SafeBrowsingDatabaseNew::IsMalwareIPMatchKillSwitchOn() {
1662 SBFullHash malware_kill_switch
= SBFullHashForString(kMalwareIPKillSwitchUrl
);
1663 std::vector
<SBFullHash
> full_hashes
;
1664 full_hashes
.push_back(malware_kill_switch
);
1665 return ContainsWhitelistedHashes(csd_whitelist_
, full_hashes
);