1 // Copyright 2018-2019 Mozilla
3 // Licensed under the Apache License, Version 2.0 (the "License"); you may not use
4 // this file except in compliance with the License. You may obtain a copy of the
5 // License at http://www.apache.org/licenses/LICENSE-2.0
6 // Unless required by applicable law or agreed to in writing, software distributed
7 // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
8 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
9 // specific language governing permissions and limitations under the License.
11 use std::{fs, path::Path};
13 use tempfile::Builder;
16 backend::{Lmdb, LmdbEnvironment, SafeMode, SafeModeEnvironment},
17 Manager, Migrator, Rkv, StoreOptions, Value,
20 macro_rules! populate_store {
23 .open_single("store", StoreOptions::create())
25 let mut writer = $env.write().expect("writer");
27 .put(&mut writer, "foo", &Value::I64(1234))
30 .put(&mut writer, "bar", &Value::Bool(true))
33 .put(&mut writer, "baz", &Value::Str("héllo, yöu"))
35 writer.commit().expect("committed");
40 fn test_open_migrator_lmdb_to_safe() {
41 let root = Builder::new()
42 .prefix("test_open_migrator_lmdb_to_safe")
45 fs::create_dir_all(root.path()).expect("dir created");
47 // Populate source environment and persist to disk.
49 let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
50 populate_store!(&src_env);
51 src_env.sync(true).expect("synced");
53 // Check if the files were written to disk.
55 let mut datamdb = root.path().to_path_buf();
56 let mut lockmdb = root.path().to_path_buf();
57 datamdb.push("data.mdb");
58 lockmdb.push("lock.mdb");
59 assert!(datamdb.exists());
60 assert!(lockmdb.exists());
62 // Verify that database was written to disk.
64 let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
66 .open_single("store", StoreOptions::default())
68 let reader = src_env.read().expect("reader");
70 store.get(&reader, "foo").expect("read"),
71 Some(Value::I64(1234))
74 store.get(&reader, "bar").expect("read"),
75 Some(Value::Bool(true))
78 store.get(&reader, "baz").expect("read"),
79 Some(Value::Str("héllo, yöu"))
84 let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
85 Migrator::open_and_migrate_lmdb_to_safe_mode(root.path(), |builder| builder, &dst_env)
88 // Verify that the database was indeed migrated.
90 let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
92 .open_single("store", StoreOptions::default())
94 let reader = dst_env.read().expect("reader");
96 store.get(&reader, "foo").expect("read"),
97 Some(Value::I64(1234))
100 store.get(&reader, "bar").expect("read"),
101 Some(Value::Bool(true))
104 store.get(&reader, "baz").expect("read"),
105 Some(Value::Str("héllo, yöu"))
108 // Check if the old files were deleted from disk.
110 let mut datamdb = root.path().to_path_buf();
111 let mut lockmdb = root.path().to_path_buf();
112 datamdb.push("data.mdb");
113 lockmdb.push("lock.mdb");
114 assert!(!datamdb.exists());
115 assert!(!lockmdb.exists());
120 fn test_open_migrator_safe_to_lmdb() {
121 let root = Builder::new()
122 .prefix("test_open_migrator_safe_to_lmdb")
125 fs::create_dir_all(root.path()).expect("dir created");
127 // Populate source environment and persist to disk.
129 let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
130 populate_store!(&src_env);
131 src_env.sync(true).expect("synced");
133 // Check if the files were written to disk.
135 let mut safebin = root.path().to_path_buf();
136 safebin.push("data.safe.bin");
137 assert!(safebin.exists());
139 // Verify that database was written to disk.
141 let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
143 .open_single("store", StoreOptions::default())
145 let reader = src_env.read().expect("reader");
147 store.get(&reader, "foo").expect("read"),
148 Some(Value::I64(1234))
151 store.get(&reader, "bar").expect("read"),
152 Some(Value::Bool(true))
155 store.get(&reader, "baz").expect("read"),
156 Some(Value::Str("héllo, yöu"))
161 let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
162 Migrator::open_and_migrate_safe_mode_to_lmdb(root.path(), |builder| builder, &dst_env)
165 // Verify that the database was indeed migrated.
167 let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
169 .open_single("store", StoreOptions::default())
171 let reader = dst_env.read().expect("reader");
173 store.get(&reader, "foo").expect("read"),
174 Some(Value::I64(1234))
177 store.get(&reader, "bar").expect("read"),
178 Some(Value::Bool(true))
181 store.get(&reader, "baz").expect("read"),
182 Some(Value::Str("héllo, yöu"))
185 // Check if the old files were deleted from disk.
187 let mut safebin = root.path().to_path_buf();
188 safebin.push("data.safe.bin");
189 assert!(!safebin.exists());
194 fn test_open_migrator_round_trip() {
195 let root = Builder::new()
196 .prefix("test_open_migrator_lmdb_to_safe")
199 fs::create_dir_all(root.path()).expect("dir created");
201 // Populate source environment and persist to disk.
203 let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
204 populate_store!(&src_env);
205 src_env.sync(true).expect("synced");
209 let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
210 Migrator::open_and_migrate_lmdb_to_safe_mode(root.path(), |builder| builder, &dst_env)
213 // Open and migrate back.
215 let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
216 Migrator::open_and_migrate_safe_mode_to_lmdb(root.path(), |builder| builder, &dst_env)
219 // Verify that the database was indeed migrated twice.
221 let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
223 .open_single("store", StoreOptions::default())
225 let reader = dst_env.read().expect("reader");
227 store.get(&reader, "foo").expect("read"),
228 Some(Value::I64(1234))
231 store.get(&reader, "bar").expect("read"),
232 Some(Value::Bool(true))
235 store.get(&reader, "baz").expect("read"),
236 Some(Value::Str("héllo, yöu"))
239 // Check if the right files are finally present on disk.
241 let mut datamdb = root.path().to_path_buf();
242 let mut lockmdb = root.path().to_path_buf();
243 let mut safebin = root.path().to_path_buf();
244 datamdb.push("data.mdb");
245 lockmdb.push("lock.mdb");
246 safebin.push("data.safe.bin");
247 assert!(datamdb.exists());
248 assert!(lockmdb.exists());
249 assert!(!safebin.exists());
254 fn test_easy_migrator_no_dir_1() {
255 let root = Builder::new()
256 .prefix("test_easy_migrator_no_dir")
259 fs::create_dir_all(root.path()).expect("dir created");
261 // This won't fail with IoError even though the path is a bogus path, because this
262 // is the "easy mode" migration which automatically handles (ignores) this error.
263 let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
264 Migrator::easy_migrate_lmdb_to_safe_mode(Path::new("bogus"), &dst_env).expect("migrated");
266 let mut datamdb = root.path().to_path_buf();
267 let mut lockmdb = root.path().to_path_buf();
268 let mut safebin = root.path().to_path_buf();
269 datamdb.push("data.mdb");
270 lockmdb.push("lock.mdb");
271 safebin.push("data.safe.bin");
272 assert!(!datamdb.exists());
273 assert!(!lockmdb.exists());
274 assert!(!safebin.exists()); // safe mode doesn't write an empty db to disk
278 fn test_easy_migrator_no_dir_2() {
279 let root = Builder::new()
280 .prefix("test_easy_migrator_no_dir")
283 fs::create_dir_all(root.path()).expect("dir created");
285 // This won't fail with IoError even though the path is a bogus path, because this
286 // is the "easy mode" migration which automatically handles (ignores) this error.
287 let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
288 Migrator::easy_migrate_safe_mode_to_lmdb(Path::new("bogus"), &dst_env).expect("migrated");
290 let mut datamdb = root.path().to_path_buf();
291 let mut lockmdb = root.path().to_path_buf();
292 let mut safebin = root.path().to_path_buf();
293 datamdb.push("data.mdb");
294 lockmdb.push("lock.mdb");
295 safebin.push("data.safe.bin");
296 assert!(datamdb.exists()); // lmdb writes an empty db to disk
297 assert!(lockmdb.exists());
298 assert!(!safebin.exists());
302 fn test_easy_migrator_invalid_1() {
303 let root = Builder::new()
304 .prefix("test_easy_migrator_invalid")
307 fs::create_dir_all(root.path()).expect("dir created");
309 let dbfile = root.path().join("data.mdb");
310 fs::write(dbfile, "bogus").expect("dbfile created");
312 // This won't fail with FileInvalid even though the database is a bogus file, because this
313 // is the "easy mode" migration which automatically handles (ignores) this error.
314 let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
315 Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), &dst_env).expect("migrated");
317 let mut datamdb = root.path().to_path_buf();
318 let mut lockmdb = root.path().to_path_buf();
319 let mut safebin = root.path().to_path_buf();
320 datamdb.push("data.mdb");
321 lockmdb.push("lock.mdb");
322 safebin.push("data.safe.bin");
323 assert!(datamdb.exists()); // corrupted db isn't deleted
324 assert!(lockmdb.exists());
325 assert!(!safebin.exists());
329 fn test_easy_migrator_invalid_2() {
330 let root = Builder::new()
331 .prefix("test_easy_migrator_invalid")
334 fs::create_dir_all(root.path()).expect("dir created");
336 let dbfile = root.path().join("data.safe.bin");
337 fs::write(dbfile, "bogus").expect("dbfile created");
339 // This won't fail with FileInvalid even though the database is a bogus file, because this
340 // is the "easy mode" migration which automatically handles (ignores) this error.
341 let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
342 Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), &dst_env).expect("migrated");
344 let mut datamdb = root.path().to_path_buf();
345 let mut lockmdb = root.path().to_path_buf();
346 let mut safebin = root.path().to_path_buf();
347 datamdb.push("data.mdb");
348 lockmdb.push("lock.mdb");
349 safebin.push("data.safe.bin");
350 assert!(datamdb.exists()); // lmdb writes an empty db to disk
351 assert!(lockmdb.exists());
352 assert!(safebin.exists()); // corrupted db isn't deleted
356 #[should_panic(expected = "migrated: SourceEmpty")]
357 fn test_migrator_lmdb_to_safe_1() {
358 let root = Builder::new()
359 .prefix("test_migrate_lmdb_to_safe")
362 fs::create_dir_all(root.path()).expect("dir created");
364 let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
365 let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
366 Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated");
370 #[should_panic(expected = "migrated: DestinationNotEmpty")]
371 fn test_migrator_lmdb_to_safe_2() {
372 let root = Builder::new()
373 .prefix("test_migrate_lmdb_to_safe")
376 fs::create_dir_all(root.path()).expect("dir created");
378 let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
379 populate_store!(&src_env);
380 let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
381 populate_store!(&dst_env);
382 Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated");
386 fn test_migrator_lmdb_to_safe_3() {
387 let root = Builder::new()
388 .prefix("test_migrate_lmdb_to_safe")
391 fs::create_dir_all(root.path()).expect("dir created");
393 let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
394 populate_store!(&src_env);
395 let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
396 Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated");
399 .open_single("store", StoreOptions::default())
401 let reader = dst_env.read().expect("reader");
403 store.get(&reader, "foo").expect("read"),
404 Some(Value::I64(1234))
407 store.get(&reader, "bar").expect("read"),
408 Some(Value::Bool(true))
411 store.get(&reader, "baz").expect("read"),
412 Some(Value::Str("héllo, yöu"))
417 #[should_panic(expected = "migrated: SourceEmpty")]
418 fn test_migrator_safe_to_lmdb_1() {
419 let root = Builder::new()
420 .prefix("test_migrate_safe_to_lmdb")
423 fs::create_dir_all(root.path()).expect("dir created");
425 let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
426 let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
427 Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated");
431 #[should_panic(expected = "migrated: DestinationNotEmpty")]
432 fn test_migrator_safe_to_lmdb_2() {
433 let root = Builder::new()
434 .prefix("test_migrate_safe_to_lmdb")
437 fs::create_dir_all(root.path()).expect("dir created");
439 let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
440 populate_store!(&src_env);
441 let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
442 populate_store!(&dst_env);
443 Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated");
447 fn test_migrator_safe_to_lmdb_3() {
448 let root = Builder::new()
449 .prefix("test_migrate_safe_to_lmdb")
452 fs::create_dir_all(root.path()).expect("dir created");
454 let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
455 populate_store!(&src_env);
456 let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
457 Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated");
460 .open_single("store", StoreOptions::default())
462 let reader = dst_env.read().expect("reader");
464 store.get(&reader, "foo").expect("read"),
465 Some(Value::I64(1234))
468 store.get(&reader, "bar").expect("read"),
469 Some(Value::Bool(true))
472 store.get(&reader, "baz").expect("read"),
473 Some(Value::Str("héllo, yöu"))
478 fn test_easy_migrator_failed_migration_1() {
479 let root = Builder::new()
480 .prefix("test_easy_migrator_failed_migration_1")
483 fs::create_dir_all(root.path()).expect("dir created");
485 let dbfile = root.path().join("data.mdb");
486 fs::write(&dbfile, "bogus").expect("bogus dbfile created");
488 // This won't fail with FileInvalid even though the database is a bogus file, because this
489 // is the "easy mode" migration which automatically handles (ignores) this error.
490 let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
491 Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), &dst_env).expect("migrated");
493 // Populate destination environment and persist to disk.
494 populate_store!(&dst_env);
495 dst_env.sync(true).expect("synced");
497 // Delete bogus file and create a valid source environment in its place.
498 fs::remove_file(&dbfile).expect("bogus dbfile removed");
499 let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
500 populate_store!(&src_env);
501 src_env.sync(true).expect("synced");
503 // Attempt to migrate again. This should *NOT* fail with DestinationNotEmpty.
504 Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), &dst_env).expect("migrated");
508 fn test_easy_migrator_failed_migration_2() {
509 let root = Builder::new()
510 .prefix("test_easy_migrator_failed_migration_2")
513 fs::create_dir_all(root.path()).expect("dir created");
515 let dbfile = root.path().join("data.safe.bin");
516 fs::write(&dbfile, "bogus").expect("bogus dbfile created");
518 // This won't fail with FileInvalid even though the database is a bogus file, because this
519 // is the "easy mode" migration which automatically handles (ignores) this error.
520 let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
521 Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), &dst_env).expect("migrated");
523 // Populate destination environment and persist to disk.
524 populate_store!(&dst_env);
525 dst_env.sync(true).expect("synced");
527 // Delete bogus file and create a valid source environment in its place.
528 fs::remove_file(&dbfile).expect("bogus dbfile removed");
529 let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
530 populate_store!(&src_env);
531 src_env.sync(true).expect("synced");
533 // Attempt to migrate again. This should *NOT* fail with DestinationNotEmpty.
534 Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), &dst_env).expect("migrated");
537 fn test_easy_migrator_from_manager_failed_migration_1() {
538 let root = Builder::new()
539 .prefix("test_easy_migrator_from_manager_failed_migration_1")
542 fs::create_dir_all(root.path()).expect("dir created");
545 let mut src_manager = Manager::<LmdbEnvironment>::singleton().write().unwrap();
546 let created_src_arc = src_manager
547 .get_or_create(root.path(), Rkv::new::<Lmdb>)
549 let src_env = created_src_arc.read().unwrap();
550 populate_store!(&src_env);
551 src_env.sync(true).expect("synced");
554 let mut dst_manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap();
555 let created_dst_arc_1 = dst_manager
556 .get_or_create(root.path(), Rkv::new::<SafeMode>)
558 let dst_env_1 = created_dst_arc_1.read().unwrap();
559 populate_store!(&dst_env_1);
560 dst_env_1.sync(true).expect("synced");
563 // Attempt to migrate again in a new env. This should *NOT* fail with DestinationNotEmpty.
564 let dst_manager = Manager::<SafeModeEnvironment>::singleton().read().unwrap();
565 let created_dst_arc_2 = dst_manager.get(root.path()).unwrap().unwrap();
566 let dst_env_2 = created_dst_arc_2.read().unwrap();
567 Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), dst_env_2).expect("migrated");
570 fn test_easy_migrator_from_manager_failed_migration_2() {
571 let root = Builder::new()
572 .prefix("test_easy_migrator_from_manager_failed_migration_2")
575 fs::create_dir_all(root.path()).expect("dir created");
578 let mut src_manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap();
579 let created_src_arc = src_manager
580 .get_or_create(root.path(), Rkv::new::<SafeMode>)
582 let src_env = created_src_arc.read().unwrap();
583 populate_store!(&src_env);
584 src_env.sync(true).expect("synced");
587 let mut dst_manager = Manager::<LmdbEnvironment>::singleton().write().unwrap();
588 let created_dst_arc_1 = dst_manager
589 .get_or_create(root.path(), Rkv::new::<Lmdb>)
591 let dst_env_1 = created_dst_arc_1.read().unwrap();
592 populate_store!(&dst_env_1);
593 dst_env_1.sync(true).expect("synced");
596 // Attempt to migrate again in a new env. This should *NOT* fail with DestinationNotEmpty.
597 let dst_manager = Manager::<LmdbEnvironment>::singleton().read().unwrap();
598 let created_dst_arc_2 = dst_manager.get(root.path()).unwrap().unwrap();
599 let dst_env_2 = created_dst_arc_2.read().unwrap();
600 Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), dst_env_2).expect("migrated");
604 fn test_easy_migrator_from_manager_failed_migration() {
605 test_easy_migrator_from_manager_failed_migration_1();
606 test_easy_migrator_from_manager_failed_migration_2();