4 * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
5 * Copyright (C) 2006 Red Hat GmbH
7 * This file is released under the GPL.
16 #include <linux/pagemap.h>
17 #include <linux/vmalloc.h>
18 #include <linux/slab.h>
20 #define DM_MSG_PREFIX "snapshots"
21 #define DM_CHUNK_SIZE_DEFAULT_SECTORS 32 /* 16KB */
23 /*-----------------------------------------------------------------
24 * Persistent snapshots, by persistent we mean that the snapshot
25 * will survive a reboot.
26 *---------------------------------------------------------------*/
29 * We need to store a record of which parts of the origin have
30 * been copied to the snapshot device. The snapshot code
31 * requires that we copy exception chunks to chunk aligned areas
32 * of the COW store. It makes sense therefore, to store the
33 * metadata in chunk size blocks.
35 * There is no backward or forward compatibility implemented,
36 * snapshots with different disk versions than the kernel will
37 * not be usable. It is expected that "lvcreate" will blank out
38 * the start of a fresh COW device before calling the snapshot
41 * The first chunk of the COW device just contains the header.
42 * After this there is a chunk filled with exception metadata,
43 * followed by as many exception chunks as can fit in the
46 * All on disk structures are in little-endian format. The end
47 * of the exceptions info is indicated by an exception with a
48 * new_chunk of 0, which is invalid since it would point to the
53 * Magic for persistent snapshots: "SnAp" - Feeble isn't it.
55 #define SNAP_MAGIC 0x70416e53
58 * The on-disk version of the metadata.
60 #define SNAPSHOT_DISK_VERSION 1
66 * Is this snapshot valid. There is no way of recovering
67 * an invalid snapshot.
72 * Simple, incrementing version. no backward
81 struct disk_exception
{
86 struct commit_callback
{
87 void (*callback
)(void *, int success
);
92 * The top level structure for a persistent exception store.
95 struct dm_snapshot
*snap
; /* up pointer to my snapshot */
98 uint32_t exceptions_per_area
;
101 * Now that we have an asynchronous kcopyd there is no
102 * need for large chunk sizes, so it wont hurt to have a
103 * whole chunks worth of metadata in memory at once.
108 * Used to keep track of which metadata area the data in
111 uint32_t current_area
;
114 * The next free chunk for an exception.
119 * The index of next free exception in the current
122 uint32_t current_committed
;
124 atomic_t pending_count
;
125 uint32_t callback_count
;
126 struct commit_callback
*callbacks
;
127 struct dm_io_client
*io_client
;
129 struct workqueue_struct
*metadata_wq
;
132 static unsigned sectors_to_pages(unsigned sectors
)
134 return sectors
/ (PAGE_SIZE
>> 9);
137 static int alloc_area(struct pstore
*ps
)
142 len
= ps
->snap
->chunk_size
<< SECTOR_SHIFT
;
145 * Allocate the chunk_size block of memory that will hold
146 * a single metadata area.
148 ps
->area
= vmalloc(len
);
155 static void free_area(struct pstore
*ps
)
162 struct io_region
*where
;
163 struct dm_io_request
*io_req
;
164 struct work_struct work
;
168 static void do_metadata(struct work_struct
*work
)
170 struct mdata_req
*req
= container_of(work
, struct mdata_req
, work
);
172 req
->result
= dm_io(req
->io_req
, 1, req
->where
, NULL
);
176 * Read or write a chunk aligned and sized block of data from a device.
178 static int chunk_io(struct pstore
*ps
, uint32_t chunk
, int rw
, int metadata
)
180 struct io_region where
= {
181 .bdev
= ps
->snap
->cow
->bdev
,
182 .sector
= ps
->snap
->chunk_size
* chunk
,
183 .count
= ps
->snap
->chunk_size
,
185 struct dm_io_request io_req
= {
187 .mem
.type
= DM_IO_VMA
,
188 .mem
.ptr
.vma
= ps
->area
,
189 .client
= ps
->io_client
,
192 struct mdata_req req
;
195 return dm_io(&io_req
, 1, &where
, NULL
);
198 req
.io_req
= &io_req
;
201 * Issue the synchronous I/O from a different thread
202 * to avoid generic_make_request recursion.
204 INIT_WORK(&req
.work
, do_metadata
);
205 queue_work(ps
->metadata_wq
, &req
.work
);
206 flush_workqueue(ps
->metadata_wq
);
212 * Read or write a metadata area. Remembering to skip the first
213 * chunk which holds the header.
215 static int area_io(struct pstore
*ps
, uint32_t area
, int rw
)
220 /* convert a metadata area index to a chunk index */
221 chunk
= 1 + ((ps
->exceptions_per_area
+ 1) * area
);
223 r
= chunk_io(ps
, chunk
, rw
, 0);
227 ps
->current_area
= area
;
231 static int zero_area(struct pstore
*ps
, uint32_t area
)
233 memset(ps
->area
, 0, ps
->snap
->chunk_size
<< SECTOR_SHIFT
);
234 return area_io(ps
, area
, WRITE
);
237 static int read_header(struct pstore
*ps
, int *new_snapshot
)
240 struct disk_header
*dh
;
242 int chunk_size_supplied
= 1;
245 * Use default chunk size (or hardsect_size, if larger) if none supplied
247 if (!ps
->snap
->chunk_size
) {
248 ps
->snap
->chunk_size
= max(DM_CHUNK_SIZE_DEFAULT_SECTORS
,
249 bdev_hardsect_size(ps
->snap
->cow
->bdev
) >> 9);
250 ps
->snap
->chunk_mask
= ps
->snap
->chunk_size
- 1;
251 ps
->snap
->chunk_shift
= ffs(ps
->snap
->chunk_size
) - 1;
252 chunk_size_supplied
= 0;
255 ps
->io_client
= dm_io_client_create(sectors_to_pages(ps
->snap
->
257 if (IS_ERR(ps
->io_client
))
258 return PTR_ERR(ps
->io_client
);
264 r
= chunk_io(ps
, 0, READ
, 1);
268 dh
= (struct disk_header
*) ps
->area
;
270 if (le32_to_cpu(dh
->magic
) == 0) {
275 if (le32_to_cpu(dh
->magic
) != SNAP_MAGIC
) {
276 DMWARN("Invalid or corrupt snapshot");
282 ps
->valid
= le32_to_cpu(dh
->valid
);
283 ps
->version
= le32_to_cpu(dh
->version
);
284 chunk_size
= le32_to_cpu(dh
->chunk_size
);
286 if (!chunk_size_supplied
|| ps
->snap
->chunk_size
== chunk_size
)
289 DMWARN("chunk size %llu in device metadata overrides "
290 "table chunk size of %llu.",
291 (unsigned long long)chunk_size
,
292 (unsigned long long)ps
->snap
->chunk_size
);
294 /* We had a bogus chunk_size. Fix stuff up. */
297 ps
->snap
->chunk_size
= chunk_size
;
298 ps
->snap
->chunk_mask
= chunk_size
- 1;
299 ps
->snap
->chunk_shift
= ffs(chunk_size
) - 1;
301 r
= dm_io_client_resize(sectors_to_pages(ps
->snap
->chunk_size
),
314 static int write_header(struct pstore
*ps
)
316 struct disk_header
*dh
;
318 memset(ps
->area
, 0, ps
->snap
->chunk_size
<< SECTOR_SHIFT
);
320 dh
= (struct disk_header
*) ps
->area
;
321 dh
->magic
= cpu_to_le32(SNAP_MAGIC
);
322 dh
->valid
= cpu_to_le32(ps
->valid
);
323 dh
->version
= cpu_to_le32(ps
->version
);
324 dh
->chunk_size
= cpu_to_le32(ps
->snap
->chunk_size
);
326 return chunk_io(ps
, 0, WRITE
, 1);
330 * Access functions for the disk exceptions, these do the endian conversions.
332 static struct disk_exception
*get_exception(struct pstore
*ps
, uint32_t index
)
334 BUG_ON(index
>= ps
->exceptions_per_area
);
336 return ((struct disk_exception
*) ps
->area
) + index
;
339 static void read_exception(struct pstore
*ps
,
340 uint32_t index
, struct disk_exception
*result
)
342 struct disk_exception
*e
= get_exception(ps
, index
);
345 result
->old_chunk
= le64_to_cpu(e
->old_chunk
);
346 result
->new_chunk
= le64_to_cpu(e
->new_chunk
);
349 static void write_exception(struct pstore
*ps
,
350 uint32_t index
, struct disk_exception
*de
)
352 struct disk_exception
*e
= get_exception(ps
, index
);
355 e
->old_chunk
= cpu_to_le64(de
->old_chunk
);
356 e
->new_chunk
= cpu_to_le64(de
->new_chunk
);
360 * Registers the exceptions that are present in the current area.
361 * 'full' is filled in to indicate if the area has been
364 static int insert_exceptions(struct pstore
*ps
, int *full
)
368 struct disk_exception de
;
370 /* presume the area is full */
373 for (i
= 0; i
< ps
->exceptions_per_area
; i
++) {
374 read_exception(ps
, i
, &de
);
377 * If the new_chunk is pointing at the start of
378 * the COW device, where the first metadata area
379 * is we know that we've hit the end of the
380 * exceptions. Therefore the area is not full.
382 if (de
.new_chunk
== 0LL) {
383 ps
->current_committed
= i
;
389 * Keep track of the start of the free chunks.
391 if (ps
->next_free
<= de
.new_chunk
)
392 ps
->next_free
= de
.new_chunk
+ 1;
395 * Otherwise we add the exception to the snapshot.
397 r
= dm_add_exception(ps
->snap
, de
.old_chunk
, de
.new_chunk
);
405 static int read_exceptions(struct pstore
*ps
)
411 * Keeping reading chunks and inserting exceptions until
412 * we find a partially full area.
414 for (area
= 0; full
; area
++) {
415 r
= area_io(ps
, area
, READ
);
419 r
= insert_exceptions(ps
, &full
);
427 static struct pstore
*get_info(struct exception_store
*store
)
429 return (struct pstore
*) store
->context
;
432 static void persistent_fraction_full(struct exception_store
*store
,
433 sector_t
*numerator
, sector_t
*denominator
)
435 *numerator
= get_info(store
)->next_free
* store
->snap
->chunk_size
;
436 *denominator
= get_dev_size(store
->snap
->cow
->bdev
);
439 static void persistent_destroy(struct exception_store
*store
)
441 struct pstore
*ps
= get_info(store
);
443 destroy_workqueue(ps
->metadata_wq
);
444 dm_io_client_destroy(ps
->io_client
);
445 vfree(ps
->callbacks
);
450 static int persistent_read_metadata(struct exception_store
*store
)
453 struct pstore
*ps
= get_info(store
);
456 * Read the snapshot header.
458 r
= read_header(ps
, &new_snapshot
);
463 * Now we know correct chunk_size, complete the initialisation.
465 ps
->exceptions_per_area
= (ps
->snap
->chunk_size
<< SECTOR_SHIFT
) /
466 sizeof(struct disk_exception
);
467 ps
->callbacks
= dm_vcalloc(ps
->exceptions_per_area
,
468 sizeof(*ps
->callbacks
));
473 * Do we need to setup a new snapshot ?
476 r
= write_header(ps
);
478 DMWARN("write_header failed");
482 r
= zero_area(ps
, 0);
484 DMWARN("zero_area(0) failed");
492 if (ps
->version
!= SNAPSHOT_DISK_VERSION
) {
493 DMWARN("unable to handle snapshot disk version %d",
499 * Metadata are valid, but snapshot is invalidated
507 r
= read_exceptions(ps
);
515 static int persistent_prepare(struct exception_store
*store
,
516 struct dm_snap_exception
*e
)
518 struct pstore
*ps
= get_info(store
);
520 sector_t size
= get_dev_size(store
->snap
->cow
->bdev
);
522 /* Is there enough room ? */
523 if (size
< ((ps
->next_free
+ 1) * store
->snap
->chunk_size
))
526 e
->new_chunk
= ps
->next_free
;
529 * Move onto the next free pending, making sure to take
530 * into account the location of the metadata chunks.
532 stride
= (ps
->exceptions_per_area
+ 1);
533 if ((++ps
->next_free
% stride
) == 1)
536 atomic_inc(&ps
->pending_count
);
540 static void persistent_commit(struct exception_store
*store
,
541 struct dm_snap_exception
*e
,
542 void (*callback
) (void *, int success
),
543 void *callback_context
)
547 struct pstore
*ps
= get_info(store
);
548 struct disk_exception de
;
549 struct commit_callback
*cb
;
551 de
.old_chunk
= e
->old_chunk
;
552 de
.new_chunk
= e
->new_chunk
;
553 write_exception(ps
, ps
->current_committed
++, &de
);
556 * Add the callback to the back of the array. This code
557 * is the only place where the callback array is
558 * manipulated, and we know that it will never be called
559 * multiple times concurrently.
561 cb
= ps
->callbacks
+ ps
->callback_count
++;
562 cb
->callback
= callback
;
563 cb
->context
= callback_context
;
566 * If there are no more exceptions in flight, or we have
567 * filled this metadata area we commit the exceptions to
570 if (atomic_dec_and_test(&ps
->pending_count
) ||
571 (ps
->current_committed
== ps
->exceptions_per_area
)) {
572 r
= area_io(ps
, ps
->current_area
, WRITE
);
577 * Have we completely filled the current area ?
579 if (ps
->current_committed
== ps
->exceptions_per_area
) {
580 ps
->current_committed
= 0;
581 r
= zero_area(ps
, ps
->current_area
+ 1);
586 for (i
= 0; i
< ps
->callback_count
; i
++) {
587 cb
= ps
->callbacks
+ i
;
588 cb
->callback(cb
->context
, r
== 0 ? 1 : 0);
591 ps
->callback_count
= 0;
595 static void persistent_drop(struct exception_store
*store
)
597 struct pstore
*ps
= get_info(store
);
600 if (write_header(ps
))
601 DMWARN("write header failed");
604 int dm_create_persistent(struct exception_store
*store
)
608 /* allocate the pstore */
609 ps
= kmalloc(sizeof(*ps
), GFP_KERNEL
);
613 ps
->snap
= store
->snap
;
615 ps
->version
= SNAPSHOT_DISK_VERSION
;
617 ps
->next_free
= 2; /* skipping the header and first area */
618 ps
->current_committed
= 0;
620 ps
->callback_count
= 0;
621 atomic_set(&ps
->pending_count
, 0);
622 ps
->callbacks
= NULL
;
624 ps
->metadata_wq
= create_singlethread_workqueue("ksnaphd");
625 if (!ps
->metadata_wq
) {
627 DMERR("couldn't start header metadata update thread");
631 store
->destroy
= persistent_destroy
;
632 store
->read_metadata
= persistent_read_metadata
;
633 store
->prepare_exception
= persistent_prepare
;
634 store
->commit_exception
= persistent_commit
;
635 store
->drop_snapshot
= persistent_drop
;
636 store
->fraction_full
= persistent_fraction_full
;
642 /*-----------------------------------------------------------------
643 * Implementation of the store for non-persistent snapshots.
644 *---------------------------------------------------------------*/
649 static void transient_destroy(struct exception_store
*store
)
651 kfree(store
->context
);
654 static int transient_read_metadata(struct exception_store
*store
)
659 static int transient_prepare(struct exception_store
*store
,
660 struct dm_snap_exception
*e
)
662 struct transient_c
*tc
= (struct transient_c
*) store
->context
;
663 sector_t size
= get_dev_size(store
->snap
->cow
->bdev
);
665 if (size
< (tc
->next_free
+ store
->snap
->chunk_size
))
668 e
->new_chunk
= sector_to_chunk(store
->snap
, tc
->next_free
);
669 tc
->next_free
+= store
->snap
->chunk_size
;
674 static void transient_commit(struct exception_store
*store
,
675 struct dm_snap_exception
*e
,
676 void (*callback
) (void *, int success
),
677 void *callback_context
)
680 callback(callback_context
, 1);
683 static void transient_fraction_full(struct exception_store
*store
,
684 sector_t
*numerator
, sector_t
*denominator
)
686 *numerator
= ((struct transient_c
*) store
->context
)->next_free
;
687 *denominator
= get_dev_size(store
->snap
->cow
->bdev
);
690 int dm_create_transient(struct exception_store
*store
)
692 struct transient_c
*tc
;
694 store
->destroy
= transient_destroy
;
695 store
->read_metadata
= transient_read_metadata
;
696 store
->prepare_exception
= transient_prepare
;
697 store
->commit_exception
= transient_commit
;
698 store
->drop_snapshot
= NULL
;
699 store
->fraction_full
= transient_fraction_full
;
701 tc
= kmalloc(sizeof(struct transient_c
), GFP_KERNEL
);