2 * linux/kernel/power/swap.c
4 * This file provides functions for reading the suspend image from
5 * and writing it to a swap partition.
7 * Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz>
8 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
10 * This file is released under the GPLv2.
14 #include <linux/module.h>
15 #include <linux/file.h>
16 #include <linux/delay.h>
17 #include <linux/bitops.h>
18 #include <linux/genhd.h>
19 #include <linux/device.h>
20 #include <linux/buffer_head.h>
21 #include <linux/bio.h>
22 #include <linux/blkdev.h>
23 #include <linux/swap.h>
24 #include <linux/swapops.h>
26 #include <linux/slab.h>
27 #include <linux/lzo.h>
28 #include <linux/vmalloc.h>
32 #define HIBERNATE_SIG "LINHIB0001"
35 * The swap map is a data structure used for keeping track of each page
36 * written to a swap partition. It consists of many swap_map_page
37 * structures that contain each an array of MAP_PAGE_ENTRIES swap entries.
38 * These structures are stored on the swap and linked together with the
39 * help of the .next_swap member.
41 * The swap map is created during suspend. The swap map pages are
42 * allocated and populated one at a time, so we only need one memory
43 * page to set up the entire structure.
45 * During resume we also only need to use one swap_map_page structure
49 #define MAP_PAGE_ENTRIES (PAGE_SIZE / sizeof(sector_t) - 1)
51 struct swap_map_page
{
52 sector_t entries
[MAP_PAGE_ENTRIES
];
57 * The swap_map_handle structure is used for handling swap in
61 struct swap_map_handle
{
62 struct swap_map_page
*cur
;
64 sector_t first_sector
;
68 struct swsusp_header
{
69 char reserved
[PAGE_SIZE
- 20 - sizeof(sector_t
) - sizeof(int)];
71 unsigned int flags
; /* Flags to pass to the "boot" kernel */
74 } __attribute__((packed
));
76 static struct swsusp_header
*swsusp_header
;
79 * The following functions are used for tracing the allocated
80 * swap pages, so that they can be freed in case of an error.
83 struct swsusp_extent
{
89 static struct rb_root swsusp_extents
= RB_ROOT
;
91 static int swsusp_extents_insert(unsigned long swap_offset
)
93 struct rb_node
**new = &(swsusp_extents
.rb_node
);
94 struct rb_node
*parent
= NULL
;
95 struct swsusp_extent
*ext
;
97 /* Figure out where to put the new node */
99 ext
= container_of(*new, struct swsusp_extent
, node
);
101 if (swap_offset
< ext
->start
) {
103 if (swap_offset
== ext
->start
- 1) {
107 new = &((*new)->rb_left
);
108 } else if (swap_offset
> ext
->end
) {
110 if (swap_offset
== ext
->end
+ 1) {
114 new = &((*new)->rb_right
);
116 /* It already is in the tree */
120 /* Add the new node and rebalance the tree. */
121 ext
= kzalloc(sizeof(struct swsusp_extent
), GFP_KERNEL
);
125 ext
->start
= swap_offset
;
126 ext
->end
= swap_offset
;
127 rb_link_node(&ext
->node
, parent
, new);
128 rb_insert_color(&ext
->node
, &swsusp_extents
);
133 * alloc_swapdev_block - allocate a swap page and register that it has
134 * been allocated, so that it can be freed in case of an error.
137 sector_t
alloc_swapdev_block(int swap
)
139 unsigned long offset
;
141 offset
= swp_offset(get_swap_page_of_type(swap
));
143 if (swsusp_extents_insert(offset
))
144 swap_free(swp_entry(swap
, offset
));
146 return swapdev_block(swap
, offset
);
152 * free_all_swap_pages - free swap pages allocated for saving image data.
153 * It also frees the extents used to register which swap entries had been
157 void free_all_swap_pages(int swap
)
159 struct rb_node
*node
;
161 while ((node
= swsusp_extents
.rb_node
)) {
162 struct swsusp_extent
*ext
;
163 unsigned long offset
;
165 ext
= container_of(node
, struct swsusp_extent
, node
);
166 rb_erase(node
, &swsusp_extents
);
167 for (offset
= ext
->start
; offset
<= ext
->end
; offset
++)
168 swap_free(swp_entry(swap
, offset
));
174 int swsusp_swap_in_use(void)
176 return (swsusp_extents
.rb_node
!= NULL
);
183 static unsigned short root_swap
= 0xffff;
184 struct block_device
*hib_resume_bdev
;
190 static int mark_swapfiles(struct swap_map_handle
*handle
, unsigned int flags
)
194 hib_bio_read_page(swsusp_resume_block
, swsusp_header
, NULL
);
195 if (!memcmp("SWAP-SPACE",swsusp_header
->sig
, 10) ||
196 !memcmp("SWAPSPACE2",swsusp_header
->sig
, 10)) {
197 memcpy(swsusp_header
->orig_sig
,swsusp_header
->sig
, 10);
198 memcpy(swsusp_header
->sig
, HIBERNATE_SIG
, 10);
199 swsusp_header
->image
= handle
->first_sector
;
200 swsusp_header
->flags
= flags
;
201 error
= hib_bio_write_page(swsusp_resume_block
,
202 swsusp_header
, NULL
);
204 printk(KERN_ERR
"PM: Swap header not found!\n");
211 * swsusp_swap_check - check if the resume device is a swap device
212 * and get its index (if so)
214 * This is called before saving image
216 static int swsusp_swap_check(void)
220 res
= swap_type_of(swsusp_resume_device
, swsusp_resume_block
,
226 res
= blkdev_get(hib_resume_bdev
, FMODE_WRITE
, NULL
);
230 res
= set_blocksize(hib_resume_bdev
, PAGE_SIZE
);
232 blkdev_put(hib_resume_bdev
, FMODE_WRITE
);
238 * write_page - Write one page to given swap location.
239 * @buf: Address we're writing.
240 * @offset: Offset of the swap page we're writing to.
241 * @bio_chain: Link the next write BIO here
244 static int write_page(void *buf
, sector_t offset
, struct bio
**bio_chain
)
252 src
= (void *)__get_free_page(__GFP_WAIT
| __GFP_HIGH
);
257 bio_chain
= NULL
; /* Go synchronous */
263 return hib_bio_write_page(offset
, src
, bio_chain
);
266 static void release_swap_writer(struct swap_map_handle
*handle
)
269 free_page((unsigned long)handle
->cur
);
273 static int get_swap_writer(struct swap_map_handle
*handle
)
277 ret
= swsusp_swap_check();
280 printk(KERN_ERR
"PM: Cannot find swap device, try "
284 handle
->cur
= (struct swap_map_page
*)get_zeroed_page(GFP_KERNEL
);
289 handle
->cur_swap
= alloc_swapdev_block(root_swap
);
290 if (!handle
->cur_swap
) {
295 handle
->first_sector
= handle
->cur_swap
;
298 release_swap_writer(handle
);
300 swsusp_close(FMODE_WRITE
);
304 static int swap_write_page(struct swap_map_handle
*handle
, void *buf
,
305 struct bio
**bio_chain
)
312 offset
= alloc_swapdev_block(root_swap
);
313 error
= write_page(buf
, offset
, bio_chain
);
316 handle
->cur
->entries
[handle
->k
++] = offset
;
317 if (handle
->k
>= MAP_PAGE_ENTRIES
) {
318 error
= hib_wait_on_bio_chain(bio_chain
);
321 offset
= alloc_swapdev_block(root_swap
);
324 handle
->cur
->next_swap
= offset
;
325 error
= write_page(handle
->cur
, handle
->cur_swap
, NULL
);
328 clear_page(handle
->cur
);
329 handle
->cur_swap
= offset
;
336 static int flush_swap_writer(struct swap_map_handle
*handle
)
338 if (handle
->cur
&& handle
->cur_swap
)
339 return write_page(handle
->cur
, handle
->cur_swap
, NULL
);
344 static int swap_writer_finish(struct swap_map_handle
*handle
,
345 unsigned int flags
, int error
)
348 flush_swap_writer(handle
);
349 printk(KERN_INFO
"PM: S");
350 error
= mark_swapfiles(handle
, flags
);
355 free_all_swap_pages(root_swap
);
356 release_swap_writer(handle
);
357 swsusp_close(FMODE_WRITE
);
362 /* We need to remember how much compressed data we need to read. */
363 #define LZO_HEADER sizeof(size_t)
365 /* Number of pages/bytes we'll compress at one time. */
366 #define LZO_UNC_PAGES 32
367 #define LZO_UNC_SIZE (LZO_UNC_PAGES * PAGE_SIZE)
369 /* Number of pages/bytes we need for compressed data (worst case). */
370 #define LZO_CMP_PAGES DIV_ROUND_UP(lzo1x_worst_compress(LZO_UNC_SIZE) + \
371 LZO_HEADER, PAGE_SIZE)
372 #define LZO_CMP_SIZE (LZO_CMP_PAGES * PAGE_SIZE)
375 * save_image - save the suspend image data
378 static int save_image(struct swap_map_handle
*handle
,
379 struct snapshot_handle
*snapshot
,
380 unsigned int nr_to_write
)
387 struct timeval start
;
390 printk(KERN_INFO
"PM: Saving image data pages (%u pages) ... ",
392 m
= nr_to_write
/ 100;
397 do_gettimeofday(&start
);
399 ret
= snapshot_read_next(snapshot
);
402 ret
= swap_write_page(handle
, data_of(*snapshot
), &bio
);
406 printk(KERN_CONT
"\b\b\b\b%3d%%", nr_pages
/ m
);
409 err2
= hib_wait_on_bio_chain(&bio
);
410 do_gettimeofday(&stop
);
414 printk(KERN_CONT
"\b\b\b\bdone\n");
416 printk(KERN_CONT
"\n");
417 swsusp_show_speed(&start
, &stop
, nr_to_write
, "Wrote");
423 * save_image_lzo - Save the suspend image data compressed with LZO.
424 * @handle: Swap mam handle to use for saving the image.
425 * @snapshot: Image to read data from.
426 * @nr_to_write: Number of pages to save.
428 static int save_image_lzo(struct swap_map_handle
*handle
,
429 struct snapshot_handle
*snapshot
,
430 unsigned int nr_to_write
)
437 struct timeval start
;
439 size_t off
, unc_len
, cmp_len
;
440 unsigned char *unc
, *cmp
, *wrk
, *page
;
442 page
= (void *)__get_free_page(__GFP_WAIT
| __GFP_HIGH
);
444 printk(KERN_ERR
"PM: Failed to allocate LZO page\n");
448 wrk
= vmalloc(LZO1X_1_MEM_COMPRESS
);
450 printk(KERN_ERR
"PM: Failed to allocate LZO workspace\n");
451 free_page((unsigned long)page
);
455 unc
= vmalloc(LZO_UNC_SIZE
);
457 printk(KERN_ERR
"PM: Failed to allocate LZO uncompressed\n");
459 free_page((unsigned long)page
);
463 cmp
= vmalloc(LZO_CMP_SIZE
);
465 printk(KERN_ERR
"PM: Failed to allocate LZO compressed\n");
468 free_page((unsigned long)page
);
473 "PM: Compressing and saving image data (%u pages) ... ",
475 m
= nr_to_write
/ 100;
480 do_gettimeofday(&start
);
482 for (off
= 0; off
< LZO_UNC_SIZE
; off
+= PAGE_SIZE
) {
483 ret
= snapshot_read_next(snapshot
);
490 memcpy(unc
+ off
, data_of(*snapshot
), PAGE_SIZE
);
493 printk(KERN_CONT
"\b\b\b\b%3d%%", nr_pages
/ m
);
501 ret
= lzo1x_1_compress(unc
, unc_len
,
502 cmp
+ LZO_HEADER
, &cmp_len
, wrk
);
504 printk(KERN_ERR
"PM: LZO compression failed\n");
508 if (unlikely(!cmp_len
||
509 cmp_len
> lzo1x_worst_compress(unc_len
))) {
510 printk(KERN_ERR
"PM: Invalid LZO compressed length\n");
515 *(size_t *)cmp
= cmp_len
;
518 * Given we are writing one page at a time to disk, we copy
519 * that much from the buffer, although the last bit will likely
520 * be smaller than full page. This is OK - we saved the length
521 * of the compressed data, so any garbage at the end will be
522 * discarded when we read it.
524 for (off
= 0; off
< LZO_HEADER
+ cmp_len
; off
+= PAGE_SIZE
) {
525 memcpy(page
, cmp
+ off
, PAGE_SIZE
);
527 ret
= swap_write_page(handle
, page
, &bio
);
534 err2
= hib_wait_on_bio_chain(&bio
);
535 do_gettimeofday(&stop
);
539 printk(KERN_CONT
"\b\b\b\bdone\n");
541 printk(KERN_CONT
"\n");
542 swsusp_show_speed(&start
, &stop
, nr_to_write
, "Wrote");
547 free_page((unsigned long)page
);
553 * enough_swap - Make sure we have enough swap to save the image.
555 * Returns TRUE or FALSE after checking the total amount of swap
556 * space avaiable from the resume partition.
559 static int enough_swap(unsigned int nr_pages
, unsigned int flags
)
561 unsigned int free_swap
= count_swap_pages(root_swap
, 1);
562 unsigned int required
;
564 pr_debug("PM: Free swap pages: %u\n", free_swap
);
566 required
= PAGES_FOR_IO
+ ((flags
& SF_NOCOMPRESS_MODE
) ?
567 nr_pages
: (nr_pages
* LZO_CMP_PAGES
) / LZO_UNC_PAGES
+ 1);
568 return free_swap
> required
;
572 * swsusp_write - Write entire image and metadata.
573 * @flags: flags to pass to the "boot" kernel in the image header
575 * It is important _NOT_ to umount filesystems at this point. We want
576 * them synced (in case something goes wrong) but we DO not want to mark
577 * filesystem clean: it is not. (And it does not matter, if we resume
578 * correctly, we'll mark system clean, anyway.)
581 int swsusp_write(unsigned int flags
)
583 struct swap_map_handle handle
;
584 struct snapshot_handle snapshot
;
585 struct swsusp_info
*header
;
589 pages
= snapshot_get_image_size();
590 error
= get_swap_writer(&handle
);
592 printk(KERN_ERR
"PM: Cannot get swap writer\n");
595 if (!enough_swap(pages
, flags
)) {
596 printk(KERN_ERR
"PM: Not enough free swap\n");
600 memset(&snapshot
, 0, sizeof(struct snapshot_handle
));
601 error
= snapshot_read_next(&snapshot
);
602 if (error
< PAGE_SIZE
) {
608 header
= (struct swsusp_info
*)data_of(snapshot
);
609 error
= swap_write_page(&handle
, header
, NULL
);
611 error
= (flags
& SF_NOCOMPRESS_MODE
) ?
612 save_image(&handle
, &snapshot
, pages
- 1) :
613 save_image_lzo(&handle
, &snapshot
, pages
- 1);
616 error
= swap_writer_finish(&handle
, flags
, error
);
621 * The following functions allow us to read data using a swap map
622 * in a file-alike way
625 static void release_swap_reader(struct swap_map_handle
*handle
)
628 free_page((unsigned long)handle
->cur
);
632 static int get_swap_reader(struct swap_map_handle
*handle
,
633 unsigned int *flags_p
)
637 *flags_p
= swsusp_header
->flags
;
639 if (!swsusp_header
->image
) /* how can this happen? */
642 handle
->cur
= (struct swap_map_page
*)get_zeroed_page(__GFP_WAIT
| __GFP_HIGH
);
646 error
= hib_bio_read_page(swsusp_header
->image
, handle
->cur
, NULL
);
648 release_swap_reader(handle
);
655 static int swap_read_page(struct swap_map_handle
*handle
, void *buf
,
656 struct bio
**bio_chain
)
663 offset
= handle
->cur
->entries
[handle
->k
];
666 error
= hib_bio_read_page(offset
, buf
, bio_chain
);
669 if (++handle
->k
>= MAP_PAGE_ENTRIES
) {
670 error
= hib_wait_on_bio_chain(bio_chain
);
672 offset
= handle
->cur
->next_swap
;
674 release_swap_reader(handle
);
676 error
= hib_bio_read_page(offset
, handle
->cur
, NULL
);
681 static int swap_reader_finish(struct swap_map_handle
*handle
)
683 release_swap_reader(handle
);
689 * load_image - load the image using the swap map handle
690 * @handle and the snapshot handle @snapshot
691 * (assume there are @nr_pages pages to load)
694 static int load_image(struct swap_map_handle
*handle
,
695 struct snapshot_handle
*snapshot
,
696 unsigned int nr_to_read
)
700 struct timeval start
;
706 printk(KERN_INFO
"PM: Loading image data pages (%u pages) ... ",
708 m
= nr_to_read
/ 100;
713 do_gettimeofday(&start
);
715 error
= snapshot_write_next(snapshot
);
718 error
= swap_read_page(handle
, data_of(*snapshot
), &bio
);
721 if (snapshot
->sync_read
)
722 error
= hib_wait_on_bio_chain(&bio
);
726 printk("\b\b\b\b%3d%%", nr_pages
/ m
);
729 err2
= hib_wait_on_bio_chain(&bio
);
730 do_gettimeofday(&stop
);
734 printk("\b\b\b\bdone\n");
735 snapshot_write_finalize(snapshot
);
736 if (!snapshot_image_loaded(snapshot
))
740 swsusp_show_speed(&start
, &stop
, nr_to_read
, "Read");
745 * load_image_lzo - Load compressed image data and decompress them with LZO.
746 * @handle: Swap map handle to use for loading data.
747 * @snapshot: Image to copy uncompressed data into.
748 * @nr_to_read: Number of pages to load.
750 static int load_image_lzo(struct swap_map_handle
*handle
,
751 struct snapshot_handle
*snapshot
,
752 unsigned int nr_to_read
)
756 struct timeval start
;
759 size_t off
, unc_len
, cmp_len
;
760 unsigned char *unc
, *cmp
, *page
;
762 page
= (void *)__get_free_page(__GFP_WAIT
| __GFP_HIGH
);
764 printk(KERN_ERR
"PM: Failed to allocate LZO page\n");
768 unc
= vmalloc(LZO_UNC_SIZE
);
770 printk(KERN_ERR
"PM: Failed to allocate LZO uncompressed\n");
771 free_page((unsigned long)page
);
775 cmp
= vmalloc(LZO_CMP_SIZE
);
777 printk(KERN_ERR
"PM: Failed to allocate LZO compressed\n");
779 free_page((unsigned long)page
);
784 "PM: Loading and decompressing image data (%u pages) ... ",
786 m
= nr_to_read
/ 100;
790 do_gettimeofday(&start
);
792 error
= snapshot_write_next(snapshot
);
797 error
= swap_read_page(handle
, page
, NULL
); /* sync */
801 cmp_len
= *(size_t *)page
;
802 if (unlikely(!cmp_len
||
803 cmp_len
> lzo1x_worst_compress(LZO_UNC_SIZE
))) {
804 printk(KERN_ERR
"PM: Invalid LZO compressed length\n");
809 memcpy(cmp
, page
, PAGE_SIZE
);
810 for (off
= PAGE_SIZE
; off
< LZO_HEADER
+ cmp_len
; off
+= PAGE_SIZE
) {
811 error
= swap_read_page(handle
, page
, NULL
); /* sync */
815 memcpy(cmp
+ off
, page
, PAGE_SIZE
);
818 unc_len
= LZO_UNC_SIZE
;
819 error
= lzo1x_decompress_safe(cmp
+ LZO_HEADER
, cmp_len
,
822 printk(KERN_ERR
"PM: LZO decompression failed\n");
826 if (unlikely(!unc_len
||
827 unc_len
> LZO_UNC_SIZE
||
828 unc_len
& (PAGE_SIZE
- 1))) {
829 printk(KERN_ERR
"PM: Invalid LZO uncompressed length\n");
834 for (off
= 0; off
< unc_len
; off
+= PAGE_SIZE
) {
835 memcpy(data_of(*snapshot
), unc
+ off
, PAGE_SIZE
);
838 printk("\b\b\b\b%3d%%", nr_pages
/ m
);
841 error
= snapshot_write_next(snapshot
);
848 do_gettimeofday(&stop
);
850 printk("\b\b\b\bdone\n");
851 snapshot_write_finalize(snapshot
);
852 if (!snapshot_image_loaded(snapshot
))
856 swsusp_show_speed(&start
, &stop
, nr_to_read
, "Read");
860 free_page((unsigned long)page
);
866 * swsusp_read - read the hibernation image.
867 * @flags_p: flags passed by the "frozen" kernel in the image header should
868 * be written into this memeory location
871 int swsusp_read(unsigned int *flags_p
)
874 struct swap_map_handle handle
;
875 struct snapshot_handle snapshot
;
876 struct swsusp_info
*header
;
878 memset(&snapshot
, 0, sizeof(struct snapshot_handle
));
879 error
= snapshot_write_next(&snapshot
);
880 if (error
< PAGE_SIZE
)
881 return error
< 0 ? error
: -EFAULT
;
882 header
= (struct swsusp_info
*)data_of(snapshot
);
883 error
= get_swap_reader(&handle
, flags_p
);
887 error
= swap_read_page(&handle
, header
, NULL
);
889 error
= (*flags_p
& SF_NOCOMPRESS_MODE
) ?
890 load_image(&handle
, &snapshot
, header
->pages
- 1) :
891 load_image_lzo(&handle
, &snapshot
, header
->pages
- 1);
893 swap_reader_finish(&handle
);
896 pr_debug("PM: Image successfully loaded\n");
898 pr_debug("PM: Error %d resuming\n", error
);
903 * swsusp_check - Check for swsusp signature in the resume device
906 int swsusp_check(void)
910 hib_resume_bdev
= blkdev_get_by_dev(swsusp_resume_device
,
912 if (!IS_ERR(hib_resume_bdev
)) {
913 set_blocksize(hib_resume_bdev
, PAGE_SIZE
);
914 clear_page(swsusp_header
);
915 error
= hib_bio_read_page(swsusp_resume_block
,
916 swsusp_header
, NULL
);
920 if (!memcmp(HIBERNATE_SIG
, swsusp_header
->sig
, 10)) {
921 memcpy(swsusp_header
->sig
, swsusp_header
->orig_sig
, 10);
922 /* Reset swap signature now */
923 error
= hib_bio_write_page(swsusp_resume_block
,
924 swsusp_header
, NULL
);
931 blkdev_put(hib_resume_bdev
, FMODE_READ
);
933 pr_debug("PM: Image signature found, resuming\n");
935 error
= PTR_ERR(hib_resume_bdev
);
939 pr_debug("PM: Image not found (code %d)\n", error
);
945 * swsusp_close - close swap device.
948 void swsusp_close(fmode_t mode
)
950 if (IS_ERR(hib_resume_bdev
)) {
951 pr_debug("PM: Image device not initialised\n");
955 blkdev_put(hib_resume_bdev
, mode
);
958 static int swsusp_header_init(void)
960 swsusp_header
= (struct swsusp_header
*) __get_free_page(GFP_KERNEL
);
962 panic("Could not allocate memory for swsusp_header\n");
966 core_initcall(swsusp_header_init
);