2 * linux/kernel/power/swap.c
4 * This file provides functions for reading the suspend image from
5 * and writing it to a swap partition.
7 * Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz>
8 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
9 * Copyright (C) 2010-2012 Bojan Smojver <bojan@rexursive.com>
11 * This file is released under the GPLv2.
15 #include <linux/module.h>
16 #include <linux/file.h>
17 #include <linux/delay.h>
18 #include <linux/bitops.h>
19 #include <linux/genhd.h>
20 #include <linux/device.h>
21 #include <linux/bio.h>
22 #include <linux/blkdev.h>
23 #include <linux/swap.h>
24 #include <linux/swapops.h>
26 #include <linux/slab.h>
27 #include <linux/lzo.h>
28 #include <linux/vmalloc.h>
29 #include <linux/cpumask.h>
30 #include <linux/atomic.h>
31 #include <linux/kthread.h>
32 #include <linux/crc32.h>
33 #include <linux/ktime.h>
37 #define HIBERNATE_SIG "S1SUSPEND"
40 * The swap map is a data structure used for keeping track of each page
41 * written to a swap partition. It consists of many swap_map_page
42 * structures that contain each an array of MAP_PAGE_ENTRIES swap entries.
43 * These structures are stored on the swap and linked together with the
44 * help of the .next_swap member.
46 * The swap map is created during suspend. The swap map pages are
47 * allocated and populated one at a time, so we only need one memory
48 * page to set up the entire structure.
50 * During resume we pick up all swap_map_page structures into a list.
53 #define MAP_PAGE_ENTRIES (PAGE_SIZE / sizeof(sector_t) - 1)
56 * Number of free pages that are not high.
58 static inline unsigned long low_free_pages(void)
60 return nr_free_pages() - nr_free_highpages();
64 * Number of pages required to be kept free while writing the image. Always
65 * half of all available low pages before the writing starts.
67 static inline unsigned long reqd_free_pages(void)
69 return low_free_pages() / 2;
72 struct swap_map_page
{
73 sector_t entries
[MAP_PAGE_ENTRIES
];
77 struct swap_map_page_list
{
78 struct swap_map_page
*map
;
79 struct swap_map_page_list
*next
;
83 * The swap_map_handle structure is used for handling swap in
87 struct swap_map_handle
{
88 struct swap_map_page
*cur
;
89 struct swap_map_page_list
*maps
;
91 sector_t first_sector
;
93 unsigned long reqd_free_pages
;
97 struct swsusp_header
{
98 char reserved
[PAGE_SIZE
- 20 - sizeof(sector_t
) - sizeof(int) -
102 unsigned int flags
; /* Flags to pass to the "boot" kernel */
107 static struct swsusp_header
*swsusp_header
;
110 * The following functions are used for tracing the allocated
111 * swap pages, so that they can be freed in case of an error.
114 struct swsusp_extent
{
120 static struct rb_root swsusp_extents
= RB_ROOT
;
122 static int swsusp_extents_insert(unsigned long swap_offset
)
124 struct rb_node
**new = &(swsusp_extents
.rb_node
);
125 struct rb_node
*parent
= NULL
;
126 struct swsusp_extent
*ext
;
128 /* Figure out where to put the new node */
130 ext
= rb_entry(*new, struct swsusp_extent
, node
);
132 if (swap_offset
< ext
->start
) {
134 if (swap_offset
== ext
->start
- 1) {
138 new = &((*new)->rb_left
);
139 } else if (swap_offset
> ext
->end
) {
141 if (swap_offset
== ext
->end
+ 1) {
145 new = &((*new)->rb_right
);
147 /* It already is in the tree */
151 /* Add the new node and rebalance the tree. */
152 ext
= kzalloc(sizeof(struct swsusp_extent
), GFP_KERNEL
);
156 ext
->start
= swap_offset
;
157 ext
->end
= swap_offset
;
158 rb_link_node(&ext
->node
, parent
, new);
159 rb_insert_color(&ext
->node
, &swsusp_extents
);
164 * alloc_swapdev_block - allocate a swap page and register that it has
165 * been allocated, so that it can be freed in case of an error.
168 sector_t
alloc_swapdev_block(int swap
)
170 unsigned long offset
;
172 offset
= swp_offset(get_swap_page_of_type(swap
));
174 if (swsusp_extents_insert(offset
))
175 swap_free(swp_entry(swap
, offset
));
177 return swapdev_block(swap
, offset
);
183 * free_all_swap_pages - free swap pages allocated for saving image data.
184 * It also frees the extents used to register which swap entries had been
188 void free_all_swap_pages(int swap
)
190 struct rb_node
*node
;
192 while ((node
= swsusp_extents
.rb_node
)) {
193 struct swsusp_extent
*ext
;
194 unsigned long offset
;
196 ext
= container_of(node
, struct swsusp_extent
, node
);
197 rb_erase(node
, &swsusp_extents
);
198 for (offset
= ext
->start
; offset
<= ext
->end
; offset
++)
199 swap_free(swp_entry(swap
, offset
));
205 int swsusp_swap_in_use(void)
207 return (swsusp_extents
.rb_node
!= NULL
);
214 static unsigned short root_swap
= 0xffff;
215 struct block_device
*hib_resume_bdev
;
221 static int mark_swapfiles(struct swap_map_handle
*handle
, unsigned int flags
)
225 hib_bio_read_page(swsusp_resume_block
, swsusp_header
, NULL
);
226 if (!memcmp("SWAP-SPACE",swsusp_header
->sig
, 10) ||
227 !memcmp("SWAPSPACE2",swsusp_header
->sig
, 10)) {
228 memcpy(swsusp_header
->orig_sig
,swsusp_header
->sig
, 10);
229 memcpy(swsusp_header
->sig
, HIBERNATE_SIG
, 10);
230 swsusp_header
->image
= handle
->first_sector
;
231 swsusp_header
->flags
= flags
;
232 if (flags
& SF_CRC32_MODE
)
233 swsusp_header
->crc32
= handle
->crc32
;
234 error
= hib_bio_write_page(swsusp_resume_block
,
235 swsusp_header
, NULL
);
237 printk(KERN_ERR
"PM: Swap header not found!\n");
244 * swsusp_swap_check - check if the resume device is a swap device
245 * and get its index (if so)
247 * This is called before saving image
249 static int swsusp_swap_check(void)
253 res
= swap_type_of(swsusp_resume_device
, swsusp_resume_block
,
259 res
= blkdev_get(hib_resume_bdev
, FMODE_WRITE
, NULL
);
263 res
= set_blocksize(hib_resume_bdev
, PAGE_SIZE
);
265 blkdev_put(hib_resume_bdev
, FMODE_WRITE
);
271 * write_page - Write one page to given swap location.
272 * @buf: Address we're writing.
273 * @offset: Offset of the swap page we're writing to.
274 * @bio_chain: Link the next write BIO here
277 static int write_page(void *buf
, sector_t offset
, struct bio
**bio_chain
)
286 src
= (void *)__get_free_page(__GFP_WAIT
| __GFP_NOWARN
|
291 ret
= hib_wait_on_bio_chain(bio_chain
); /* Free pages */
294 src
= (void *)__get_free_page(__GFP_WAIT
|
301 bio_chain
= NULL
; /* Go synchronous */
308 return hib_bio_write_page(offset
, src
, bio_chain
);
311 static void release_swap_writer(struct swap_map_handle
*handle
)
314 free_page((unsigned long)handle
->cur
);
318 static int get_swap_writer(struct swap_map_handle
*handle
)
322 ret
= swsusp_swap_check();
325 printk(KERN_ERR
"PM: Cannot find swap device, try "
329 handle
->cur
= (struct swap_map_page
*)get_zeroed_page(GFP_KERNEL
);
334 handle
->cur_swap
= alloc_swapdev_block(root_swap
);
335 if (!handle
->cur_swap
) {
340 handle
->reqd_free_pages
= reqd_free_pages();
341 handle
->first_sector
= handle
->cur_swap
;
344 release_swap_writer(handle
);
346 swsusp_close(FMODE_WRITE
);
350 static int swap_write_page(struct swap_map_handle
*handle
, void *buf
,
351 struct bio
**bio_chain
)
358 offset
= alloc_swapdev_block(root_swap
);
359 error
= write_page(buf
, offset
, bio_chain
);
362 handle
->cur
->entries
[handle
->k
++] = offset
;
363 if (handle
->k
>= MAP_PAGE_ENTRIES
) {
364 offset
= alloc_swapdev_block(root_swap
);
367 handle
->cur
->next_swap
= offset
;
368 error
= write_page(handle
->cur
, handle
->cur_swap
, bio_chain
);
371 clear_page(handle
->cur
);
372 handle
->cur_swap
= offset
;
375 if (bio_chain
&& low_free_pages() <= handle
->reqd_free_pages
) {
376 error
= hib_wait_on_bio_chain(bio_chain
);
380 * Recalculate the number of required free pages, to
381 * make sure we never take more than half.
383 handle
->reqd_free_pages
= reqd_free_pages();
390 static int flush_swap_writer(struct swap_map_handle
*handle
)
392 if (handle
->cur
&& handle
->cur_swap
)
393 return write_page(handle
->cur
, handle
->cur_swap
, NULL
);
398 static int swap_writer_finish(struct swap_map_handle
*handle
,
399 unsigned int flags
, int error
)
402 flush_swap_writer(handle
);
403 printk(KERN_INFO
"PM: S");
404 error
= mark_swapfiles(handle
, flags
);
409 free_all_swap_pages(root_swap
);
410 release_swap_writer(handle
);
411 swsusp_close(FMODE_WRITE
);
416 /* We need to remember how much compressed data we need to read. */
417 #define LZO_HEADER sizeof(size_t)
419 /* Number of pages/bytes we'll compress at one time. */
420 #define LZO_UNC_PAGES 32
421 #define LZO_UNC_SIZE (LZO_UNC_PAGES * PAGE_SIZE)
423 /* Number of pages/bytes we need for compressed data (worst case). */
424 #define LZO_CMP_PAGES DIV_ROUND_UP(lzo1x_worst_compress(LZO_UNC_SIZE) + \
425 LZO_HEADER, PAGE_SIZE)
426 #define LZO_CMP_SIZE (LZO_CMP_PAGES * PAGE_SIZE)
428 /* Maximum number of threads for compression/decompression. */
429 #define LZO_THREADS 3
431 /* Minimum/maximum number of pages for read buffering. */
432 #define LZO_MIN_RD_PAGES 1024
433 #define LZO_MAX_RD_PAGES 8192
437 * save_image - save the suspend image data
440 static int save_image(struct swap_map_handle
*handle
,
441 struct snapshot_handle
*snapshot
,
442 unsigned int nr_to_write
)
452 printk(KERN_INFO
"PM: Saving image data pages (%u pages)...\n",
454 m
= nr_to_write
/ 10;
461 ret
= snapshot_read_next(snapshot
);
464 ret
= swap_write_page(handle
, data_of(*snapshot
), &bio
);
468 printk(KERN_INFO
"PM: Image saving progress: %3d%%\n",
472 err2
= hib_wait_on_bio_chain(&bio
);
477 printk(KERN_INFO
"PM: Image saving done.\n");
478 swsusp_show_speed(start
, stop
, nr_to_write
, "Wrote");
483 * Structure used for CRC32.
486 struct task_struct
*thr
; /* thread */
487 atomic_t ready
; /* ready to start flag */
488 atomic_t stop
; /* ready to stop flag */
489 unsigned run_threads
; /* nr current threads */
490 wait_queue_head_t go
; /* start crc update */
491 wait_queue_head_t done
; /* crc update done */
492 u32
*crc32
; /* points to handle's crc32 */
493 size_t *unc_len
[LZO_THREADS
]; /* uncompressed lengths */
494 unsigned char *unc
[LZO_THREADS
]; /* uncompressed data */
498 * CRC32 update function that runs in its own thread.
500 static int crc32_threadfn(void *data
)
502 struct crc_data
*d
= data
;
506 wait_event(d
->go
, atomic_read(&d
->ready
) ||
507 kthread_should_stop());
508 if (kthread_should_stop()) {
510 atomic_set(&d
->stop
, 1);
514 atomic_set(&d
->ready
, 0);
516 for (i
= 0; i
< d
->run_threads
; i
++)
517 *d
->crc32
= crc32_le(*d
->crc32
,
518 d
->unc
[i
], *d
->unc_len
[i
]);
519 atomic_set(&d
->stop
, 1);
525 * Structure used for LZO data compression.
528 struct task_struct
*thr
; /* thread */
529 atomic_t ready
; /* ready to start flag */
530 atomic_t stop
; /* ready to stop flag */
531 int ret
; /* return code */
532 wait_queue_head_t go
; /* start compression */
533 wait_queue_head_t done
; /* compression done */
534 size_t unc_len
; /* uncompressed length */
535 size_t cmp_len
; /* compressed length */
536 unsigned char unc
[LZO_UNC_SIZE
]; /* uncompressed buffer */
537 unsigned char cmp
[LZO_CMP_SIZE
]; /* compressed buffer */
538 unsigned char wrk
[LZO1X_1_MEM_COMPRESS
]; /* compression workspace */
542 * Compression function that runs in its own thread.
544 static int lzo_compress_threadfn(void *data
)
546 struct cmp_data
*d
= data
;
549 wait_event(d
->go
, atomic_read(&d
->ready
) ||
550 kthread_should_stop());
551 if (kthread_should_stop()) {
554 atomic_set(&d
->stop
, 1);
558 atomic_set(&d
->ready
, 0);
560 d
->ret
= lzo1x_1_compress(d
->unc
, d
->unc_len
,
561 d
->cmp
+ LZO_HEADER
, &d
->cmp_len
,
563 atomic_set(&d
->stop
, 1);
570 * save_image_lzo - Save the suspend image data compressed with LZO.
571 * @handle: Swap map handle to use for saving the image.
572 * @snapshot: Image to read data from.
573 * @nr_to_write: Number of pages to save.
575 static int save_image_lzo(struct swap_map_handle
*handle
,
576 struct snapshot_handle
*snapshot
,
577 unsigned int nr_to_write
)
587 unsigned thr
, run_threads
, nr_threads
;
588 unsigned char *page
= NULL
;
589 struct cmp_data
*data
= NULL
;
590 struct crc_data
*crc
= NULL
;
593 * We'll limit the number of threads for compression to limit memory
596 nr_threads
= num_online_cpus() - 1;
597 nr_threads
= clamp_val(nr_threads
, 1, LZO_THREADS
);
599 page
= (void *)__get_free_page(__GFP_WAIT
| __GFP_HIGH
);
601 printk(KERN_ERR
"PM: Failed to allocate LZO page\n");
606 data
= vmalloc(sizeof(*data
) * nr_threads
);
608 printk(KERN_ERR
"PM: Failed to allocate LZO data\n");
612 for (thr
= 0; thr
< nr_threads
; thr
++)
613 memset(&data
[thr
], 0, offsetof(struct cmp_data
, go
));
615 crc
= kmalloc(sizeof(*crc
), GFP_KERNEL
);
617 printk(KERN_ERR
"PM: Failed to allocate crc\n");
621 memset(crc
, 0, offsetof(struct crc_data
, go
));
624 * Start the compression threads.
626 for (thr
= 0; thr
< nr_threads
; thr
++) {
627 init_waitqueue_head(&data
[thr
].go
);
628 init_waitqueue_head(&data
[thr
].done
);
630 data
[thr
].thr
= kthread_run(lzo_compress_threadfn
,
632 "image_compress/%u", thr
);
633 if (IS_ERR(data
[thr
].thr
)) {
634 data
[thr
].thr
= NULL
;
636 "PM: Cannot start compression threads\n");
643 * Start the CRC32 thread.
645 init_waitqueue_head(&crc
->go
);
646 init_waitqueue_head(&crc
->done
);
649 crc
->crc32
= &handle
->crc32
;
650 for (thr
= 0; thr
< nr_threads
; thr
++) {
651 crc
->unc
[thr
] = data
[thr
].unc
;
652 crc
->unc_len
[thr
] = &data
[thr
].unc_len
;
655 crc
->thr
= kthread_run(crc32_threadfn
, crc
, "image_crc32");
656 if (IS_ERR(crc
->thr
)) {
658 printk(KERN_ERR
"PM: Cannot start CRC32 thread\n");
664 * Adjust the number of required free pages after all allocations have
665 * been done. We don't want to run out of pages when writing.
667 handle
->reqd_free_pages
= reqd_free_pages();
670 "PM: Using %u thread(s) for compression.\n"
671 "PM: Compressing and saving image data (%u pages)...\n",
672 nr_threads
, nr_to_write
);
673 m
= nr_to_write
/ 10;
680 for (thr
= 0; thr
< nr_threads
; thr
++) {
681 for (off
= 0; off
< LZO_UNC_SIZE
; off
+= PAGE_SIZE
) {
682 ret
= snapshot_read_next(snapshot
);
689 memcpy(data
[thr
].unc
+ off
,
690 data_of(*snapshot
), PAGE_SIZE
);
694 "PM: Image saving progress: "
702 data
[thr
].unc_len
= off
;
704 atomic_set(&data
[thr
].ready
, 1);
705 wake_up(&data
[thr
].go
);
711 crc
->run_threads
= thr
;
712 atomic_set(&crc
->ready
, 1);
715 for (run_threads
= thr
, thr
= 0; thr
< run_threads
; thr
++) {
716 wait_event(data
[thr
].done
,
717 atomic_read(&data
[thr
].stop
));
718 atomic_set(&data
[thr
].stop
, 0);
723 printk(KERN_ERR
"PM: LZO compression failed\n");
727 if (unlikely(!data
[thr
].cmp_len
||
729 lzo1x_worst_compress(data
[thr
].unc_len
))) {
731 "PM: Invalid LZO compressed length\n");
736 *(size_t *)data
[thr
].cmp
= data
[thr
].cmp_len
;
739 * Given we are writing one page at a time to disk, we
740 * copy that much from the buffer, although the last
741 * bit will likely be smaller than full page. This is
742 * OK - we saved the length of the compressed data, so
743 * any garbage at the end will be discarded when we
747 off
< LZO_HEADER
+ data
[thr
].cmp_len
;
749 memcpy(page
, data
[thr
].cmp
+ off
, PAGE_SIZE
);
751 ret
= swap_write_page(handle
, page
, &bio
);
757 wait_event(crc
->done
, atomic_read(&crc
->stop
));
758 atomic_set(&crc
->stop
, 0);
762 err2
= hib_wait_on_bio_chain(&bio
);
767 printk(KERN_INFO
"PM: Image saving done.\n");
768 swsusp_show_speed(start
, stop
, nr_to_write
, "Wrote");
772 kthread_stop(crc
->thr
);
776 for (thr
= 0; thr
< nr_threads
; thr
++)
778 kthread_stop(data
[thr
].thr
);
781 if (page
) free_page((unsigned long)page
);
787 * enough_swap - Make sure we have enough swap to save the image.
789 * Returns TRUE or FALSE after checking the total amount of swap
790 * space avaiable from the resume partition.
793 static int enough_swap(unsigned int nr_pages
, unsigned int flags
)
795 unsigned int free_swap
= count_swap_pages(root_swap
, 1);
796 unsigned int required
;
798 pr_debug("PM: Free swap pages: %u\n", free_swap
);
800 required
= PAGES_FOR_IO
+ nr_pages
;
801 return free_swap
> required
;
805 * swsusp_write - Write entire image and metadata.
806 * @flags: flags to pass to the "boot" kernel in the image header
808 * It is important _NOT_ to umount filesystems at this point. We want
809 * them synced (in case something goes wrong) but we DO not want to mark
810 * filesystem clean: it is not. (And it does not matter, if we resume
811 * correctly, we'll mark system clean, anyway.)
814 int swsusp_write(unsigned int flags
)
816 struct swap_map_handle handle
;
817 struct snapshot_handle snapshot
;
818 struct swsusp_info
*header
;
822 pages
= snapshot_get_image_size();
823 error
= get_swap_writer(&handle
);
825 printk(KERN_ERR
"PM: Cannot get swap writer\n");
828 if (flags
& SF_NOCOMPRESS_MODE
) {
829 if (!enough_swap(pages
, flags
)) {
830 printk(KERN_ERR
"PM: Not enough free swap\n");
835 memset(&snapshot
, 0, sizeof(struct snapshot_handle
));
836 error
= snapshot_read_next(&snapshot
);
837 if (error
< PAGE_SIZE
) {
843 header
= (struct swsusp_info
*)data_of(snapshot
);
844 error
= swap_write_page(&handle
, header
, NULL
);
846 error
= (flags
& SF_NOCOMPRESS_MODE
) ?
847 save_image(&handle
, &snapshot
, pages
- 1) :
848 save_image_lzo(&handle
, &snapshot
, pages
- 1);
851 error
= swap_writer_finish(&handle
, flags
, error
);
856 * The following functions allow us to read data using a swap map
857 * in a file-alike way
860 static void release_swap_reader(struct swap_map_handle
*handle
)
862 struct swap_map_page_list
*tmp
;
864 while (handle
->maps
) {
865 if (handle
->maps
->map
)
866 free_page((unsigned long)handle
->maps
->map
);
868 handle
->maps
= handle
->maps
->next
;
874 static int get_swap_reader(struct swap_map_handle
*handle
,
875 unsigned int *flags_p
)
878 struct swap_map_page_list
*tmp
, *last
;
881 *flags_p
= swsusp_header
->flags
;
883 if (!swsusp_header
->image
) /* how can this happen? */
887 last
= handle
->maps
= NULL
;
888 offset
= swsusp_header
->image
;
890 tmp
= kmalloc(sizeof(*handle
->maps
), GFP_KERNEL
);
892 release_swap_reader(handle
);
895 memset(tmp
, 0, sizeof(*tmp
));
902 tmp
->map
= (struct swap_map_page
*)
903 __get_free_page(__GFP_WAIT
| __GFP_HIGH
);
905 release_swap_reader(handle
);
909 error
= hib_bio_read_page(offset
, tmp
->map
, NULL
);
911 release_swap_reader(handle
);
914 offset
= tmp
->map
->next_swap
;
917 handle
->cur
= handle
->maps
->map
;
921 static int swap_read_page(struct swap_map_handle
*handle
, void *buf
,
922 struct bio
**bio_chain
)
926 struct swap_map_page_list
*tmp
;
930 offset
= handle
->cur
->entries
[handle
->k
];
933 error
= hib_bio_read_page(offset
, buf
, bio_chain
);
936 if (++handle
->k
>= MAP_PAGE_ENTRIES
) {
938 free_page((unsigned long)handle
->maps
->map
);
940 handle
->maps
= handle
->maps
->next
;
943 release_swap_reader(handle
);
945 handle
->cur
= handle
->maps
->map
;
950 static int swap_reader_finish(struct swap_map_handle
*handle
)
952 release_swap_reader(handle
);
958 * load_image - load the image using the swap map handle
959 * @handle and the snapshot handle @snapshot
960 * (assume there are @nr_pages pages to load)
963 static int load_image(struct swap_map_handle
*handle
,
964 struct snapshot_handle
*snapshot
,
965 unsigned int nr_to_read
)
975 printk(KERN_INFO
"PM: Loading image data pages (%u pages)...\n",
984 ret
= snapshot_write_next(snapshot
);
987 ret
= swap_read_page(handle
, data_of(*snapshot
), &bio
);
990 if (snapshot
->sync_read
)
991 ret
= hib_wait_on_bio_chain(&bio
);
995 printk(KERN_INFO
"PM: Image loading progress: %3d%%\n",
999 err2
= hib_wait_on_bio_chain(&bio
);
1004 printk(KERN_INFO
"PM: Image loading done.\n");
1005 snapshot_write_finalize(snapshot
);
1006 if (!snapshot_image_loaded(snapshot
))
1009 swsusp_show_speed(start
, stop
, nr_to_read
, "Read");
1014 * Structure used for LZO data decompression.
1017 struct task_struct
*thr
; /* thread */
1018 atomic_t ready
; /* ready to start flag */
1019 atomic_t stop
; /* ready to stop flag */
1020 int ret
; /* return code */
1021 wait_queue_head_t go
; /* start decompression */
1022 wait_queue_head_t done
; /* decompression done */
1023 size_t unc_len
; /* uncompressed length */
1024 size_t cmp_len
; /* compressed length */
1025 unsigned char unc
[LZO_UNC_SIZE
]; /* uncompressed buffer */
1026 unsigned char cmp
[LZO_CMP_SIZE
]; /* compressed buffer */
1030 * Deompression function that runs in its own thread.
1032 static int lzo_decompress_threadfn(void *data
)
1034 struct dec_data
*d
= data
;
1037 wait_event(d
->go
, atomic_read(&d
->ready
) ||
1038 kthread_should_stop());
1039 if (kthread_should_stop()) {
1042 atomic_set(&d
->stop
, 1);
1046 atomic_set(&d
->ready
, 0);
1048 d
->unc_len
= LZO_UNC_SIZE
;
1049 d
->ret
= lzo1x_decompress_safe(d
->cmp
+ LZO_HEADER
, d
->cmp_len
,
1050 d
->unc
, &d
->unc_len
);
1051 atomic_set(&d
->stop
, 1);
1058 * load_image_lzo - Load compressed image data and decompress them with LZO.
1059 * @handle: Swap map handle to use for loading data.
1060 * @snapshot: Image to copy uncompressed data into.
1061 * @nr_to_read: Number of pages to load.
1063 static int load_image_lzo(struct swap_map_handle
*handle
,
1064 struct snapshot_handle
*snapshot
,
1065 unsigned int nr_to_read
)
1075 unsigned i
, thr
, run_threads
, nr_threads
;
1076 unsigned ring
= 0, pg
= 0, ring_size
= 0,
1077 have
= 0, want
, need
, asked
= 0;
1078 unsigned long read_pages
= 0;
1079 unsigned char **page
= NULL
;
1080 struct dec_data
*data
= NULL
;
1081 struct crc_data
*crc
= NULL
;
1084 * We'll limit the number of threads for decompression to limit memory
1087 nr_threads
= num_online_cpus() - 1;
1088 nr_threads
= clamp_val(nr_threads
, 1, LZO_THREADS
);
1090 page
= vmalloc(sizeof(*page
) * LZO_MAX_RD_PAGES
);
1092 printk(KERN_ERR
"PM: Failed to allocate LZO page\n");
1097 data
= vmalloc(sizeof(*data
) * nr_threads
);
1099 printk(KERN_ERR
"PM: Failed to allocate LZO data\n");
1103 for (thr
= 0; thr
< nr_threads
; thr
++)
1104 memset(&data
[thr
], 0, offsetof(struct dec_data
, go
));
1106 crc
= kmalloc(sizeof(*crc
), GFP_KERNEL
);
1108 printk(KERN_ERR
"PM: Failed to allocate crc\n");
1112 memset(crc
, 0, offsetof(struct crc_data
, go
));
1115 * Start the decompression threads.
1117 for (thr
= 0; thr
< nr_threads
; thr
++) {
1118 init_waitqueue_head(&data
[thr
].go
);
1119 init_waitqueue_head(&data
[thr
].done
);
1121 data
[thr
].thr
= kthread_run(lzo_decompress_threadfn
,
1123 "image_decompress/%u", thr
);
1124 if (IS_ERR(data
[thr
].thr
)) {
1125 data
[thr
].thr
= NULL
;
1127 "PM: Cannot start decompression threads\n");
1134 * Start the CRC32 thread.
1136 init_waitqueue_head(&crc
->go
);
1137 init_waitqueue_head(&crc
->done
);
1140 crc
->crc32
= &handle
->crc32
;
1141 for (thr
= 0; thr
< nr_threads
; thr
++) {
1142 crc
->unc
[thr
] = data
[thr
].unc
;
1143 crc
->unc_len
[thr
] = &data
[thr
].unc_len
;
1146 crc
->thr
= kthread_run(crc32_threadfn
, crc
, "image_crc32");
1147 if (IS_ERR(crc
->thr
)) {
1149 printk(KERN_ERR
"PM: Cannot start CRC32 thread\n");
1155 * Set the number of pages for read buffering.
1156 * This is complete guesswork, because we'll only know the real
1157 * picture once prepare_image() is called, which is much later on
1158 * during the image load phase. We'll assume the worst case and
1159 * say that none of the image pages are from high memory.
1161 if (low_free_pages() > snapshot_get_image_size())
1162 read_pages
= (low_free_pages() - snapshot_get_image_size()) / 2;
1163 read_pages
= clamp_val(read_pages
, LZO_MIN_RD_PAGES
, LZO_MAX_RD_PAGES
);
1165 for (i
= 0; i
< read_pages
; i
++) {
1166 page
[i
] = (void *)__get_free_page(i
< LZO_CMP_PAGES
?
1167 __GFP_WAIT
| __GFP_HIGH
:
1168 __GFP_WAIT
| __GFP_NOWARN
|
1172 if (i
< LZO_CMP_PAGES
) {
1175 "PM: Failed to allocate LZO pages\n");
1183 want
= ring_size
= i
;
1186 "PM: Using %u thread(s) for decompression.\n"
1187 "PM: Loading and decompressing image data (%u pages)...\n",
1188 nr_threads
, nr_to_read
);
1189 m
= nr_to_read
/ 10;
1194 start
= ktime_get();
1196 ret
= snapshot_write_next(snapshot
);
1201 for (i
= 0; !eof
&& i
< want
; i
++) {
1202 ret
= swap_read_page(handle
, page
[ring
], &bio
);
1205 * On real read error, finish. On end of data,
1206 * set EOF flag and just exit the read loop.
1209 handle
->cur
->entries
[handle
->k
]) {
1216 if (++ring
>= ring_size
)
1223 * We are out of data, wait for some more.
1229 ret
= hib_wait_on_bio_chain(&bio
);
1238 if (crc
->run_threads
) {
1239 wait_event(crc
->done
, atomic_read(&crc
->stop
));
1240 atomic_set(&crc
->stop
, 0);
1241 crc
->run_threads
= 0;
1244 for (thr
= 0; have
&& thr
< nr_threads
; thr
++) {
1245 data
[thr
].cmp_len
= *(size_t *)page
[pg
];
1246 if (unlikely(!data
[thr
].cmp_len
||
1248 lzo1x_worst_compress(LZO_UNC_SIZE
))) {
1250 "PM: Invalid LZO compressed length\n");
1255 need
= DIV_ROUND_UP(data
[thr
].cmp_len
+ LZO_HEADER
,
1266 off
< LZO_HEADER
+ data
[thr
].cmp_len
;
1268 memcpy(data
[thr
].cmp
+ off
,
1269 page
[pg
], PAGE_SIZE
);
1272 if (++pg
>= ring_size
)
1276 atomic_set(&data
[thr
].ready
, 1);
1277 wake_up(&data
[thr
].go
);
1281 * Wait for more data while we are decompressing.
1283 if (have
< LZO_CMP_PAGES
&& asked
) {
1284 ret
= hib_wait_on_bio_chain(&bio
);
1293 for (run_threads
= thr
, thr
= 0; thr
< run_threads
; thr
++) {
1294 wait_event(data
[thr
].done
,
1295 atomic_read(&data
[thr
].stop
));
1296 atomic_set(&data
[thr
].stop
, 0);
1298 ret
= data
[thr
].ret
;
1302 "PM: LZO decompression failed\n");
1306 if (unlikely(!data
[thr
].unc_len
||
1307 data
[thr
].unc_len
> LZO_UNC_SIZE
||
1308 data
[thr
].unc_len
& (PAGE_SIZE
- 1))) {
1310 "PM: Invalid LZO uncompressed length\n");
1316 off
< data
[thr
].unc_len
; off
+= PAGE_SIZE
) {
1317 memcpy(data_of(*snapshot
),
1318 data
[thr
].unc
+ off
, PAGE_SIZE
);
1320 if (!(nr_pages
% m
))
1322 "PM: Image loading progress: "
1327 ret
= snapshot_write_next(snapshot
);
1329 crc
->run_threads
= thr
+ 1;
1330 atomic_set(&crc
->ready
, 1);
1337 crc
->run_threads
= thr
;
1338 atomic_set(&crc
->ready
, 1);
1343 if (crc
->run_threads
) {
1344 wait_event(crc
->done
, atomic_read(&crc
->stop
));
1345 atomic_set(&crc
->stop
, 0);
1349 printk(KERN_INFO
"PM: Image loading done.\n");
1350 snapshot_write_finalize(snapshot
);
1351 if (!snapshot_image_loaded(snapshot
))
1354 if (swsusp_header
->flags
& SF_CRC32_MODE
) {
1355 if(handle
->crc32
!= swsusp_header
->crc32
) {
1357 "PM: Invalid image CRC32!\n");
1363 swsusp_show_speed(start
, stop
, nr_to_read
, "Read");
1365 for (i
= 0; i
< ring_size
; i
++)
1366 free_page((unsigned long)page
[i
]);
1369 kthread_stop(crc
->thr
);
1373 for (thr
= 0; thr
< nr_threads
; thr
++)
1375 kthread_stop(data
[thr
].thr
);
1384 * swsusp_read - read the hibernation image.
1385 * @flags_p: flags passed by the "frozen" kernel in the image header should
1386 * be written into this memory location
1389 int swsusp_read(unsigned int *flags_p
)
1392 struct swap_map_handle handle
;
1393 struct snapshot_handle snapshot
;
1394 struct swsusp_info
*header
;
1396 memset(&snapshot
, 0, sizeof(struct snapshot_handle
));
1397 error
= snapshot_write_next(&snapshot
);
1398 if (error
< PAGE_SIZE
)
1399 return error
< 0 ? error
: -EFAULT
;
1400 header
= (struct swsusp_info
*)data_of(snapshot
);
1401 error
= get_swap_reader(&handle
, flags_p
);
1405 error
= swap_read_page(&handle
, header
, NULL
);
1407 error
= (*flags_p
& SF_NOCOMPRESS_MODE
) ?
1408 load_image(&handle
, &snapshot
, header
->pages
- 1) :
1409 load_image_lzo(&handle
, &snapshot
, header
->pages
- 1);
1411 swap_reader_finish(&handle
);
1414 pr_debug("PM: Image successfully loaded\n");
1416 pr_debug("PM: Error %d resuming\n", error
);
1421 * swsusp_check - Check for swsusp signature in the resume device
1424 int swsusp_check(void)
1428 hib_resume_bdev
= blkdev_get_by_dev(swsusp_resume_device
,
1430 if (!IS_ERR(hib_resume_bdev
)) {
1431 set_blocksize(hib_resume_bdev
, PAGE_SIZE
);
1432 clear_page(swsusp_header
);
1433 error
= hib_bio_read_page(swsusp_resume_block
,
1434 swsusp_header
, NULL
);
1438 if (!memcmp(HIBERNATE_SIG
, swsusp_header
->sig
, 10)) {
1439 memcpy(swsusp_header
->sig
, swsusp_header
->orig_sig
, 10);
1440 /* Reset swap signature now */
1441 error
= hib_bio_write_page(swsusp_resume_block
,
1442 swsusp_header
, NULL
);
1449 blkdev_put(hib_resume_bdev
, FMODE_READ
);
1451 pr_debug("PM: Image signature found, resuming\n");
1453 error
= PTR_ERR(hib_resume_bdev
);
1457 pr_debug("PM: Image not found (code %d)\n", error
);
1463 * swsusp_close - close swap device.
1466 void swsusp_close(fmode_t mode
)
1468 if (IS_ERR(hib_resume_bdev
)) {
1469 pr_debug("PM: Image device not initialised\n");
1473 blkdev_put(hib_resume_bdev
, mode
);
1477 * swsusp_unmark - Unmark swsusp signature in the resume device
1480 #ifdef CONFIG_SUSPEND
1481 int swsusp_unmark(void)
1485 hib_bio_read_page(swsusp_resume_block
, swsusp_header
, NULL
);
1486 if (!memcmp(HIBERNATE_SIG
,swsusp_header
->sig
, 10)) {
1487 memcpy(swsusp_header
->sig
,swsusp_header
->orig_sig
, 10);
1488 error
= hib_bio_write_page(swsusp_resume_block
,
1489 swsusp_header
, NULL
);
1491 printk(KERN_ERR
"PM: Cannot find swsusp signature!\n");
1496 * We just returned from suspend, we don't need the image any more.
1498 free_all_swap_pages(root_swap
);
1504 static int swsusp_header_init(void)
1506 swsusp_header
= (struct swsusp_header
*) __get_free_page(GFP_KERNEL
);
1508 panic("Could not allocate memory for swsusp_header\n");
1512 core_initcall(swsusp_header_init
);