2 * linux/kernel/power/snapshot.c
4 * This file provides system snapshot/restore functionality for swsusp.
6 * Copyright (C) 1998-2005 Pavel Machek <pavel@suse.cz>
7 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
9 * This file is released under the GPLv2.
13 #include <linux/version.h>
14 #include <linux/module.h>
16 #include <linux/suspend.h>
17 #include <linux/delay.h>
18 #include <linux/bitops.h>
19 #include <linux/spinlock.h>
20 #include <linux/kernel.h>
22 #include <linux/device.h>
23 #include <linux/init.h>
24 #include <linux/bootmem.h>
25 #include <linux/syscalls.h>
26 #include <linux/console.h>
27 #include <linux/highmem.h>
28 #include <linux/list.h>
30 #include <asm/uaccess.h>
31 #include <asm/mmu_context.h>
32 #include <asm/pgtable.h>
33 #include <asm/tlbflush.h>
38 static int swsusp_page_is_free(struct page
*);
39 static void swsusp_set_page_forbidden(struct page
*);
40 static void swsusp_unset_page_forbidden(struct page
*);
43 * Preferred image size in bytes (tunable via /sys/power/image_size).
44 * When it is set to N, swsusp will do its best to ensure the image
45 * size will not exceed N bytes, but if that is impossible, it will
46 * try to create the smallest image possible.
48 unsigned long image_size
= 500 * 1024 * 1024;
50 /* List of PBEs needed for restoring the pages that were allocated before
51 * the suspend and included in the suspend image, but have also been
52 * allocated by the "resume" kernel, so their contents cannot be written
53 * directly to their "original" page frames.
55 struct pbe
*restore_pblist
;
57 /* Pointer to an auxiliary buffer (1 page) */
61 * @safe_needed - on resume, for storing the PBE list and the image,
62 * we can only use memory pages that do not conflict with the pages
63 * used before suspend. The unsafe pages have PageNosaveFree set
64 * and we count them using unsafe_pages.
66 * Each allocated image page is marked as PageNosave and PageNosaveFree
67 * so that swsusp_free() can release it.
72 #define PG_UNSAFE_CLEAR 1
73 #define PG_UNSAFE_KEEP 0
75 static unsigned int allocated_unsafe_pages
;
77 static void *get_image_page(gfp_t gfp_mask
, int safe_needed
)
81 res
= (void *)get_zeroed_page(gfp_mask
);
83 while (res
&& swsusp_page_is_free(virt_to_page(res
))) {
84 /* The page is unsafe, mark it for swsusp_free() */
85 swsusp_set_page_forbidden(virt_to_page(res
));
86 allocated_unsafe_pages
++;
87 res
= (void *)get_zeroed_page(gfp_mask
);
90 swsusp_set_page_forbidden(virt_to_page(res
));
91 swsusp_set_page_free(virt_to_page(res
));
96 unsigned long get_safe_page(gfp_t gfp_mask
)
98 return (unsigned long)get_image_page(gfp_mask
, PG_SAFE
);
101 static struct page
*alloc_image_page(gfp_t gfp_mask
)
105 page
= alloc_page(gfp_mask
);
107 swsusp_set_page_forbidden(page
);
108 swsusp_set_page_free(page
);
114 * free_image_page - free page represented by @addr, allocated with
115 * get_image_page (page flags set by it must be cleared)
118 static inline void free_image_page(void *addr
, int clear_nosave_free
)
122 BUG_ON(!virt_addr_valid(addr
));
124 page
= virt_to_page(addr
);
126 swsusp_unset_page_forbidden(page
);
127 if (clear_nosave_free
)
128 swsusp_unset_page_free(page
);
133 /* struct linked_page is used to build chains of pages */
135 #define LINKED_PAGE_DATA_SIZE (PAGE_SIZE - sizeof(void *))
138 struct linked_page
*next
;
139 char data
[LINKED_PAGE_DATA_SIZE
];
140 } __attribute__((packed
));
143 free_list_of_pages(struct linked_page
*list
, int clear_page_nosave
)
146 struct linked_page
*lp
= list
->next
;
148 free_image_page(list
, clear_page_nosave
);
154 * struct chain_allocator is used for allocating small objects out of
155 * a linked list of pages called 'the chain'.
157 * The chain grows each time when there is no room for a new object in
158 * the current page. The allocated objects cannot be freed individually.
159 * It is only possible to free them all at once, by freeing the entire
162 * NOTE: The chain allocator may be inefficient if the allocated objects
163 * are not much smaller than PAGE_SIZE.
166 struct chain_allocator
{
167 struct linked_page
*chain
; /* the chain */
168 unsigned int used_space
; /* total size of objects allocated out
169 * of the current page
171 gfp_t gfp_mask
; /* mask for allocating pages */
172 int safe_needed
; /* if set, only "safe" pages are allocated */
176 chain_init(struct chain_allocator
*ca
, gfp_t gfp_mask
, int safe_needed
)
179 ca
->used_space
= LINKED_PAGE_DATA_SIZE
;
180 ca
->gfp_mask
= gfp_mask
;
181 ca
->safe_needed
= safe_needed
;
184 static void *chain_alloc(struct chain_allocator
*ca
, unsigned int size
)
188 if (LINKED_PAGE_DATA_SIZE
- ca
->used_space
< size
) {
189 struct linked_page
*lp
;
191 lp
= get_image_page(ca
->gfp_mask
, ca
->safe_needed
);
195 lp
->next
= ca
->chain
;
199 ret
= ca
->chain
->data
+ ca
->used_space
;
200 ca
->used_space
+= size
;
205 * Data types related to memory bitmaps.
207 * Memory bitmap is a structure consiting of many linked lists of
208 * objects. The main list's elements are of type struct zone_bitmap
209 * and each of them corresonds to one zone. For each zone bitmap
210 * object there is a list of objects of type struct bm_block that
211 * represent each blocks of bitmap in which information is stored.
213 * struct memory_bitmap contains a pointer to the main list of zone
214 * bitmap objects, a struct bm_position used for browsing the bitmap,
215 * and a pointer to the list of pages used for allocating all of the
216 * zone bitmap objects and bitmap block objects.
218 * NOTE: It has to be possible to lay out the bitmap in memory
219 * using only allocations of order 0. Additionally, the bitmap is
220 * designed to work with arbitrary number of zones (this is over the
221 * top for now, but let's avoid making unnecessary assumptions ;-).
223 * struct zone_bitmap contains a pointer to a list of bitmap block
224 * objects and a pointer to the bitmap block object that has been
225 * most recently used for setting bits. Additionally, it contains the
226 * pfns that correspond to the start and end of the represented zone.
228 * struct bm_block contains a pointer to the memory page in which
229 * information is stored (in the form of a block of bitmap)
230 * It also contains the pfns that correspond to the start and end of
231 * the represented memory area.
234 #define BM_END_OF_MAP (~0UL)
236 #define BM_BITS_PER_BLOCK (PAGE_SIZE * BITS_PER_BYTE)
239 struct list_head hook
; /* hook into a list of bitmap blocks */
240 unsigned long start_pfn
; /* pfn represented by the first bit */
241 unsigned long end_pfn
; /* pfn represented by the last bit plus 1 */
242 unsigned long *data
; /* bitmap representing pages */
245 static inline unsigned long bm_block_bits(struct bm_block
*bb
)
247 return bb
->end_pfn
- bb
->start_pfn
;
250 /* strcut bm_position is used for browsing memory bitmaps */
253 struct bm_block
*block
;
257 struct memory_bitmap
{
258 struct list_head blocks
; /* list of bitmap blocks */
259 struct linked_page
*p_list
; /* list of pages used to store zone
260 * bitmap objects and bitmap block
263 struct bm_position cur
; /* most recently used bit position */
266 /* Functions that operate on memory bitmaps */
268 static void memory_bm_position_reset(struct memory_bitmap
*bm
)
270 bm
->cur
.block
= list_entry(bm
->blocks
.next
, struct bm_block
, hook
);
274 static void memory_bm_free(struct memory_bitmap
*bm
, int clear_nosave_free
);
277 * create_bm_block_list - create a list of block bitmap objects
278 * @pages - number of pages to track
279 * @list - list to put the allocated blocks into
280 * @ca - chain allocator to be used for allocating memory
282 static int create_bm_block_list(unsigned long pages
,
283 struct list_head
*list
,
284 struct chain_allocator
*ca
)
286 unsigned int nr_blocks
= DIV_ROUND_UP(pages
, BM_BITS_PER_BLOCK
);
288 while (nr_blocks
-- > 0) {
291 bb
= chain_alloc(ca
, sizeof(struct bm_block
));
294 list_add(&bb
->hook
, list
);
301 struct list_head hook
;
307 * free_mem_extents - free a list of memory extents
308 * @list - list of extents to empty
310 static void free_mem_extents(struct list_head
*list
)
312 struct mem_extent
*ext
, *aux
;
314 list_for_each_entry_safe(ext
, aux
, list
, hook
) {
315 list_del(&ext
->hook
);
321 * create_mem_extents - create a list of memory extents representing
322 * contiguous ranges of PFNs
323 * @list - list to put the extents into
324 * @gfp_mask - mask to use for memory allocations
326 static int create_mem_extents(struct list_head
*list
, gfp_t gfp_mask
)
330 INIT_LIST_HEAD(list
);
332 for_each_populated_zone(zone
) {
333 unsigned long zone_start
, zone_end
;
334 struct mem_extent
*ext
, *cur
, *aux
;
336 zone_start
= zone
->zone_start_pfn
;
337 zone_end
= zone
->zone_start_pfn
+ zone
->spanned_pages
;
339 list_for_each_entry(ext
, list
, hook
)
340 if (zone_start
<= ext
->end
)
343 if (&ext
->hook
== list
|| zone_end
< ext
->start
) {
344 /* New extent is necessary */
345 struct mem_extent
*new_ext
;
347 new_ext
= kzalloc(sizeof(struct mem_extent
), gfp_mask
);
349 free_mem_extents(list
);
352 new_ext
->start
= zone_start
;
353 new_ext
->end
= zone_end
;
354 list_add_tail(&new_ext
->hook
, &ext
->hook
);
358 /* Merge this zone's range of PFNs with the existing one */
359 if (zone_start
< ext
->start
)
360 ext
->start
= zone_start
;
361 if (zone_end
> ext
->end
)
364 /* More merging may be possible */
366 list_for_each_entry_safe_continue(cur
, aux
, list
, hook
) {
367 if (zone_end
< cur
->start
)
369 if (zone_end
< cur
->end
)
371 list_del(&cur
->hook
);
380 * memory_bm_create - allocate memory for a memory bitmap
383 memory_bm_create(struct memory_bitmap
*bm
, gfp_t gfp_mask
, int safe_needed
)
385 struct chain_allocator ca
;
386 struct list_head mem_extents
;
387 struct mem_extent
*ext
;
390 chain_init(&ca
, gfp_mask
, safe_needed
);
391 INIT_LIST_HEAD(&bm
->blocks
);
393 error
= create_mem_extents(&mem_extents
, gfp_mask
);
397 list_for_each_entry(ext
, &mem_extents
, hook
) {
399 unsigned long pfn
= ext
->start
;
400 unsigned long pages
= ext
->end
- ext
->start
;
402 bb
= list_entry(bm
->blocks
.prev
, struct bm_block
, hook
);
404 error
= create_bm_block_list(pages
, bm
->blocks
.prev
, &ca
);
408 list_for_each_entry_continue(bb
, &bm
->blocks
, hook
) {
409 bb
->data
= get_image_page(gfp_mask
, safe_needed
);
416 if (pages
>= BM_BITS_PER_BLOCK
) {
417 pfn
+= BM_BITS_PER_BLOCK
;
418 pages
-= BM_BITS_PER_BLOCK
;
420 /* This is executed only once in the loop */
427 bm
->p_list
= ca
.chain
;
428 memory_bm_position_reset(bm
);
430 free_mem_extents(&mem_extents
);
434 bm
->p_list
= ca
.chain
;
435 memory_bm_free(bm
, PG_UNSAFE_CLEAR
);
440 * memory_bm_free - free memory occupied by the memory bitmap @bm
442 static void memory_bm_free(struct memory_bitmap
*bm
, int clear_nosave_free
)
446 list_for_each_entry(bb
, &bm
->blocks
, hook
)
448 free_image_page(bb
->data
, clear_nosave_free
);
450 free_list_of_pages(bm
->p_list
, clear_nosave_free
);
452 INIT_LIST_HEAD(&bm
->blocks
);
456 * memory_bm_find_bit - find the bit in the bitmap @bm that corresponds
457 * to given pfn. The cur_zone_bm member of @bm and the cur_block member
458 * of @bm->cur_zone_bm are updated.
460 static int memory_bm_find_bit(struct memory_bitmap
*bm
, unsigned long pfn
,
461 void **addr
, unsigned int *bit_nr
)
466 * Check if the pfn corresponds to the current bitmap block and find
467 * the block where it fits if this is not the case.
470 if (pfn
< bb
->start_pfn
)
471 list_for_each_entry_continue_reverse(bb
, &bm
->blocks
, hook
)
472 if (pfn
>= bb
->start_pfn
)
475 if (pfn
>= bb
->end_pfn
)
476 list_for_each_entry_continue(bb
, &bm
->blocks
, hook
)
477 if (pfn
>= bb
->start_pfn
&& pfn
< bb
->end_pfn
)
480 if (&bb
->hook
== &bm
->blocks
)
483 /* The block has been found */
485 pfn
-= bb
->start_pfn
;
486 bm
->cur
.bit
= pfn
+ 1;
492 static void memory_bm_set_bit(struct memory_bitmap
*bm
, unsigned long pfn
)
498 error
= memory_bm_find_bit(bm
, pfn
, &addr
, &bit
);
503 static int mem_bm_set_bit_check(struct memory_bitmap
*bm
, unsigned long pfn
)
509 error
= memory_bm_find_bit(bm
, pfn
, &addr
, &bit
);
515 static void memory_bm_clear_bit(struct memory_bitmap
*bm
, unsigned long pfn
)
521 error
= memory_bm_find_bit(bm
, pfn
, &addr
, &bit
);
523 clear_bit(bit
, addr
);
526 static int memory_bm_test_bit(struct memory_bitmap
*bm
, unsigned long pfn
)
532 error
= memory_bm_find_bit(bm
, pfn
, &addr
, &bit
);
534 return test_bit(bit
, addr
);
537 static bool memory_bm_pfn_present(struct memory_bitmap
*bm
, unsigned long pfn
)
542 return !memory_bm_find_bit(bm
, pfn
, &addr
, &bit
);
546 * memory_bm_next_pfn - find the pfn that corresponds to the next set bit
547 * in the bitmap @bm. If the pfn cannot be found, BM_END_OF_MAP is
550 * It is required to run memory_bm_position_reset() before the first call to
554 static unsigned long memory_bm_next_pfn(struct memory_bitmap
*bm
)
562 bit
= find_next_bit(bb
->data
, bm_block_bits(bb
), bit
);
563 if (bit
< bm_block_bits(bb
))
566 bb
= list_entry(bb
->hook
.next
, struct bm_block
, hook
);
569 } while (&bb
->hook
!= &bm
->blocks
);
571 memory_bm_position_reset(bm
);
572 return BM_END_OF_MAP
;
575 bm
->cur
.bit
= bit
+ 1;
576 return bb
->start_pfn
+ bit
;
580 * This structure represents a range of page frames the contents of which
581 * should not be saved during the suspend.
584 struct nosave_region
{
585 struct list_head list
;
586 unsigned long start_pfn
;
587 unsigned long end_pfn
;
590 static LIST_HEAD(nosave_regions
);
593 * register_nosave_region - register a range of page frames the contents
594 * of which should not be saved during the suspend (to be used in the early
595 * initialization code)
599 __register_nosave_region(unsigned long start_pfn
, unsigned long end_pfn
,
602 struct nosave_region
*region
;
604 if (start_pfn
>= end_pfn
)
607 if (!list_empty(&nosave_regions
)) {
608 /* Try to extend the previous region (they should be sorted) */
609 region
= list_entry(nosave_regions
.prev
,
610 struct nosave_region
, list
);
611 if (region
->end_pfn
== start_pfn
) {
612 region
->end_pfn
= end_pfn
;
617 /* during init, this shouldn't fail */
618 region
= kmalloc(sizeof(struct nosave_region
), GFP_KERNEL
);
621 /* This allocation cannot fail */
622 region
= alloc_bootmem(sizeof(struct nosave_region
));
623 region
->start_pfn
= start_pfn
;
624 region
->end_pfn
= end_pfn
;
625 list_add_tail(®ion
->list
, &nosave_regions
);
627 printk(KERN_INFO
"PM: Registered nosave memory: %016lx - %016lx\n",
628 start_pfn
<< PAGE_SHIFT
, end_pfn
<< PAGE_SHIFT
);
632 * Set bits in this map correspond to the page frames the contents of which
633 * should not be saved during the suspend.
635 static struct memory_bitmap
*forbidden_pages_map
;
637 /* Set bits in this map correspond to free page frames. */
638 static struct memory_bitmap
*free_pages_map
;
641 * Each page frame allocated for creating the image is marked by setting the
642 * corresponding bits in forbidden_pages_map and free_pages_map simultaneously
645 void swsusp_set_page_free(struct page
*page
)
648 memory_bm_set_bit(free_pages_map
, page_to_pfn(page
));
651 static int swsusp_page_is_free(struct page
*page
)
653 return free_pages_map
?
654 memory_bm_test_bit(free_pages_map
, page_to_pfn(page
)) : 0;
657 void swsusp_unset_page_free(struct page
*page
)
660 memory_bm_clear_bit(free_pages_map
, page_to_pfn(page
));
663 static void swsusp_set_page_forbidden(struct page
*page
)
665 if (forbidden_pages_map
)
666 memory_bm_set_bit(forbidden_pages_map
, page_to_pfn(page
));
669 int swsusp_page_is_forbidden(struct page
*page
)
671 return forbidden_pages_map
?
672 memory_bm_test_bit(forbidden_pages_map
, page_to_pfn(page
)) : 0;
675 static void swsusp_unset_page_forbidden(struct page
*page
)
677 if (forbidden_pages_map
)
678 memory_bm_clear_bit(forbidden_pages_map
, page_to_pfn(page
));
682 * mark_nosave_pages - set bits corresponding to the page frames the
683 * contents of which should not be saved in a given bitmap.
686 static void mark_nosave_pages(struct memory_bitmap
*bm
)
688 struct nosave_region
*region
;
690 if (list_empty(&nosave_regions
))
693 list_for_each_entry(region
, &nosave_regions
, list
) {
696 pr_debug("PM: Marking nosave pages: %016lx - %016lx\n",
697 region
->start_pfn
<< PAGE_SHIFT
,
698 region
->end_pfn
<< PAGE_SHIFT
);
700 for (pfn
= region
->start_pfn
; pfn
< region
->end_pfn
; pfn
++)
701 if (pfn_valid(pfn
)) {
703 * It is safe to ignore the result of
704 * mem_bm_set_bit_check() here, since we won't
705 * touch the PFNs for which the error is
708 mem_bm_set_bit_check(bm
, pfn
);
714 * create_basic_memory_bitmaps - create bitmaps needed for marking page
715 * frames that should not be saved and free page frames. The pointers
716 * forbidden_pages_map and free_pages_map are only modified if everything
717 * goes well, because we don't want the bits to be used before both bitmaps
721 int create_basic_memory_bitmaps(void)
723 struct memory_bitmap
*bm1
, *bm2
;
726 BUG_ON(forbidden_pages_map
|| free_pages_map
);
728 bm1
= kzalloc(sizeof(struct memory_bitmap
), GFP_KERNEL
);
732 error
= memory_bm_create(bm1
, GFP_KERNEL
, PG_ANY
);
734 goto Free_first_object
;
736 bm2
= kzalloc(sizeof(struct memory_bitmap
), GFP_KERNEL
);
738 goto Free_first_bitmap
;
740 error
= memory_bm_create(bm2
, GFP_KERNEL
, PG_ANY
);
742 goto Free_second_object
;
744 forbidden_pages_map
= bm1
;
745 free_pages_map
= bm2
;
746 mark_nosave_pages(forbidden_pages_map
);
748 pr_debug("PM: Basic memory bitmaps created\n");
755 memory_bm_free(bm1
, PG_UNSAFE_CLEAR
);
762 * free_basic_memory_bitmaps - free memory bitmaps allocated by
763 * create_basic_memory_bitmaps(). The auxiliary pointers are necessary
764 * so that the bitmaps themselves are not referred to while they are being
768 void free_basic_memory_bitmaps(void)
770 struct memory_bitmap
*bm1
, *bm2
;
772 BUG_ON(!(forbidden_pages_map
&& free_pages_map
));
774 bm1
= forbidden_pages_map
;
775 bm2
= free_pages_map
;
776 forbidden_pages_map
= NULL
;
777 free_pages_map
= NULL
;
778 memory_bm_free(bm1
, PG_UNSAFE_CLEAR
);
780 memory_bm_free(bm2
, PG_UNSAFE_CLEAR
);
783 pr_debug("PM: Basic memory bitmaps freed\n");
787 * snapshot_additional_pages - estimate the number of additional pages
788 * be needed for setting up the suspend image data structures for given
789 * zone (usually the returned value is greater than the exact number)
792 unsigned int snapshot_additional_pages(struct zone
*zone
)
796 res
= DIV_ROUND_UP(zone
->spanned_pages
, BM_BITS_PER_BLOCK
);
797 res
+= DIV_ROUND_UP(res
* sizeof(struct bm_block
), PAGE_SIZE
);
801 #ifdef CONFIG_HIGHMEM
803 * count_free_highmem_pages - compute the total number of free highmem
804 * pages, system-wide.
807 static unsigned int count_free_highmem_pages(void)
810 unsigned int cnt
= 0;
812 for_each_populated_zone(zone
)
813 if (is_highmem(zone
))
814 cnt
+= zone_page_state(zone
, NR_FREE_PAGES
);
820 * saveable_highmem_page - Determine whether a highmem page should be
821 * included in the suspend image.
823 * We should save the page if it isn't Nosave or NosaveFree, or Reserved,
824 * and it isn't a part of a free chunk of pages.
826 static struct page
*saveable_highmem_page(struct zone
*zone
, unsigned long pfn
)
833 page
= pfn_to_page(pfn
);
834 if (page_zone(page
) != zone
)
837 BUG_ON(!PageHighMem(page
));
839 if (swsusp_page_is_forbidden(page
) || swsusp_page_is_free(page
) ||
847 * count_highmem_pages - compute the total number of saveable highmem
851 static unsigned int count_highmem_pages(void)
856 for_each_populated_zone(zone
) {
857 unsigned long pfn
, max_zone_pfn
;
859 if (!is_highmem(zone
))
862 mark_free_pages(zone
);
863 max_zone_pfn
= zone
->zone_start_pfn
+ zone
->spanned_pages
;
864 for (pfn
= zone
->zone_start_pfn
; pfn
< max_zone_pfn
; pfn
++)
865 if (saveable_highmem_page(zone
, pfn
))
871 static inline void *saveable_highmem_page(struct zone
*z
, unsigned long p
)
875 #endif /* CONFIG_HIGHMEM */
878 * saveable_page - Determine whether a non-highmem page should be included
879 * in the suspend image.
881 * We should save the page if it isn't Nosave, and is not in the range
882 * of pages statically defined as 'unsaveable', and it isn't a part of
883 * a free chunk of pages.
885 static struct page
*saveable_page(struct zone
*zone
, unsigned long pfn
)
892 page
= pfn_to_page(pfn
);
893 if (page_zone(page
) != zone
)
896 BUG_ON(PageHighMem(page
));
898 if (swsusp_page_is_forbidden(page
) || swsusp_page_is_free(page
))
901 if (PageReserved(page
)
902 && (!kernel_page_present(page
) || pfn_is_nosave(pfn
)))
909 * count_data_pages - compute the total number of saveable non-highmem
913 static unsigned int count_data_pages(void)
916 unsigned long pfn
, max_zone_pfn
;
919 for_each_populated_zone(zone
) {
920 if (is_highmem(zone
))
923 mark_free_pages(zone
);
924 max_zone_pfn
= zone
->zone_start_pfn
+ zone
->spanned_pages
;
925 for (pfn
= zone
->zone_start_pfn
; pfn
< max_zone_pfn
; pfn
++)
926 if (saveable_page(zone
, pfn
))
932 /* This is needed, because copy_page and memcpy are not usable for copying
935 static inline void do_copy_page(long *dst
, long *src
)
939 for (n
= PAGE_SIZE
/ sizeof(long); n
; n
--)
945 * safe_copy_page - check if the page we are going to copy is marked as
946 * present in the kernel page tables (this always is the case if
947 * CONFIG_DEBUG_PAGEALLOC is not set and in that case
948 * kernel_page_present() always returns 'true').
950 static void safe_copy_page(void *dst
, struct page
*s_page
)
952 if (kernel_page_present(s_page
)) {
953 do_copy_page(dst
, page_address(s_page
));
955 kernel_map_pages(s_page
, 1, 1);
956 do_copy_page(dst
, page_address(s_page
));
957 kernel_map_pages(s_page
, 1, 0);
962 #ifdef CONFIG_HIGHMEM
963 static inline struct page
*
964 page_is_saveable(struct zone
*zone
, unsigned long pfn
)
966 return is_highmem(zone
) ?
967 saveable_highmem_page(zone
, pfn
) : saveable_page(zone
, pfn
);
970 static void copy_data_page(unsigned long dst_pfn
, unsigned long src_pfn
)
972 struct page
*s_page
, *d_page
;
975 s_page
= pfn_to_page(src_pfn
);
976 d_page
= pfn_to_page(dst_pfn
);
977 if (PageHighMem(s_page
)) {
978 src
= kmap_atomic(s_page
, KM_USER0
);
979 dst
= kmap_atomic(d_page
, KM_USER1
);
980 do_copy_page(dst
, src
);
981 kunmap_atomic(src
, KM_USER0
);
982 kunmap_atomic(dst
, KM_USER1
);
984 if (PageHighMem(d_page
)) {
985 /* Page pointed to by src may contain some kernel
986 * data modified by kmap_atomic()
988 safe_copy_page(buffer
, s_page
);
989 dst
= kmap_atomic(d_page
, KM_USER0
);
990 memcpy(dst
, buffer
, PAGE_SIZE
);
991 kunmap_atomic(dst
, KM_USER0
);
993 safe_copy_page(page_address(d_page
), s_page
);
998 #define page_is_saveable(zone, pfn) saveable_page(zone, pfn)
1000 static inline void copy_data_page(unsigned long dst_pfn
, unsigned long src_pfn
)
1002 safe_copy_page(page_address(pfn_to_page(dst_pfn
)),
1003 pfn_to_page(src_pfn
));
1005 #endif /* CONFIG_HIGHMEM */
1008 copy_data_pages(struct memory_bitmap
*copy_bm
, struct memory_bitmap
*orig_bm
)
1013 for_each_populated_zone(zone
) {
1014 unsigned long max_zone_pfn
;
1016 mark_free_pages(zone
);
1017 max_zone_pfn
= zone
->zone_start_pfn
+ zone
->spanned_pages
;
1018 for (pfn
= zone
->zone_start_pfn
; pfn
< max_zone_pfn
; pfn
++)
1019 if (page_is_saveable(zone
, pfn
))
1020 memory_bm_set_bit(orig_bm
, pfn
);
1022 memory_bm_position_reset(orig_bm
);
1023 memory_bm_position_reset(copy_bm
);
1025 pfn
= memory_bm_next_pfn(orig_bm
);
1026 if (unlikely(pfn
== BM_END_OF_MAP
))
1028 copy_data_page(memory_bm_next_pfn(copy_bm
), pfn
);
1032 /* Total number of image pages */
1033 static unsigned int nr_copy_pages
;
1034 /* Number of pages needed for saving the original pfns of the image pages */
1035 static unsigned int nr_meta_pages
;
1037 * Numbers of normal and highmem page frames allocated for hibernation image
1038 * before suspending devices.
1040 unsigned int alloc_normal
, alloc_highmem
;
1042 * Memory bitmap used for marking saveable pages (during hibernation) or
1043 * hibernation image pages (during restore)
1045 static struct memory_bitmap orig_bm
;
1047 * Memory bitmap used during hibernation for marking allocated page frames that
1048 * will contain copies of saveable pages. During restore it is initially used
1049 * for marking hibernation image pages, but then the set bits from it are
1050 * duplicated in @orig_bm and it is released. On highmem systems it is next
1051 * used for marking "safe" highmem pages, but it has to be reinitialized for
1054 static struct memory_bitmap copy_bm
;
1057 * swsusp_free - free pages allocated for the suspend.
1059 * Suspend pages are alocated before the atomic copy is made, so we
1060 * need to release them after the resume.
1063 void swsusp_free(void)
1066 unsigned long pfn
, max_zone_pfn
;
1068 for_each_populated_zone(zone
) {
1069 max_zone_pfn
= zone
->zone_start_pfn
+ zone
->spanned_pages
;
1070 for (pfn
= zone
->zone_start_pfn
; pfn
< max_zone_pfn
; pfn
++)
1071 if (pfn_valid(pfn
)) {
1072 struct page
*page
= pfn_to_page(pfn
);
1074 if (swsusp_page_is_forbidden(page
) &&
1075 swsusp_page_is_free(page
)) {
1076 swsusp_unset_page_forbidden(page
);
1077 swsusp_unset_page_free(page
);
1084 restore_pblist
= NULL
;
1090 /* Helper functions used for the shrinking of memory. */
1092 #define GFP_IMAGE (GFP_KERNEL | __GFP_NOWARN)
1095 * preallocate_image_pages - Allocate a number of pages for hibernation image
1096 * @nr_pages: Number of page frames to allocate.
1097 * @mask: GFP flags to use for the allocation.
1099 * Return value: Number of page frames actually allocated
1101 static unsigned long preallocate_image_pages(unsigned long nr_pages
, gfp_t mask
)
1103 unsigned long nr_alloc
= 0;
1105 while (nr_pages
> 0) {
1108 page
= alloc_image_page(mask
);
1111 memory_bm_set_bit(©_bm
, page_to_pfn(page
));
1112 if (PageHighMem(page
))
1123 static unsigned long preallocate_image_memory(unsigned long nr_pages
)
1125 return preallocate_image_pages(nr_pages
, GFP_IMAGE
);
1128 #ifdef CONFIG_HIGHMEM
1129 static unsigned long preallocate_image_highmem(unsigned long nr_pages
)
1131 return preallocate_image_pages(nr_pages
, GFP_IMAGE
| __GFP_HIGHMEM
);
1135 * __fraction - Compute (an approximation of) x * (multiplier / base)
1137 static unsigned long __fraction(u64 x
, u64 multiplier
, u64 base
)
1141 return (unsigned long)x
;
1144 static unsigned long preallocate_highmem_fraction(unsigned long nr_pages
,
1145 unsigned long highmem
,
1146 unsigned long total
)
1148 unsigned long alloc
= __fraction(nr_pages
, highmem
, total
);
1150 return preallocate_image_pages(alloc
, GFP_IMAGE
| __GFP_HIGHMEM
);
1152 #else /* CONFIG_HIGHMEM */
1153 static inline unsigned long preallocate_image_highmem(unsigned long nr_pages
)
1158 static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages
,
1159 unsigned long highmem
,
1160 unsigned long total
)
1164 #endif /* CONFIG_HIGHMEM */
1167 * free_unnecessary_pages - Release preallocated pages not needed for the image
1169 static void free_unnecessary_pages(void)
1171 unsigned long save_highmem
, to_free_normal
, to_free_highmem
;
1173 to_free_normal
= alloc_normal
- count_data_pages();
1174 save_highmem
= count_highmem_pages();
1175 if (alloc_highmem
> save_highmem
) {
1176 to_free_highmem
= alloc_highmem
- save_highmem
;
1178 to_free_highmem
= 0;
1179 to_free_normal
-= save_highmem
- alloc_highmem
;
1182 memory_bm_position_reset(©_bm
);
1184 while (to_free_normal
> 0 || to_free_highmem
> 0) {
1185 unsigned long pfn
= memory_bm_next_pfn(©_bm
);
1186 struct page
*page
= pfn_to_page(pfn
);
1188 if (PageHighMem(page
)) {
1189 if (!to_free_highmem
)
1194 if (!to_free_normal
)
1199 memory_bm_clear_bit(©_bm
, pfn
);
1200 swsusp_unset_page_forbidden(page
);
1201 swsusp_unset_page_free(page
);
1207 * minimum_image_size - Estimate the minimum acceptable size of an image
1208 * @saveable: Number of saveable pages in the system.
1210 * We want to avoid attempting to free too much memory too hard, so estimate the
1211 * minimum acceptable size of a hibernation image to use as the lower limit for
1212 * preallocating memory.
1214 * We assume that the minimum image size should be proportional to
1216 * [number of saveable pages] - [number of pages that can be freed in theory]
1218 * where the second term is the sum of (1) reclaimable slab pages, (2) active
1219 * and (3) inactive anonymouns pages, (4) active and (5) inactive file pages,
1220 * minus mapped file pages.
1222 static unsigned long minimum_image_size(unsigned long saveable
)
1226 size
= global_page_state(NR_SLAB_RECLAIMABLE
)
1227 + global_page_state(NR_ACTIVE_ANON
)
1228 + global_page_state(NR_INACTIVE_ANON
)
1229 + global_page_state(NR_ACTIVE_FILE
)
1230 + global_page_state(NR_INACTIVE_FILE
)
1231 - global_page_state(NR_FILE_MAPPED
);
1233 return saveable
<= size
? 0 : saveable
- size
;
1237 * hibernate_preallocate_memory - Preallocate memory for hibernation image
1239 * To create a hibernation image it is necessary to make a copy of every page
1240 * frame in use. We also need a number of page frames to be free during
1241 * hibernation for allocations made while saving the image and for device
1242 * drivers, in case they need to allocate memory from their hibernation
1243 * callbacks (these two numbers are given by PAGES_FOR_IO and SPARE_PAGES,
1244 * respectively, both of which are rough estimates). To make this happen, we
1245 * compute the total number of available page frames and allocate at least
1247 * ([page frames total] + PAGES_FOR_IO + [metadata pages]) / 2 + 2 * SPARE_PAGES
1249 * of them, which corresponds to the maximum size of a hibernation image.
1251 * If image_size is set below the number following from the above formula,
1252 * the preallocation of memory is continued until the total number of saveable
1253 * pages in the system is below the requested image size or the minimum
1254 * acceptable image size returned by minimum_image_size(), whichever is greater.
1256 int hibernate_preallocate_memory(void)
1259 unsigned long saveable
, size
, max_size
, count
, highmem
, pages
= 0;
1260 unsigned long alloc
, save_highmem
, pages_highmem
;
1261 struct timeval start
, stop
;
1264 printk(KERN_INFO
"PM: Preallocating image memory... ");
1265 do_gettimeofday(&start
);
1267 error
= memory_bm_create(&orig_bm
, GFP_IMAGE
, PG_ANY
);
1271 error
= memory_bm_create(©_bm
, GFP_IMAGE
, PG_ANY
);
1278 /* Count the number of saveable data pages. */
1279 save_highmem
= count_highmem_pages();
1280 saveable
= count_data_pages();
1283 * Compute the total number of page frames we can use (count) and the
1284 * number of pages needed for image metadata (size).
1287 saveable
+= save_highmem
;
1288 highmem
= save_highmem
;
1290 for_each_populated_zone(zone
) {
1291 size
+= snapshot_additional_pages(zone
);
1292 if (is_highmem(zone
))
1293 highmem
+= zone_page_state(zone
, NR_FREE_PAGES
);
1295 count
+= zone_page_state(zone
, NR_FREE_PAGES
);
1298 count
-= totalreserve_pages
;
1300 /* Compute the maximum number of saveable pages to leave in memory. */
1301 max_size
= (count
- (size
+ PAGES_FOR_IO
)) / 2 - 2 * SPARE_PAGES
;
1302 size
= DIV_ROUND_UP(image_size
, PAGE_SIZE
);
1303 if (size
> max_size
)
1306 * If the maximum is not less than the current number of saveable pages
1307 * in memory, allocate page frames for the image and we're done.
1309 if (size
>= saveable
) {
1310 pages
= preallocate_image_highmem(save_highmem
);
1311 pages
+= preallocate_image_memory(saveable
- pages
);
1315 /* Estimate the minimum size of the image. */
1316 pages
= minimum_image_size(saveable
);
1318 size
= min_t(unsigned long, pages
, max_size
);
1321 * Let the memory management subsystem know that we're going to need a
1322 * large number of page frames to allocate and make it free some memory.
1323 * NOTE: If this is not done, performance will be hurt badly in some
1326 shrink_all_memory(saveable
- size
);
1329 * The number of saveable pages in memory was too high, so apply some
1330 * pressure to decrease it. First, make room for the largest possible
1331 * image and fail if that doesn't work. Next, try to decrease the size
1332 * of the image as much as indicated by 'size' using allocations from
1333 * highmem and non-highmem zones separately.
1335 pages_highmem
= preallocate_image_highmem(highmem
/ 2);
1336 alloc
= (count
- max_size
) - pages_highmem
;
1337 pages
= preallocate_image_memory(alloc
);
1340 size
= max_size
- size
;
1342 size
= preallocate_highmem_fraction(size
, highmem
, count
);
1343 pages_highmem
+= size
;
1345 pages
+= preallocate_image_memory(alloc
);
1346 pages
+= pages_highmem
;
1349 * We only need as many page frames for the image as there are saveable
1350 * pages in memory, but we have allocated more. Release the excessive
1353 free_unnecessary_pages();
1356 do_gettimeofday(&stop
);
1357 printk(KERN_CONT
"done (allocated %lu pages)\n", pages
);
1358 swsusp_show_speed(&start
, &stop
, pages
, "Allocated");
1363 printk(KERN_CONT
"\n");
1368 #ifdef CONFIG_HIGHMEM
1370 * count_pages_for_highmem - compute the number of non-highmem pages
1371 * that will be necessary for creating copies of highmem pages.
1374 static unsigned int count_pages_for_highmem(unsigned int nr_highmem
)
1376 unsigned int free_highmem
= count_free_highmem_pages() + alloc_highmem
;
1378 if (free_highmem
>= nr_highmem
)
1381 nr_highmem
-= free_highmem
;
1387 count_pages_for_highmem(unsigned int nr_highmem
) { return 0; }
1388 #endif /* CONFIG_HIGHMEM */
1391 * enough_free_mem - Make sure we have enough free memory for the
1395 static int enough_free_mem(unsigned int nr_pages
, unsigned int nr_highmem
)
1398 unsigned int free
= alloc_normal
;
1400 for_each_populated_zone(zone
)
1401 if (!is_highmem(zone
))
1402 free
+= zone_page_state(zone
, NR_FREE_PAGES
);
1404 nr_pages
+= count_pages_for_highmem(nr_highmem
);
1405 pr_debug("PM: Normal pages needed: %u + %u, available pages: %u\n",
1406 nr_pages
, PAGES_FOR_IO
, free
);
1408 return free
> nr_pages
+ PAGES_FOR_IO
;
1411 #ifdef CONFIG_HIGHMEM
1413 * get_highmem_buffer - if there are some highmem pages in the suspend
1414 * image, we may need the buffer to copy them and/or load their data.
1417 static inline int get_highmem_buffer(int safe_needed
)
1419 buffer
= get_image_page(GFP_ATOMIC
| __GFP_COLD
, safe_needed
);
1420 return buffer
? 0 : -ENOMEM
;
1424 * alloc_highmem_image_pages - allocate some highmem pages for the image.
1425 * Try to allocate as many pages as needed, but if the number of free
1426 * highmem pages is lesser than that, allocate them all.
1429 static inline unsigned int
1430 alloc_highmem_pages(struct memory_bitmap
*bm
, unsigned int nr_highmem
)
1432 unsigned int to_alloc
= count_free_highmem_pages();
1434 if (to_alloc
> nr_highmem
)
1435 to_alloc
= nr_highmem
;
1437 nr_highmem
-= to_alloc
;
1438 while (to_alloc
-- > 0) {
1441 page
= alloc_image_page(__GFP_HIGHMEM
);
1442 memory_bm_set_bit(bm
, page_to_pfn(page
));
1447 static inline int get_highmem_buffer(int safe_needed
) { return 0; }
1449 static inline unsigned int
1450 alloc_highmem_pages(struct memory_bitmap
*bm
, unsigned int n
) { return 0; }
1451 #endif /* CONFIG_HIGHMEM */
1454 * swsusp_alloc - allocate memory for the suspend image
1456 * We first try to allocate as many highmem pages as there are
1457 * saveable highmem pages in the system. If that fails, we allocate
1458 * non-highmem pages for the copies of the remaining highmem ones.
1460 * In this approach it is likely that the copies of highmem pages will
1461 * also be located in the high memory, because of the way in which
1462 * copy_data_pages() works.
1466 swsusp_alloc(struct memory_bitmap
*orig_bm
, struct memory_bitmap
*copy_bm
,
1467 unsigned int nr_pages
, unsigned int nr_highmem
)
1469 if (nr_highmem
> 0) {
1470 if (get_highmem_buffer(PG_ANY
))
1472 if (nr_highmem
> alloc_highmem
) {
1473 nr_highmem
-= alloc_highmem
;
1474 nr_pages
+= alloc_highmem_pages(copy_bm
, nr_highmem
);
1477 if (nr_pages
> alloc_normal
) {
1478 nr_pages
-= alloc_normal
;
1479 while (nr_pages
-- > 0) {
1482 page
= alloc_image_page(GFP_ATOMIC
| __GFP_COLD
);
1485 memory_bm_set_bit(copy_bm
, page_to_pfn(page
));
1496 asmlinkage
int swsusp_save(void)
1498 unsigned int nr_pages
, nr_highmem
;
1500 printk(KERN_INFO
"PM: Creating hibernation image: \n");
1502 drain_local_pages(NULL
);
1503 nr_pages
= count_data_pages();
1504 nr_highmem
= count_highmem_pages();
1505 printk(KERN_INFO
"PM: Need to copy %u pages\n", nr_pages
+ nr_highmem
);
1507 if (!enough_free_mem(nr_pages
, nr_highmem
)) {
1508 printk(KERN_ERR
"PM: Not enough free memory\n");
1512 if (swsusp_alloc(&orig_bm
, ©_bm
, nr_pages
, nr_highmem
)) {
1513 printk(KERN_ERR
"PM: Memory allocation failed\n");
1517 /* During allocating of suspend pagedir, new cold pages may appear.
1520 drain_local_pages(NULL
);
1521 copy_data_pages(©_bm
, &orig_bm
);
1524 * End of critical section. From now on, we can write to memory,
1525 * but we should not touch disk. This specially means we must _not_
1526 * touch swap space! Except we must write out our image of course.
1529 nr_pages
+= nr_highmem
;
1530 nr_copy_pages
= nr_pages
;
1531 nr_meta_pages
= DIV_ROUND_UP(nr_pages
* sizeof(long), PAGE_SIZE
);
1533 printk(KERN_INFO
"PM: Hibernation image created (%d pages copied)\n",
1539 #ifndef CONFIG_ARCH_HIBERNATION_HEADER
1540 static int init_header_complete(struct swsusp_info
*info
)
1542 memcpy(&info
->uts
, init_utsname(), sizeof(struct new_utsname
));
1543 info
->version_code
= LINUX_VERSION_CODE
;
1547 static char *check_image_kernel(struct swsusp_info
*info
)
1549 if (info
->version_code
!= LINUX_VERSION_CODE
)
1550 return "kernel version";
1551 if (strcmp(info
->uts
.sysname
,init_utsname()->sysname
))
1552 return "system type";
1553 if (strcmp(info
->uts
.release
,init_utsname()->release
))
1554 return "kernel release";
1555 if (strcmp(info
->uts
.version
,init_utsname()->version
))
1557 if (strcmp(info
->uts
.machine
,init_utsname()->machine
))
1561 #endif /* CONFIG_ARCH_HIBERNATION_HEADER */
1563 unsigned long snapshot_get_image_size(void)
1565 return nr_copy_pages
+ nr_meta_pages
+ 1;
1568 static int init_header(struct swsusp_info
*info
)
1570 memset(info
, 0, sizeof(struct swsusp_info
));
1571 info
->num_physpages
= num_physpages
;
1572 info
->image_pages
= nr_copy_pages
;
1573 info
->pages
= snapshot_get_image_size();
1574 info
->size
= info
->pages
;
1575 info
->size
<<= PAGE_SHIFT
;
1576 return init_header_complete(info
);
1580 * pack_pfns - pfns corresponding to the set bits found in the bitmap @bm
1581 * are stored in the array @buf[] (1 page at a time)
1585 pack_pfns(unsigned long *buf
, struct memory_bitmap
*bm
)
1589 for (j
= 0; j
< PAGE_SIZE
/ sizeof(long); j
++) {
1590 buf
[j
] = memory_bm_next_pfn(bm
);
1591 if (unlikely(buf
[j
] == BM_END_OF_MAP
))
1597 * snapshot_read_next - used for reading the system memory snapshot.
1599 * On the first call to it @handle should point to a zeroed
1600 * snapshot_handle structure. The structure gets updated and a pointer
1601 * to it should be passed to this function every next time.
1603 * The @count parameter should contain the number of bytes the caller
1604 * wants to read from the snapshot. It must not be zero.
1606 * On success the function returns a positive number. Then, the caller
1607 * is allowed to read up to the returned number of bytes from the memory
1608 * location computed by the data_of() macro. The number returned
1609 * may be smaller than @count, but this only happens if the read would
1610 * cross a page boundary otherwise.
1612 * The function returns 0 to indicate the end of data stream condition,
1613 * and a negative number is returned on error. In such cases the
1614 * structure pointed to by @handle is not updated and should not be used
1618 int snapshot_read_next(struct snapshot_handle
*handle
, size_t count
)
1620 if (handle
->cur
> nr_meta_pages
+ nr_copy_pages
)
1624 /* This makes the buffer be freed by swsusp_free() */
1625 buffer
= get_image_page(GFP_ATOMIC
, PG_ANY
);
1629 if (!handle
->offset
) {
1632 error
= init_header((struct swsusp_info
*)buffer
);
1635 handle
->buffer
= buffer
;
1636 memory_bm_position_reset(&orig_bm
);
1637 memory_bm_position_reset(©_bm
);
1639 if (handle
->prev
< handle
->cur
) {
1640 if (handle
->cur
<= nr_meta_pages
) {
1641 memset(buffer
, 0, PAGE_SIZE
);
1642 pack_pfns(buffer
, &orig_bm
);
1646 page
= pfn_to_page(memory_bm_next_pfn(©_bm
));
1647 if (PageHighMem(page
)) {
1648 /* Highmem pages are copied to the buffer,
1649 * because we can't return with a kmapped
1650 * highmem page (we may not be called again).
1654 kaddr
= kmap_atomic(page
, KM_USER0
);
1655 memcpy(buffer
, kaddr
, PAGE_SIZE
);
1656 kunmap_atomic(kaddr
, KM_USER0
);
1657 handle
->buffer
= buffer
;
1659 handle
->buffer
= page_address(page
);
1662 handle
->prev
= handle
->cur
;
1664 handle
->buf_offset
= handle
->cur_offset
;
1665 if (handle
->cur_offset
+ count
>= PAGE_SIZE
) {
1666 count
= PAGE_SIZE
- handle
->cur_offset
;
1667 handle
->cur_offset
= 0;
1670 handle
->cur_offset
+= count
;
1672 handle
->offset
+= count
;
1677 * mark_unsafe_pages - mark the pages that cannot be used for storing
1678 * the image during resume, because they conflict with the pages that
1679 * had been used before suspend
1682 static int mark_unsafe_pages(struct memory_bitmap
*bm
)
1685 unsigned long pfn
, max_zone_pfn
;
1687 /* Clear page flags */
1688 for_each_populated_zone(zone
) {
1689 max_zone_pfn
= zone
->zone_start_pfn
+ zone
->spanned_pages
;
1690 for (pfn
= zone
->zone_start_pfn
; pfn
< max_zone_pfn
; pfn
++)
1692 swsusp_unset_page_free(pfn_to_page(pfn
));
1695 /* Mark pages that correspond to the "original" pfns as "unsafe" */
1696 memory_bm_position_reset(bm
);
1698 pfn
= memory_bm_next_pfn(bm
);
1699 if (likely(pfn
!= BM_END_OF_MAP
)) {
1700 if (likely(pfn_valid(pfn
)))
1701 swsusp_set_page_free(pfn_to_page(pfn
));
1705 } while (pfn
!= BM_END_OF_MAP
);
1707 allocated_unsafe_pages
= 0;
1713 duplicate_memory_bitmap(struct memory_bitmap
*dst
, struct memory_bitmap
*src
)
1717 memory_bm_position_reset(src
);
1718 pfn
= memory_bm_next_pfn(src
);
1719 while (pfn
!= BM_END_OF_MAP
) {
1720 memory_bm_set_bit(dst
, pfn
);
1721 pfn
= memory_bm_next_pfn(src
);
1725 static int check_header(struct swsusp_info
*info
)
1729 reason
= check_image_kernel(info
);
1730 if (!reason
&& info
->num_physpages
!= num_physpages
)
1731 reason
= "memory size";
1733 printk(KERN_ERR
"PM: Image mismatch: %s\n", reason
);
1740 * load header - check the image header and copy data from it
1744 load_header(struct swsusp_info
*info
)
1748 restore_pblist
= NULL
;
1749 error
= check_header(info
);
1751 nr_copy_pages
= info
->image_pages
;
1752 nr_meta_pages
= info
->pages
- info
->image_pages
- 1;
1758 * unpack_orig_pfns - for each element of @buf[] (1 page at a time) set
1759 * the corresponding bit in the memory bitmap @bm
1761 static int unpack_orig_pfns(unsigned long *buf
, struct memory_bitmap
*bm
)
1765 for (j
= 0; j
< PAGE_SIZE
/ sizeof(long); j
++) {
1766 if (unlikely(buf
[j
] == BM_END_OF_MAP
))
1769 if (memory_bm_pfn_present(bm
, buf
[j
]))
1770 memory_bm_set_bit(bm
, buf
[j
]);
1778 /* List of "safe" pages that may be used to store data loaded from the suspend
1781 static struct linked_page
*safe_pages_list
;
1783 #ifdef CONFIG_HIGHMEM
1784 /* struct highmem_pbe is used for creating the list of highmem pages that
1785 * should be restored atomically during the resume from disk, because the page
1786 * frames they have occupied before the suspend are in use.
1788 struct highmem_pbe
{
1789 struct page
*copy_page
; /* data is here now */
1790 struct page
*orig_page
; /* data was here before the suspend */
1791 struct highmem_pbe
*next
;
1794 /* List of highmem PBEs needed for restoring the highmem pages that were
1795 * allocated before the suspend and included in the suspend image, but have
1796 * also been allocated by the "resume" kernel, so their contents cannot be
1797 * written directly to their "original" page frames.
1799 static struct highmem_pbe
*highmem_pblist
;
1802 * count_highmem_image_pages - compute the number of highmem pages in the
1803 * suspend image. The bits in the memory bitmap @bm that correspond to the
1804 * image pages are assumed to be set.
1807 static unsigned int count_highmem_image_pages(struct memory_bitmap
*bm
)
1810 unsigned int cnt
= 0;
1812 memory_bm_position_reset(bm
);
1813 pfn
= memory_bm_next_pfn(bm
);
1814 while (pfn
!= BM_END_OF_MAP
) {
1815 if (PageHighMem(pfn_to_page(pfn
)))
1818 pfn
= memory_bm_next_pfn(bm
);
1824 * prepare_highmem_image - try to allocate as many highmem pages as
1825 * there are highmem image pages (@nr_highmem_p points to the variable
1826 * containing the number of highmem image pages). The pages that are
1827 * "safe" (ie. will not be overwritten when the suspend image is
1828 * restored) have the corresponding bits set in @bm (it must be
1831 * NOTE: This function should not be called if there are no highmem
1835 static unsigned int safe_highmem_pages
;
1837 static struct memory_bitmap
*safe_highmem_bm
;
1840 prepare_highmem_image(struct memory_bitmap
*bm
, unsigned int *nr_highmem_p
)
1842 unsigned int to_alloc
;
1844 if (memory_bm_create(bm
, GFP_ATOMIC
, PG_SAFE
))
1847 if (get_highmem_buffer(PG_SAFE
))
1850 to_alloc
= count_free_highmem_pages();
1851 if (to_alloc
> *nr_highmem_p
)
1852 to_alloc
= *nr_highmem_p
;
1854 *nr_highmem_p
= to_alloc
;
1856 safe_highmem_pages
= 0;
1857 while (to_alloc
-- > 0) {
1860 page
= alloc_page(__GFP_HIGHMEM
);
1861 if (!swsusp_page_is_free(page
)) {
1862 /* The page is "safe", set its bit the bitmap */
1863 memory_bm_set_bit(bm
, page_to_pfn(page
));
1864 safe_highmem_pages
++;
1866 /* Mark the page as allocated */
1867 swsusp_set_page_forbidden(page
);
1868 swsusp_set_page_free(page
);
1870 memory_bm_position_reset(bm
);
1871 safe_highmem_bm
= bm
;
1876 * get_highmem_page_buffer - for given highmem image page find the buffer
1877 * that suspend_write_next() should set for its caller to write to.
1879 * If the page is to be saved to its "original" page frame or a copy of
1880 * the page is to be made in the highmem, @buffer is returned. Otherwise,
1881 * the copy of the page is to be made in normal memory, so the address of
1882 * the copy is returned.
1884 * If @buffer is returned, the caller of suspend_write_next() will write
1885 * the page's contents to @buffer, so they will have to be copied to the
1886 * right location on the next call to suspend_write_next() and it is done
1887 * with the help of copy_last_highmem_page(). For this purpose, if
1888 * @buffer is returned, @last_highmem page is set to the page to which
1889 * the data will have to be copied from @buffer.
1892 static struct page
*last_highmem_page
;
1895 get_highmem_page_buffer(struct page
*page
, struct chain_allocator
*ca
)
1897 struct highmem_pbe
*pbe
;
1900 if (swsusp_page_is_forbidden(page
) && swsusp_page_is_free(page
)) {
1901 /* We have allocated the "original" page frame and we can
1902 * use it directly to store the loaded page.
1904 last_highmem_page
= page
;
1907 /* The "original" page frame has not been allocated and we have to
1908 * use a "safe" page frame to store the loaded page.
1910 pbe
= chain_alloc(ca
, sizeof(struct highmem_pbe
));
1913 return ERR_PTR(-ENOMEM
);
1915 pbe
->orig_page
= page
;
1916 if (safe_highmem_pages
> 0) {
1919 /* Copy of the page will be stored in high memory */
1921 tmp
= pfn_to_page(memory_bm_next_pfn(safe_highmem_bm
));
1922 safe_highmem_pages
--;
1923 last_highmem_page
= tmp
;
1924 pbe
->copy_page
= tmp
;
1926 /* Copy of the page will be stored in normal memory */
1927 kaddr
= safe_pages_list
;
1928 safe_pages_list
= safe_pages_list
->next
;
1929 pbe
->copy_page
= virt_to_page(kaddr
);
1931 pbe
->next
= highmem_pblist
;
1932 highmem_pblist
= pbe
;
1937 * copy_last_highmem_page - copy the contents of a highmem image from
1938 * @buffer, where the caller of snapshot_write_next() has place them,
1939 * to the right location represented by @last_highmem_page .
1942 static void copy_last_highmem_page(void)
1944 if (last_highmem_page
) {
1947 dst
= kmap_atomic(last_highmem_page
, KM_USER0
);
1948 memcpy(dst
, buffer
, PAGE_SIZE
);
1949 kunmap_atomic(dst
, KM_USER0
);
1950 last_highmem_page
= NULL
;
1954 static inline int last_highmem_page_copied(void)
1956 return !last_highmem_page
;
1959 static inline void free_highmem_data(void)
1961 if (safe_highmem_bm
)
1962 memory_bm_free(safe_highmem_bm
, PG_UNSAFE_CLEAR
);
1965 free_image_page(buffer
, PG_UNSAFE_CLEAR
);
1968 static inline int get_safe_write_buffer(void) { return 0; }
1971 count_highmem_image_pages(struct memory_bitmap
*bm
) { return 0; }
1974 prepare_highmem_image(struct memory_bitmap
*bm
, unsigned int *nr_highmem_p
)
1979 static inline void *
1980 get_highmem_page_buffer(struct page
*page
, struct chain_allocator
*ca
)
1982 return ERR_PTR(-EINVAL
);
1985 static inline void copy_last_highmem_page(void) {}
1986 static inline int last_highmem_page_copied(void) { return 1; }
1987 static inline void free_highmem_data(void) {}
1988 #endif /* CONFIG_HIGHMEM */
1991 * prepare_image - use the memory bitmap @bm to mark the pages that will
1992 * be overwritten in the process of restoring the system memory state
1993 * from the suspend image ("unsafe" pages) and allocate memory for the
1996 * The idea is to allocate a new memory bitmap first and then allocate
1997 * as many pages as needed for the image data, but not to assign these
1998 * pages to specific tasks initially. Instead, we just mark them as
1999 * allocated and create a lists of "safe" pages that will be used
2000 * later. On systems with high memory a list of "safe" highmem pages is
2004 #define PBES_PER_LINKED_PAGE (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))
2007 prepare_image(struct memory_bitmap
*new_bm
, struct memory_bitmap
*bm
)
2009 unsigned int nr_pages
, nr_highmem
;
2010 struct linked_page
*sp_list
, *lp
;
2013 /* If there is no highmem, the buffer will not be necessary */
2014 free_image_page(buffer
, PG_UNSAFE_CLEAR
);
2017 nr_highmem
= count_highmem_image_pages(bm
);
2018 error
= mark_unsafe_pages(bm
);
2022 error
= memory_bm_create(new_bm
, GFP_ATOMIC
, PG_SAFE
);
2026 duplicate_memory_bitmap(new_bm
, bm
);
2027 memory_bm_free(bm
, PG_UNSAFE_KEEP
);
2028 if (nr_highmem
> 0) {
2029 error
= prepare_highmem_image(bm
, &nr_highmem
);
2033 /* Reserve some safe pages for potential later use.
2035 * NOTE: This way we make sure there will be enough safe pages for the
2036 * chain_alloc() in get_buffer(). It is a bit wasteful, but
2037 * nr_copy_pages cannot be greater than 50% of the memory anyway.
2040 /* nr_copy_pages cannot be lesser than allocated_unsafe_pages */
2041 nr_pages
= nr_copy_pages
- nr_highmem
- allocated_unsafe_pages
;
2042 nr_pages
= DIV_ROUND_UP(nr_pages
, PBES_PER_LINKED_PAGE
);
2043 while (nr_pages
> 0) {
2044 lp
= get_image_page(GFP_ATOMIC
, PG_SAFE
);
2053 /* Preallocate memory for the image */
2054 safe_pages_list
= NULL
;
2055 nr_pages
= nr_copy_pages
- nr_highmem
- allocated_unsafe_pages
;
2056 while (nr_pages
> 0) {
2057 lp
= (struct linked_page
*)get_zeroed_page(GFP_ATOMIC
);
2062 if (!swsusp_page_is_free(virt_to_page(lp
))) {
2063 /* The page is "safe", add it to the list */
2064 lp
->next
= safe_pages_list
;
2065 safe_pages_list
= lp
;
2067 /* Mark the page as allocated */
2068 swsusp_set_page_forbidden(virt_to_page(lp
));
2069 swsusp_set_page_free(virt_to_page(lp
));
2072 /* Free the reserved safe pages so that chain_alloc() can use them */
2075 free_image_page(sp_list
, PG_UNSAFE_CLEAR
);
2086 * get_buffer - compute the address that snapshot_write_next() should
2087 * set for its caller to write to.
2090 static void *get_buffer(struct memory_bitmap
*bm
, struct chain_allocator
*ca
)
2094 unsigned long pfn
= memory_bm_next_pfn(bm
);
2096 if (pfn
== BM_END_OF_MAP
)
2097 return ERR_PTR(-EFAULT
);
2099 page
= pfn_to_page(pfn
);
2100 if (PageHighMem(page
))
2101 return get_highmem_page_buffer(page
, ca
);
2103 if (swsusp_page_is_forbidden(page
) && swsusp_page_is_free(page
))
2104 /* We have allocated the "original" page frame and we can
2105 * use it directly to store the loaded page.
2107 return page_address(page
);
2109 /* The "original" page frame has not been allocated and we have to
2110 * use a "safe" page frame to store the loaded page.
2112 pbe
= chain_alloc(ca
, sizeof(struct pbe
));
2115 return ERR_PTR(-ENOMEM
);
2117 pbe
->orig_address
= page_address(page
);
2118 pbe
->address
= safe_pages_list
;
2119 safe_pages_list
= safe_pages_list
->next
;
2120 pbe
->next
= restore_pblist
;
2121 restore_pblist
= pbe
;
2122 return pbe
->address
;
2126 * snapshot_write_next - used for writing the system memory snapshot.
2128 * On the first call to it @handle should point to a zeroed
2129 * snapshot_handle structure. The structure gets updated and a pointer
2130 * to it should be passed to this function every next time.
2132 * The @count parameter should contain the number of bytes the caller
2133 * wants to write to the image. It must not be zero.
2135 * On success the function returns a positive number. Then, the caller
2136 * is allowed to write up to the returned number of bytes to the memory
2137 * location computed by the data_of() macro. The number returned
2138 * may be smaller than @count, but this only happens if the write would
2139 * cross a page boundary otherwise.
2141 * The function returns 0 to indicate the "end of file" condition,
2142 * and a negative number is returned on error. In such cases the
2143 * structure pointed to by @handle is not updated and should not be used
2147 int snapshot_write_next(struct snapshot_handle
*handle
, size_t count
)
2149 static struct chain_allocator ca
;
2152 /* Check if we have already loaded the entire image */
2153 if (handle
->prev
&& handle
->cur
> nr_meta_pages
+ nr_copy_pages
)
2156 if (handle
->offset
== 0) {
2158 /* This makes the buffer be freed by swsusp_free() */
2159 buffer
= get_image_page(GFP_ATOMIC
, PG_ANY
);
2164 handle
->buffer
= buffer
;
2166 handle
->sync_read
= 1;
2167 if (handle
->prev
< handle
->cur
) {
2168 if (handle
->prev
== 0) {
2169 error
= load_header(buffer
);
2173 error
= memory_bm_create(©_bm
, GFP_ATOMIC
, PG_ANY
);
2177 } else if (handle
->prev
<= nr_meta_pages
) {
2178 error
= unpack_orig_pfns(buffer
, ©_bm
);
2182 if (handle
->prev
== nr_meta_pages
) {
2183 error
= prepare_image(&orig_bm
, ©_bm
);
2187 chain_init(&ca
, GFP_ATOMIC
, PG_SAFE
);
2188 memory_bm_position_reset(&orig_bm
);
2189 restore_pblist
= NULL
;
2190 handle
->buffer
= get_buffer(&orig_bm
, &ca
);
2191 handle
->sync_read
= 0;
2192 if (IS_ERR(handle
->buffer
))
2193 return PTR_ERR(handle
->buffer
);
2196 copy_last_highmem_page();
2197 handle
->buffer
= get_buffer(&orig_bm
, &ca
);
2198 if (IS_ERR(handle
->buffer
))
2199 return PTR_ERR(handle
->buffer
);
2200 if (handle
->buffer
!= buffer
)
2201 handle
->sync_read
= 0;
2203 handle
->prev
= handle
->cur
;
2205 handle
->buf_offset
= handle
->cur_offset
;
2206 if (handle
->cur_offset
+ count
>= PAGE_SIZE
) {
2207 count
= PAGE_SIZE
- handle
->cur_offset
;
2208 handle
->cur_offset
= 0;
2211 handle
->cur_offset
+= count
;
2213 handle
->offset
+= count
;
2218 * snapshot_write_finalize - must be called after the last call to
2219 * snapshot_write_next() in case the last page in the image happens
2220 * to be a highmem page and its contents should be stored in the
2221 * highmem. Additionally, it releases the memory that will not be
2225 void snapshot_write_finalize(struct snapshot_handle
*handle
)
2227 copy_last_highmem_page();
2228 /* Free only if we have loaded the image entirely */
2229 if (handle
->prev
&& handle
->cur
> nr_meta_pages
+ nr_copy_pages
) {
2230 memory_bm_free(&orig_bm
, PG_UNSAFE_CLEAR
);
2231 free_highmem_data();
2235 int snapshot_image_loaded(struct snapshot_handle
*handle
)
2237 return !(!nr_copy_pages
|| !last_highmem_page_copied() ||
2238 handle
->cur
<= nr_meta_pages
+ nr_copy_pages
);
2241 #ifdef CONFIG_HIGHMEM
2242 /* Assumes that @buf is ready and points to a "safe" page */
2244 swap_two_pages_data(struct page
*p1
, struct page
*p2
, void *buf
)
2246 void *kaddr1
, *kaddr2
;
2248 kaddr1
= kmap_atomic(p1
, KM_USER0
);
2249 kaddr2
= kmap_atomic(p2
, KM_USER1
);
2250 memcpy(buf
, kaddr1
, PAGE_SIZE
);
2251 memcpy(kaddr1
, kaddr2
, PAGE_SIZE
);
2252 memcpy(kaddr2
, buf
, PAGE_SIZE
);
2253 kunmap_atomic(kaddr1
, KM_USER0
);
2254 kunmap_atomic(kaddr2
, KM_USER1
);
2258 * restore_highmem - for each highmem page that was allocated before
2259 * the suspend and included in the suspend image, and also has been
2260 * allocated by the "resume" kernel swap its current (ie. "before
2261 * resume") contents with the previous (ie. "before suspend") one.
2263 * If the resume eventually fails, we can call this function once
2264 * again and restore the "before resume" highmem state.
2267 int restore_highmem(void)
2269 struct highmem_pbe
*pbe
= highmem_pblist
;
2275 buf
= get_image_page(GFP_ATOMIC
, PG_SAFE
);
2280 swap_two_pages_data(pbe
->copy_page
, pbe
->orig_page
, buf
);
2283 free_image_page(buf
, PG_UNSAFE_CLEAR
);
2286 #endif /* CONFIG_HIGHMEM */