2 * kernel/power/prepare_image.c
4 * Copyright (C) 2003-2007 Nigel Cunningham (nigel at suspend2 net)
6 * This file is released under the GPLv2.
8 * We need to eat memory until we can:
9 * 1. Perform the save without changing anything (RAM_NEEDED < #pages)
10 * 2. Fit it all in available space (suspendActiveAllocator->available_space() >=
11 * main_storage_needed())
12 * 3. Reload the pagedir and pageset1 to places that don't collide with their
13 * final destinations, not knowing to what extent the resumed kernel will
14 * overlap with the one loaded at boot time. I think the resumed kernel
15 * should overlap completely, but I don't want to rely on this as it is
16 * an unproven assumption. We therefore assume there will be no overlap at
18 * 4. Meet the user's requested limit (if any) on the size of the image.
19 * The limit is in MB, so pages/256 (assuming 4K pages).
23 #include <linux/module.h>
24 #include <linux/highmem.h>
25 #include <linux/freezer.h>
26 #include <linux/hardirq.h>
28 #include "pageflags.h"
33 #include "prepare_image.h"
39 static int num_nosave
= 0;
40 static int header_space_allocated
= 0;
41 static int main_storage_allocated
= 0;
42 static int storage_available
= 0;
43 int extra_pd1_pages_allowance
= MIN_EXTRA_PAGES_ALLOWANCE
;
44 int image_size_limit
= 0;
47 * The atomic copy of pageset1 is stored in pageset2 pages.
48 * But if pageset1 is larger (normally only just after boot),
49 * we need to allocate extra pages to store the atomic copy.
50 * The following data struct and functions are used to handle
51 * the allocation and freeing of that memory.
54 static int extra_pages_allocated
;
62 static struct extras
*extras_list
;
64 /* suspend_free_extra_pagedir_memory
66 * Description: Free previously allocated extra pagedir memory.
68 void suspend_free_extra_pagedir_memory(void)
70 /* Free allocated pages */
72 struct extras
*this = extras_list
;
75 extras_list
= this->next
;
77 for (i
= 0; i
< (1 << this->order
); i
++)
78 ClearPageNosave(this->page
+ i
);
80 __free_pages(this->page
, this->order
);
84 extra_pages_allocated
= 0;
87 /* suspend_allocate_extra_pagedir_memory
89 * Description: Allocate memory for making the atomic copy of pagedir1 in the
90 * case where it is bigger than pagedir2.
91 * Arguments: int num_to_alloc: Number of extra pages needed.
92 * Result: int. Number of extra pages we now have allocated.
94 static int suspend_allocate_extra_pagedir_memory(int extra_pages_needed
)
96 int j
, order
, num_to_alloc
= extra_pages_needed
- extra_pages_allocated
;
97 unsigned long flags
= GFP_ATOMIC
| __GFP_NOWARN
;
102 order
= fls(num_to_alloc
);
103 if (order
>= MAX_ORDER
)
104 order
= MAX_ORDER
- 1;
106 while (num_to_alloc
) {
107 struct page
*newpage
;
109 struct extras
*extras_entry
;
111 while ((1 << order
) > num_to_alloc
)
114 extras_entry
= (struct extras
*) kmalloc(sizeof(struct extras
),
118 return extra_pages_allocated
;
120 virt
= __get_free_pages(flags
, order
);
121 while (!virt
&& order
) {
123 virt
= __get_free_pages(flags
, order
);
128 return extra_pages_allocated
;
131 newpage
= virt_to_page(virt
);
133 extras_entry
->page
= newpage
;
134 extras_entry
->order
= order
;
135 extras_entry
->next
= NULL
;
138 extras_entry
->next
= extras_list
;
140 extras_list
= extras_entry
;
142 for (j
= 0; j
< (1 << order
); j
++) {
143 SetPageNosave(newpage
+ j
);
144 SetPagePageset1Copy(newpage
+ j
);
147 extra_pages_allocated
+= (1 << order
);
148 num_to_alloc
-= (1 << order
);
151 return extra_pages_allocated
;
155 * real_nr_free_pages: Count pcp pages for a zone type or all zones
156 * (-1 for all, otherwise zone_idx() result desired).
158 int real_nr_free_pages(unsigned long zone_idx_mask
)
161 int result
= 0, i
= 0, cpu
;
164 for_each_zone(zone
) {
165 if (!populated_zone(zone
))
168 if (!(zone_idx_mask
& (1 << zone_idx(zone
))))
171 for_each_online_cpu(cpu
) {
172 struct per_cpu_pageset
*pset
= zone_pcp(zone
, cpu
);
174 for (i
= 0; i
< ARRAY_SIZE(pset
->pcp
); i
++) {
175 struct per_cpu_pages
*pcp
;
178 result
+= pcp
->count
;
182 result
+= zone
->free_pages
;
188 * Discover how much extra memory will be required by the drivers
189 * when they're asked to suspend. We can then ensure that amount
190 * of memory is available when we really want it.
192 static void get_extra_pd1_allowance(void)
194 int orig_num_free
= real_nr_free_pages(all_zones_mask
), final
;
196 suspend_prepare_status(CLEAR_BAR
, "Finding allowance for drivers.");
198 device_suspend(PMSG_FREEZE
);
199 local_irq_disable(); /* irqs might have been re-enabled on us */
200 device_power_down(PMSG_FREEZE
);
202 final
= real_nr_free_pages(all_zones_mask
);
208 extra_pd1_pages_allowance
= max(
209 orig_num_free
- final
+ MIN_EXTRA_PAGES_ALLOWANCE
,
210 MIN_EXTRA_PAGES_ALLOWANCE
);
214 * Amount of storage needed, possibly taking into account the
215 * expected compression ratio and possibly also ignoring our
216 * allowance for extra pages.
218 static int main_storage_needed(int use_ecr
,
219 int ignore_extra_pd1_allow
)
221 return ((pagedir1
.size
+ pagedir2
.size
+
222 (ignore_extra_pd1_allow
? 0 : extra_pd1_pages_allowance
)) *
223 (use_ecr
? suspend_expected_compression_ratio() : 100) / 100);
227 * Storage needed for the image header, in bytes until the return.
229 static int header_storage_needed(void)
231 int bytes
= (int) sizeof(struct suspend_header
) +
232 suspend_header_storage_for_modules() +
233 suspend_pageflags_space_needed();
235 return DIV_ROUND_UP(bytes
, PAGE_SIZE
);
239 * When freeing memory, pages from either pageset might be freed.
241 * When seeking to free memory to be able to suspend, for every ps1 page freed,
242 * we need 2 less pages for the atomic copy because there is one less page to
243 * copy and one more page into which data can be copied.
245 * Freeing ps2 pages saves us nothing directly. No more memory is available
246 * for the atomic copy. Indirectly, a ps1 page might be freed (slab?), but
247 * that's too much work to figure out.
249 * => ps1_to_free functions
251 * Of course if we just want to reduce the image size, because of storage
252 * limitations or an image size limit either ps will do.
254 * => any_to_free function
257 static int highpages_ps1_to_free(void)
259 return max_t(int, 0, DIV_ROUND_UP(get_highmem_size(pagedir1
) -
260 get_highmem_size(pagedir2
), 2) - real_nr_free_high_pages());
263 static int lowpages_ps1_to_free(void)
265 return max_t(int, 0, DIV_ROUND_UP(get_lowmem_size(pagedir1
) +
266 extra_pd1_pages_allowance
+ MIN_FREE_RAM
+
267 suspend_memory_for_modules() - get_lowmem_size(pagedir2
) -
268 real_nr_free_low_pages() - extra_pages_allocated
, 2));
271 static int current_image_size(void)
273 return pagedir1
.size
+ pagedir2
.size
+ header_space_allocated
;
276 static int any_to_free(int use_image_size_limit
)
278 int user_limit
= (use_image_size_limit
&& image_size_limit
> 0) ?
279 max_t(int, 0, current_image_size() - (image_size_limit
<< 8))
282 int storage_limit
= max_t(int, 0,
283 main_storage_needed(1, 1) - storage_available
);
285 return max(user_limit
, storage_limit
);
290 * Calculates the amount by which the image size needs to be reduced to meet
293 static int amount_needed(int use_image_size_limit
)
295 return max(highpages_ps1_to_free() + lowpages_ps1_to_free(),
296 any_to_free(use_image_size_limit
));
299 static int image_not_ready(int use_image_size_limit
)
301 suspend_message(SUSPEND_EAT_MEMORY
, SUSPEND_LOW
, 1,
302 "Amount still needed (%d) > 0:%d. Header: %d < %d: %d,"
303 " Storage allocd: %d < %d: %d.\n",
304 amount_needed(use_image_size_limit
),
305 (amount_needed(use_image_size_limit
) > 0),
306 header_space_allocated
, header_storage_needed(),
307 header_space_allocated
< header_storage_needed(),
308 main_storage_allocated
,
309 main_storage_needed(1, 1),
310 main_storage_allocated
< main_storage_needed(1, 1));
312 suspend_cond_pause(0, NULL
);
314 return ((amount_needed(use_image_size_limit
) > 0) ||
315 header_space_allocated
< header_storage_needed() ||
316 main_storage_allocated
< main_storage_needed(1, 1));
319 static void display_stats(int always
, int sub_extra_pd1_allow
)
322 snprintf(buffer
, 254,
323 "Free:%d(%d). Sets:%d(%d),%d(%d). Header:%d/%d. Nosave:%d-%d"
324 "=%d. Storage:%u/%u(%u=>%u). Needed:%d,%d,%d(%d,%d,%d,%d)\n",
327 real_nr_free_pages(all_zones_mask
),
328 real_nr_free_low_pages(),
331 pagedir1
.size
, pagedir1
.size
- get_highmem_size(pagedir1
),
332 pagedir2
.size
, pagedir2
.size
- get_highmem_size(pagedir2
),
335 header_space_allocated
, header_storage_needed(),
338 num_nosave
, extra_pages_allocated
,
339 num_nosave
- extra_pages_allocated
,
342 main_storage_allocated
,
344 main_storage_needed(1, sub_extra_pd1_allow
),
345 main_storage_needed(1, 1),
348 lowpages_ps1_to_free(), highpages_ps1_to_free(),
350 MIN_FREE_RAM
, suspend_memory_for_modules(),
351 extra_pd1_pages_allowance
, image_size_limit
<< 8);
356 suspend_message(SUSPEND_EAT_MEMORY
, SUSPEND_MEDIUM
, 1, buffer
);
359 /* generate_free_page_map
361 * Description: This routine generates a bitmap of free pages from the
362 * lists used by the memory manager. We then use the bitmap
363 * to quickly calculate which pages to save and in which
366 static void generate_free_page_map(void)
368 int order
, loop
, cpu
;
370 unsigned long flags
, i
;
373 for_each_zone(zone
) {
374 if (!populated_zone(zone
))
377 spin_lock_irqsave(&zone
->lock
, flags
);
379 for(i
=0; i
< zone
->spanned_pages
; i
++)
380 ClearPageNosaveFree(pfn_to_page(
381 zone
->zone_start_pfn
+ i
));
383 for (order
= MAX_ORDER
- 1; order
>= 0; --order
)
384 list_for_each_entry(page
,
385 &zone
->free_area
[order
].free_list
, lru
)
386 for(loop
=0; loop
< (1 << order
); loop
++)
387 SetPageNosaveFree(page
+loop
);
390 for_each_online_cpu(cpu
) {
391 struct per_cpu_pageset
*pset
= zone_pcp(zone
, cpu
);
393 for (i
= 0; i
< ARRAY_SIZE(pset
->pcp
); i
++) {
394 struct per_cpu_pages
*pcp
;
398 list_for_each_entry(page
, &pcp
->list
, lru
)
399 SetPageNosaveFree(page
);
403 spin_unlock_irqrestore(&zone
->lock
, flags
);
407 /* size_of_free_region
409 * Description: Return the number of pages that are free, beginning with and
410 * including this one.
412 static int size_of_free_region(struct page
*page
)
414 struct zone
*zone
= page_zone(page
);
415 struct page
*posn
= page
, *last_in_zone
=
416 pfn_to_page(zone
->zone_start_pfn
) + zone
->spanned_pages
- 1;
418 while (posn
<= last_in_zone
&& PageNosaveFree(posn
))
420 return (posn
- page
);
425 * This routine generates our lists of pages to be stored in each
426 * pageset. Since we store the data using extents, and adding new
427 * extents might allocate a new extent page, this routine may well
428 * be called more than once.
430 static void flag_image_pages(void)
439 set_highmem_size(pagedir1
, 0);
440 set_highmem_size(pagedir2
, 0);
444 clear_dyn_pageflags(pageset1_map
);
446 generate_free_page_map();
449 * Pages not to be saved are marked Nosave irrespective of being reserved
451 for_each_zone(zone
) {
452 if (!populated_zone(zone
))
455 for (loop
= 0; loop
< zone
->spanned_pages
; loop
++) {
456 unsigned long pfn
= zone
->zone_start_pfn
+ loop
;
463 page
= pfn_to_page(pfn
);
465 chunk_size
= size_of_free_region(page
);
467 num_free
+= chunk_size
;
468 loop
+= chunk_size
- 1;
472 if (!saveable(zone
, &loop
)) {
477 if (PagePageset2(page
)) {
479 if (PageHighMem(page
))
480 inc_highmem_size(pagedir2
);
482 SetPagePageset1Copy(page
);
483 if (PageResave(page
)) {
484 SetPagePageset1(page
);
485 ClearPagePageset1Copy(page
);
487 if (PageHighMem(page
))
488 inc_highmem_size(pagedir1
);
492 SetPagePageset1(page
);
493 if (PageHighMem(page
))
494 inc_highmem_size(pagedir1
);
499 suspend_message(SUSPEND_EAT_MEMORY
, SUSPEND_MEDIUM
, 0,
500 "Count data pages: Set1 (%d) + Set2 (%d) + Nosave (%d) + "
501 "NumFree (%d) = %d.\n",
502 pagedir1
.size
, pagedir2
.size
, num_nosave
, num_free
,
503 pagedir1
.size
+ pagedir2
.size
+ num_nosave
+ num_free
);
506 void suspend_recalculate_image_contents(int atomic_copy
)
508 clear_dyn_pageflags(pageset1_map
);
511 BITMAP_FOR_EACH_SET(pageset2_map
, pfn
)
512 ClearPagePageset1Copy(pfn_to_page(pfn
));
513 /* Need to call this before getting pageset1_size! */
514 suspend_mark_pages_for_pageset2();
519 storage_available
= suspendActiveAllocator
->storage_available();
526 * Allocate [more] memory and storage for the image.
528 static void update_image(void)
530 int result
, param_used
, wanted
, got
;
532 suspend_recalculate_image_contents(0);
534 /* Include allowance for growth in pagedir1 while writing pagedir 2 */
535 wanted
= pagedir1
.size
+ extra_pd1_pages_allowance
-
536 get_lowmem_size(pagedir2
);
537 if (wanted
> extra_pages_allocated
) {
538 got
= suspend_allocate_extra_pagedir_memory(wanted
);
540 suspend_message(SUSPEND_EAT_MEMORY
, SUSPEND_LOW
, 1,
541 "Want %d extra pages for pageset1, got %d.\n",
547 thaw_kernel_threads();
550 * Allocate remaining storage space, if possible, up to the
551 * maximum we know we'll need. It's okay to allocate the
552 * maximum if the writer is the swapwriter, but
553 * we don't want to grab all available space on an NFS share.
554 * We therefore ignore the expected compression ratio here,
555 * thereby trying to allocate the maximum image size we could
556 * need (assuming compression doesn't expand the image), but
557 * don't complain if we can't get the full amount we're after.
560 suspendActiveAllocator
->allocate_storage(
561 min(storage_available
, main_storage_needed(0, 0)));
563 main_storage_allocated
= suspendActiveAllocator
->storage_allocated();
565 param_used
= header_storage_needed();
567 result
= suspendActiveAllocator
->allocate_header_space(param_used
);
570 suspend_message(SUSPEND_EAT_MEMORY
, SUSPEND_LOW
, 1,
571 "Still need to get more storage space for header.\n");
573 header_space_allocated
= param_used
;
575 if (freeze_processes()) {
576 set_result_state(SUSPEND_FREEZING_FAILED
);
577 set_result_state(SUSPEND_ABORTED
);
580 allocate_checksum_pages();
582 suspend_recalculate_image_contents(0);
587 * Try to freeze processes.
590 static int attempt_to_freeze(void)
594 /* Stop processes before checking again */
596 suspend_prepare_status(CLEAR_BAR
, "Freezing processes & syncing filesystems.");
597 result
= freeze_processes();
600 set_result_state(SUSPEND_ABORTED
);
601 set_result_state(SUSPEND_FREEZING_FAILED
);
609 * Try to free some memory, either to meet hard or soft constraints on the image
613 * - Pageset1 must be < half of memory;
614 * - We must have enough memory free at resume time to have pageset1
615 * be able to be loaded in pages that don't conflict with where it has to
618 * - User specificied image size limit.
620 static void eat_memory(void)
622 int amount_wanted
= 0;
623 int free_flags
= 0, did_eat_memory
= 0;
626 * Note that if we have enough storage space and enough free memory, we
627 * may exit without eating anything. We give up when the last 10
628 * iterations ate no extra pages because we're not going to get much
629 * more anyway, but the few pages we get will take a lot of time.
631 * We freeze processes before beginning, and then unfreeze them if we
632 * need to eat memory until we think we have enough. If our attempts
633 * to freeze fail, we give up and abort.
636 suspend_recalculate_image_contents(0);
637 amount_wanted
= amount_needed(1);
639 switch (image_size_limit
) {
640 case -1: /* Don't eat any memory */
641 if (amount_wanted
> 0) {
642 set_result_state(SUSPEND_ABORTED
);
643 set_result_state(SUSPEND_WOULD_EAT_MEMORY
);
647 case -2: /* Free caches only */
649 suspend_recalculate_image_contents(0);
650 amount_wanted
= amount_needed(1);
654 free_flags
= GFP_ATOMIC
| __GFP_HIGHMEM
;
657 if (amount_wanted
> 0 && !test_result_state(SUSPEND_ABORTED
) &&
658 image_size_limit
!= -1) {
662 suspend_prepare_status(CLEAR_BAR
, "Seeking to free %dMB of memory.", MB(amount_wanted
));
664 thaw_kernel_threads();
666 for (zone_idx
= 0; zone_idx
< MAX_NR_ZONES
; zone_idx
++) {
667 int zone_type_free
= max_t(int, (zone_idx
== ZONE_HIGHMEM
) ?
668 highpages_ps1_to_free() :
669 lowpages_ps1_to_free(), amount_wanted
);
671 if (zone_type_free
< 0)
674 for_each_zone(zone
) {
675 if (zone_idx(zone
) != zone_idx
)
678 shrink_one_zone(zone
, zone_type_free
);
682 suspend_recalculate_image_contents(0);
684 amount_wanted
= amount_needed(1);
685 zone_type_free
= max_t(int, (zone_idx
== ZONE_HIGHMEM
) ?
686 highpages_ps1_to_free() :
687 lowpages_ps1_to_free(), amount_wanted
);
689 if (zone_type_free
< 0)
694 suspend_cond_pause(0, NULL
);
696 if (freeze_processes()) {
697 set_result_state(SUSPEND_FREEZING_FAILED
);
698 set_result_state(SUSPEND_ABORTED
);
702 if (did_eat_memory
) {
703 unsigned long orig_state
= get_suspend_state();
704 /* Freeze_processes will call sys_sync too */
705 restore_suspend_state(orig_state
);
706 suspend_recalculate_image_contents(0);
709 /* Blank out image size display */
710 suspend_update_status(100, 100, NULL
);
713 /* suspend_prepare_image
715 * Entry point to the whole image preparation section.
718 * - Freeze processes;
719 * - Ensure image size constraints are met;
720 * - Complete all the preparation for saving the image,
721 * including allocation of storage. The only memory
722 * that should be needed when we're finished is that
723 * for actually storing the image (and we know how
724 * much is needed for that because the modules tell
726 * - Make sure that all dirty buffers are written out.
729 int suspend_prepare_image(void)
731 int result
= 1, tries
= 1;
733 header_space_allocated
= 0;
734 main_storage_allocated
= 0;
736 if (attempt_to_freeze())
739 if (!extra_pd1_pages_allowance
)
740 get_extra_pd1_allowance();
742 storage_available
= suspendActiveAllocator
->storage_available();
744 if (!storage_available
) {
745 printk(KERN_ERR
"You need some storage available to be able to suspend.\n");
746 set_result_state(SUSPEND_ABORTED
);
747 set_result_state(SUSPEND_NOSTORAGE_AVAILABLE
);
752 suspend_prepare_status(CLEAR_BAR
, "Preparing Image. Try %d.", tries
);
756 if (test_result_state(SUSPEND_ABORTED
))
763 } while (image_not_ready(1) && tries
<= MAX_TRIES
&&
764 !test_result_state(SUSPEND_ABORTED
));
766 result
= image_not_ready(0);
768 if (!test_result_state(SUSPEND_ABORTED
)) {
771 abort_suspend(SUSPEND_UNABLE_TO_PREPARE_IMAGE
,
772 "Unable to successfully prepare the image.\n");
775 suspend_cond_pause(1, "Image preparation complete.");
782 #ifdef CONFIG_SUSPEND2_EXPORTS
783 EXPORT_SYMBOL_GPL(real_nr_free_pages
);