Make symbol exports GPL.

[linux-2.6/suspend2-2.6.18.git] / kernel / power / prepare_image.c

/*
 * kernel/power/prepare_image.c
 *
 * Copyright (C) 2003-2007 Nigel Cunningham (nigel at suspend2 net)
 *
 * This file is released under the GPLv2.
 *
 * We need to eat memory until we can:
 * 1. Perform the save without changing anything (RAM_NEEDED < #pages)
 * 2. Fit it all in available space (suspendActiveAllocator->available_space() >=
 *    main_storage_needed())
 * 3. Reload the pagedir and pageset1 to places that don't collide with their
 *    final destinations, not knowing to what extent the resumed kernel will
 *    overlap with the one loaded at boot time. I think the resumed kernel
 *    should overlap completely, but I don't want to rely on this as it is 
 *    an unproven assumption. We therefore assume there will be no overlap at
 *    all (worse case).
 * 4. Meet the user's requested limit (if any) on the size of the image.
 *    The limit is in MB, so pages/256 (assuming 4K pages).
 *
 */

#include <linux/module.h>
#include <linux/highmem.h>
#include <linux/freezer.h>
#include <linux/hardirq.h>

#include "pageflags.h"
#include "modules.h"
#include "io.h"
#include "ui.h"
#include "extent.h"
#include "prepare_image.h"
#include "block_io.h"
#include "suspend.h"
#include "checksum.h"
#include "sysfs.h"

static int are_frozen = 0, num_nosave = 0;
static int header_space_allocated = 0;
static int storage_allocated = 0;
static int storage_available = 0;
int extra_pd1_pages_allowance = MIN_EXTRA_PAGES_ALLOWANCE;

static int extra_pages_allocated;

struct extras {
        struct page *page;
        int order;
        struct extras *next;
};

static struct extras *extras_list;

/*
 * real_nr_free_pages: Count pcp pages for a zone type or all zones
 * (-1 for all, otherwise zone_idx() result desired).
 */
int real_nr_free_pages(unsigned long zone_idx_mask)
{
        struct zone *zone;
        int result = 0, i = 0, cpu;

        /* PCP lists */
        for_each_zone(zone) {
                if (!populated_zone(zone))
                        continue;
                
                if (!(zone_idx_mask & (1 << zone_idx(zone))))
                        continue;

                for_each_online_cpu(cpu) {
                        struct per_cpu_pageset *pset = zone_pcp(zone, cpu);

                        for (i = 0; i < ARRAY_SIZE(pset->pcp); i++) {
                                struct per_cpu_pages *pcp;

                                pcp = &pset->pcp[i];
                                result += pcp->count;
                        }
                }

                result += zone->free_pages;
        }
        return result;
}

/*
 * Discover how much extra memory will be required by the drivers
 * when they're asked to suspend. We can then ensure that amount
 * of memory is available when we really want it.
 */
static void get_extra_pd1_allowance(void)
{
        int orig_num_free = real_nr_free_pages(all_zones_mask), final;
        
        suspend_prepare_status(CLEAR_BAR, "Finding allowance for drivers.");

        device_suspend(PMSG_FREEZE);
        local_irq_disable(); /* irqs might have been re-enabled on us */
        device_power_down(PMSG_FREEZE);
        
        final = real_nr_free_pages(all_zones_mask);

        device_power_up();
        local_irq_enable();
        device_resume();

        extra_pd1_pages_allowance = max(
                orig_num_free - final + MIN_EXTRA_PAGES_ALLOWANCE,
                MIN_EXTRA_PAGES_ALLOWANCE);
}

/* suspend_free_extra_pagedir_memory
 *
 * Description: Free previously allocated extra pagedir memory.
 */
void suspend_free_extra_pagedir_memory(void)
{
        /* Free allocated pages */
        while (extras_list) {
                struct extras *this = extras_list;
                int i;

                extras_list = this->next;

                for (i = 0; i < (1 << this->order); i++)
                        ClearPageNosave(this->page + i);

                __free_pages(this->page, this->order);
                kfree(this);
        }

        extra_pages_allocated = 0;
}

/* suspend_allocate_extra_pagedir_memory
 *
 * Description: Allocate memory for making the atomic copy of pagedir1 in the
 *              case where it is bigger than pagedir2.
 * Arguments:   int     num_to_alloc: Number of extra pages needed.
 * Result:      int. Zero on success. One if unable to allocate enough memory.
 */
static int suspend_allocate_extra_pagedir_memory(int extra_pages_needed, int highmem)
{
        int j, order, num_to_alloc = extra_pages_needed - extra_pages_allocated;
        int allocd = 0;
        unsigned long flags = GFP_ATOMIC | __GFP_NOWARN;

        if (num_to_alloc < 1)
                return 0;

        if (highmem)
                flags |= __GFP_HIGHMEM;

        order = fls(num_to_alloc);
        if (order >= MAX_ORDER)
                order = MAX_ORDER - 1;

        while (num_to_alloc) {
                struct page *newpage;
                unsigned long virt;
                struct extras *extras_entry;
                        
                while ((1 << order) > num_to_alloc)
                        order--;

                virt = __get_free_pages(flags, order);
                while (!virt && order) {
                        order--;
                        virt = __get_free_pages(flags, order);
                }

                if (!virt)
                        return allocd;

                newpage = virt_to_page(virt);

                extras_entry = (struct extras *) kmalloc(sizeof(struct extras),
                        GFP_ATOMIC);

                if (!extras_entry) {
                        __free_pages(newpage, order);
                        return allocd;
                }

                extras_entry->page = newpage;
                extras_entry->order = order;
                extras_entry->next = NULL;

                if (extras_list)
                        extras_entry->next = extras_list;

                extras_list = extras_entry;

                for (j = 0; j < (1 << order); j++) {
                        SetPageNosave(newpage + j);
                        SetPagePageset1Copy(newpage + j);
                }

                allocd += (1 << order);
                extra_pages_allocated += (1 << order);
                num_to_alloc -= (1 << order);
        }

        return allocd;
}

/*
 * Amount of storage needed, possibly taking into account the
 * expected compression ratio and possibly also ignoring our
 * allowance for extra pages.
 */
static int main_storage_needed(int use_ecr,
                int ignore_extra_pd1_allow)
{
        return ((pagedir1.size + pagedir2.size +
          (ignore_extra_pd1_allow ? 0 : extra_pd1_pages_allowance)) *
         (use_ecr ? suspend_expected_compression_ratio() : 100) / 100);
}

/*
 * Storage needed for the image header, in bytes until the return.
 */
static int header_storage_needed(void)
{
        int bytes = (int) sizeof(struct suspend_header) +
                        suspend_header_storage_for_modules() +
                        suspend_pageflags_space_needed();

        return DIV_ROUND_UP(bytes, PAGE_SIZE);
}

#ifdef CONFIG_HIGHMEM
static int highpages_needed_to_suspend(void)
{
        int result = max_t(int,
                (get_highmem_size(pagedir1) - get_highmem_size(pagedir2)) / 2 -
                real_nr_free_high_pages(),
                0);

        return result;
}

static int lowpages_needed_to_suspend(void)
{
        int result = max_t(int, (get_lowmem_size(pagedir1) -
                get_lowmem_size(pagedir2) + extra_pd1_pages_allowance)/2 -
                real_nr_free_low_pages(), 0);

        return result;
}

static int ram_to_suspend(void)
{
        int first, second;

        first = (1 + max_t(int, (pagedir1.size + extra_pd1_pages_allowance -
                        get_lowmem_size(pagedir2)) / 2, 0) +
                MIN_FREE_RAM + suspend_memory_for_modules());

        second = highpages_needed_to_suspend() + lowpages_needed_to_suspend();

        return max(first, second);
}
#else
static inline int highpages_needed_to_suspend(void)
{
        return 0;
}

static int ram_to_suspend(void)
{
        return (1 + max_t(int, (pagedir1.size + extra_pd1_pages_allowance -
                        get_lowmem_size(pagedir2)) / 2, 0) +
                MIN_FREE_RAM + suspend_memory_for_modules());
}

static int lowpages_needed_to_suspend(void)
{
        return ram_to_suspend();
}

#endif

static void display_stats(int always, int sub_extra_pd1_allow)
{ 
        char buffer[255];
        snprintf(buffer, 254, 
                "Free:%d(%d). Sets:%d(%d),%d(%d). Header:%d. Nosave:%d-%d"
                "=%d. Storage:%u/%u(%u). Needed:(%d+%d)%d-%d-%d)%d|(%d-%d)%d|"
                "%d.\n",
                
                /* Free */
                real_nr_free_pages(all_zones_mask),
                real_nr_free_low_pages(),
                
                /* Sets */
                pagedir1.size, pagedir1.size - get_highmem_size(pagedir1),
                pagedir2.size, pagedir2.size - get_highmem_size(pagedir2),

                /* Header */
                header_storage_needed(),

                /* Nosave */
                num_nosave, extra_pages_allocated,
                num_nosave - extra_pages_allocated,

                /* Storage - converted to pages for comparison */
                storage_allocated,
                main_storage_needed(1, sub_extra_pd1_allow),
                storage_available,

                /* Needed */
                MIN_FREE_RAM, suspend_memory_for_modules(),
                ram_to_suspend(), real_nr_free_low_pages(),
                extra_pages_allocated,
                ram_to_suspend() - real_nr_free_low_pages() -
                                extra_pages_allocated,
                main_storage_needed(1, sub_extra_pd1_allow), storage_available,
                main_storage_needed(1, sub_extra_pd1_allow) - storage_available,
                (image_size_limit > 0) ? (main_storage_needed(1,
                        sub_extra_pd1_allow) - (image_size_limit << 8)) : 0);

        if (always)
                printk(buffer);
        else
                suspend_message(SUSPEND_EAT_MEMORY, SUSPEND_MEDIUM, 1, buffer);
}

/* generate_free_page_map
 *
 * Description: This routine generates a bitmap of free pages from the
 *              lists used by the memory manager. We then use the bitmap
 *              to quickly calculate which pages to save and in which
 *              pagesets.
 */
static void generate_free_page_map(void) 
{
        int order, loop, cpu;
        struct page *page;
        unsigned long flags, i;
        struct zone *zone;

        for_each_zone(zone) {
                if (!populated_zone(zone))
                        continue;
                
                spin_lock_irqsave(&zone->lock, flags);

                for(i=0; i < zone->spanned_pages; i++)
                        ClearPageNosaveFree(pfn_to_page(
                                                zone->zone_start_pfn + i));
        
                for (order = MAX_ORDER - 1; order >= 0; --order)
                        list_for_each_entry(page,
                                        &zone->free_area[order].free_list, lru)
                                for(loop=0; loop < (1 << order); loop++)
                                        SetPageNosaveFree(page+loop);

                
                for_each_online_cpu(cpu) {
                        struct per_cpu_pageset *pset = zone_pcp(zone, cpu);

                        for (i = 0; i < ARRAY_SIZE(pset->pcp); i++) {
                                struct per_cpu_pages *pcp;
                                struct page *page;

                                pcp = &pset->pcp[i];
                                list_for_each_entry(page, &pcp->list, lru)
                                        SetPageNosaveFree(page);
                        }
                }
                
                spin_unlock_irqrestore(&zone->lock, flags);
        }
}

/* size_of_free_region
 * 
 * Description: Return the number of pages that are free, beginning with and 
 *              including this one.
 */
static int size_of_free_region(struct page *page)
{
        struct zone *zone = page_zone(page);
        struct page *posn = page, *last_in_zone =
                pfn_to_page(zone->zone_start_pfn) + zone->spanned_pages - 1;

        while (posn <= last_in_zone && PageNosaveFree(posn))
                posn++;
        return (posn - page);
}

/* flag_image_pages
 *
 * This routine generates our lists of pages to be stored in each
 * pageset. Since we store the data using extents, and adding new
 * extents might allocate a new extent page, this routine may well
 * be called more than once.
 */
static void flag_image_pages(void)
{
        int num_free = 0;
        unsigned long loop;
        struct zone *zone;

        pagedir1.size = 0;
        pagedir2.size = 0;

        set_highmem_size(pagedir1, 0);
        set_highmem_size(pagedir2, 0);

        num_nosave = 0;

        clear_dyn_pageflags(pageset1_map);

        generate_free_page_map();

        /*
         * Pages not to be saved are marked Nosave irrespective of being reserved
         */
        for_each_zone(zone) {
                if (!populated_zone(zone))
                        continue;

                for (loop = 0; loop < zone->spanned_pages; loop++) {
                        unsigned long pfn = zone->zone_start_pfn + loop;
                        struct page *page;
                        int chunk_size;

                        if (!pfn_valid(pfn))
                                continue;

                        page = pfn_to_page(pfn);

                        chunk_size = size_of_free_region(page);
                        if (chunk_size) {
                                num_free += chunk_size;
                                loop += chunk_size - 1;
                                continue;
                        }

                        if (!saveable(zone, &loop)) {
                                num_nosave++;
                                continue;
                        }

                        if (PagePageset2(page)) {
                                pagedir2.size++;
                                if (PageHighMem(page))
                                        inc_highmem_size(pagedir2);
                                else
                                        SetPagePageset1Copy(page);
                                if (PageResave(page)) {
                                        SetPagePageset1(page);
                                        ClearPagePageset1Copy(page);
                                        pagedir1.size++;
                                        if (PageHighMem(page))
                                                inc_highmem_size(pagedir1);
                                }
                        } else {
                                pagedir1.size++;
                                SetPagePageset1(page);
                                if (PageHighMem(page))
                                        inc_highmem_size(pagedir1);
                        }
                }
        }

        suspend_message(SUSPEND_EAT_MEMORY, SUSPEND_MEDIUM, 0,
                "Count data pages: Set1 (%d) + Set2 (%d) + Nosave (%d) + "
                "NumFree (%d) = %d.\n",
                pagedir1.size, pagedir2.size, num_nosave, num_free,
                pagedir1.size + pagedir2.size + num_nosave + num_free);
}

/* amount_needed
 *
 * Calculates the amount by which the image size needs to be reduced to meet
 * our constraints.
 */
static int amount_needed(int use_image_size_limit)
{
        int max1 = max(ram_to_suspend() - real_nr_free_low_pages() -
                        extra_pages_allocated,
                        main_storage_needed(1, 0) - storage_available);

        if (!use_image_size_limit || image_size_limit < 1)
                return max1;

        return max(max1, main_storage_needed(1, 0) - (image_size_limit << 8));
}

void suspend_recalculate_image_contents(int atomic_copy) 
{
        clear_dyn_pageflags(pageset1_map);
        if (!atomic_copy) {
                int pfn;
                BITMAP_FOR_EACH_SET(pageset2_map, pfn)
                        ClearPagePageset1Copy(pfn_to_page(pfn));
                /* Need to call this before getting pageset1_size! */
                suspend_mark_pages_for_pageset2();
        }
        flag_image_pages();

        if (!atomic_copy) {
                storage_available = suspendActiveAllocator->storage_available();
                display_stats(0, 0);
        }
        return;
}

/* update_image
 *
 * Allocate [more] memory and storage for the image.
 */
static int update_image(void) 
{ 
        int result2, param_used, wanted;

        suspend_recalculate_image_contents(0);

        /* Include allowance for growth in pagedir1 while writing pagedir 2 */
        wanted = pagedir1.size +  extra_pd1_pages_allowance -
                get_lowmem_size(pagedir2);
        if (suspend_allocate_extra_pagedir_memory(wanted, 0) < wanted) { 
                suspend_message(SUSPEND_EAT_MEMORY, SUSPEND_LOW, 1,
                        "Still need to get more pages for pagedir 1.\n");
                return 1;
        }

        thaw_kernel_threads();

        param_used = main_storage_needed(1, 0);
        if ((result2 = suspendActiveAllocator->allocate_storage(param_used))) {
                suspend_message(SUSPEND_EAT_MEMORY, SUSPEND_LOW, 1,
                        "Allocate storage returned %d. Still need to get more"
                        " storage space for the image proper.\n",
                        result2);
                storage_allocated = suspendActiveAllocator->storage_allocated();
                if (freeze_processes()) {
                        set_result_state(SUSPEND_FREEZING_FAILED);
                        set_result_state(SUSPEND_ABORTED);
                }
                return 1;
        }

        /* 
         * Allocate remaining storage space, if possible, up to the
         * maximum we know we'll need. It's okay to allocate the
         * maximum if the writer is the swapwriter, but
         * we don't want to grab all available space on an NFS share.
         * We therefore ignore the expected compression ratio here,
         * thereby trying to allocate the maximum image size we could
         * need (assuming compression doesn't expand the image), but
         * don't complain if we can't get the full amount we're after.
         */

        suspendActiveAllocator->allocate_storage(
                min(storage_available, main_storage_needed(0, 0)));

        storage_allocated = suspendActiveAllocator->storage_allocated();

        /* Allocate the header storage after allocating main storage
         * so that the overhead for metadata doesn't change the amount
         * of storage needed for the header itself.
         */

        param_used = header_storage_needed();

        result2 = suspendActiveAllocator->allocate_header_space(param_used);

        if (freeze_processes()) {
                set_result_state(SUSPEND_FREEZING_FAILED);
                set_result_state(SUSPEND_ABORTED);
        }

        if (result2) {
                suspend_message(SUSPEND_EAT_MEMORY, SUSPEND_LOW, 1,
                        "Still need to get more storage space for header.\n");
                return 1;
        }

        header_space_allocated = param_used;

        allocate_checksum_pages();

        suspend_recalculate_image_contents(0);

        suspend_message(SUSPEND_EAT_MEMORY, SUSPEND_LOW, 1,
                "Amount still needed (%d) > 0:%d. Header: %d < %d: %d,"
                " Storage allocd: %d < %d: %d.\n",
                        amount_needed(1),
                        (amount_needed(1) > 0),
                        header_space_allocated, header_storage_needed(),
                        header_space_allocated < header_storage_needed(),
                        storage_allocated,
                        main_storage_needed(1, 1),
                        storage_allocated < main_storage_needed(1, 1));

        suspend_cond_pause(0, NULL);

        return ((amount_needed(1) > 0) ||
                header_space_allocated < header_storage_needed() ||
                 storage_allocated < main_storage_needed(1, 1));
}

/* attempt_to_freeze
 * 
 * Try to freeze processes.
 */

static int attempt_to_freeze(void)
{
        int result;
        
        /* Stop processes before checking again */
        thaw_processes();
        suspend_prepare_status(CLEAR_BAR, "Freezing processes & syncing filesystems.");
        result = freeze_processes();

        if (result) {
                set_result_state(SUSPEND_ABORTED);
                set_result_state(SUSPEND_FREEZING_FAILED);
        } else
                are_frozen = 1;

        return result;
}

/* eat_memory
 *
 * Try to free some memory, either to meet hard or soft constraints on the image
 * characteristics.
 * 
 * Hard constraints:
 * - Pageset1 must be < half of memory;
 * - We must have enough memory free at resume time to have pageset1
 *   be able to be loaded in pages that don't conflict with where it has to
 *   be restored.
 * Soft constraints
 * - User specificied image size limit.
 */
static void eat_memory(void)
{
        int amount_wanted = 0;
        int free_flags = 0, did_eat_memory = 0;
        
        /*
         * Note that if we have enough storage space and enough free memory, we
         * may exit without eating anything. We give up when the last 10
         * iterations ate no extra pages because we're not going to get much
         * more anyway, but the few pages we get will take a lot of time.
         *
         * We freeze processes before beginning, and then unfreeze them if we
         * need to eat memory until we think we have enough. If our attempts
         * to freeze fail, we give up and abort.
         */

        suspend_recalculate_image_contents(0);
        amount_wanted = amount_needed(1);

        switch (image_size_limit) {
                case -1: /* Don't eat any memory */
                        if (amount_wanted > 0) {
                                set_result_state(SUSPEND_ABORTED);
                                set_result_state(SUSPEND_WOULD_EAT_MEMORY);
                                return;
                        }
                        break;
                case -2:  /* Free caches only */
                        drop_pagecache();
                        suspend_recalculate_image_contents(0);
                        amount_wanted = amount_needed(1);
                        did_eat_memory = 1;
                        break;
                default:
                        free_flags = GFP_ATOMIC | __GFP_HIGHMEM;
        }

        thaw_kernel_threads();

        if (amount_wanted > 0 && !test_result_state(SUSPEND_ABORTED) &&
                        image_size_limit != -1) {
                struct zone *zone;
                int zone_idx, to_free = amount_needed(1);

                suspend_prepare_status(CLEAR_BAR, "Seeking to free %dMB of memory.", MB(amount_wanted));
                for (zone_idx = 0; zone_idx < MAX_NR_ZONES; zone_idx++) {
                        int zone_type_free = max_t(int, (zone_idx == ZONE_HIGHMEM) ?
                                highpages_needed_to_suspend() :
                                lowpages_needed_to_suspend(), to_free);

                        if (zone_type_free < 0)
                                break;

                        for_each_zone(zone) {
                                if (zone_idx(zone) != zone_idx)
                                        continue;

                                shrink_one_zone(zone, zone_type_free);

                                did_eat_memory = 1;

                                suspend_recalculate_image_contents(0);

                                to_free = amount_needed(1);
                                zone_type_free = max_t(int, (zone_idx == ZONE_HIGHMEM) ?
                                        highpages_needed_to_suspend() :
                                        lowpages_needed_to_suspend(), to_free);

                                if (zone_type_free < 0)
                                        break;
                        }
                }

                suspend_cond_pause(0, NULL);
        }

        if (freeze_processes()) {
                set_result_state(SUSPEND_FREEZING_FAILED);
                set_result_state(SUSPEND_ABORTED);
        }
        
        if (did_eat_memory) {
                unsigned long orig_state = get_suspend_state();
                /* Freeze_processes will call sys_sync too */
                restore_suspend_state(orig_state);
                suspend_recalculate_image_contents(0);
        }

        /* Blank out image size display */
        suspend_update_status(100, 100, NULL);
}

/* suspend_prepare_image
 *
 * Entry point to the whole image preparation section.
 *
 * We do four things:
 * - Freeze processes;
 * - Ensure image size constraints are met;
 * - Complete all the preparation for saving the image,
 *   including allocation of storage. The only memory
 *   that should be needed when we're finished is that
 *   for actually storing the image (and we know how
 *   much is needed for that because the modules tell
 *   us).
 * - Make sure that all dirty buffers are written out.
 */
#define MAX_TRIES 2
int suspend_prepare_image(void)
{
        int result = 1, tries = 1;

        are_frozen = 0;

        header_space_allocated = 0;
        storage_allocated = 0;

        if (attempt_to_freeze())
                return 1;

        if (!extra_pd1_pages_allowance)
                get_extra_pd1_allowance();

        storage_available = suspendActiveAllocator->storage_available();

        if (!storage_available) {
                printk(KERN_ERR "You need some storage available to be able to suspend.\n");
                set_result_state(SUSPEND_ABORTED);
                set_result_state(SUSPEND_NOSTORAGE_AVAILABLE);
                return 1;
        }

        do {
                suspend_prepare_status(CLEAR_BAR, "Preparing Image. Try %d.", tries);
        
                eat_memory();

                if (test_result_state(SUSPEND_ABORTED))
                        break;

                result = update_image();

                suspend_cond_pause(0, NULL);
                
                tries++;

        } while (result && (tries <= MAX_TRIES) && (!test_result_state(SUSPEND_ABORTED)) &&
                (!test_result_state(SUSPEND_UNABLE_TO_FREE_ENOUGH_MEMORY)));

        if (!test_result_state(SUSPEND_ABORTED)) {
                if (amount_needed(0) > 0) {
                        display_stats(1, 0);
                        abort_suspend(SUSPEND_UNABLE_TO_PREPARE_IMAGE,
                                "Unable to successfully prepare the image.\n");
                } else {
                        result = 0;
                        suspend_cond_pause(1, "Image preparation complete.");
                }
        }

        return result;
}

#ifdef CONFIG_SUSPEND2_EXPORTS
EXPORT_SYMBOL_GPL(real_nr_free_pages);
#endif