kernel/power/swap.c

   1 /*
   2  * linux/kernel/power/swap.c
   3  *
   4  * This file provides functions for reading the suspend image from
   5  * and writing it to a swap partition.
   6  *
   7  * Copyright (C) 1998,2001-2005 Pavel Machek <pavel@suse.cz>
   8  * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
   9  *
  10  * This file is released under the GPLv2.
  11  *
  12  */
  13
  14 #include <linux/module.h>
  15 #include <linux/smp_lock.h>
  16 #include <linux/file.h>
  17 #include <linux/utsname.h>
  18 #include <linux/version.h>
  19 #include <linux/delay.h>
  20 #include <linux/bitops.h>
  21 #include <linux/genhd.h>
  22 #include <linux/device.h>
  23 #include <linux/buffer_head.h>
  24 #include <linux/bio.h>
  25 #include <linux/blkdev.h>
  26 #include <linux/swap.h>
  27 #include <linux/swapops.h>
  28 #include <linux/pm.h>
  29
  30 #include "power.h"
  31
  32 extern char resume_file[];
  33
  34 #define SWSUSP_SIG      "S1SUSPEND"
  35
  36 static struct swsusp_header {
  37         char reserved[PAGE_SIZE - 20 - sizeof(swp_entry_t)];
  38         swp_entry_t image;
  39         char    orig_sig[10];
  40         char    sig[10];
  41 } __attribute__((packed, aligned(PAGE_SIZE))) swsusp_header;
  42
  43 /*
  44  * General things
  45  */
  46
  47 static unsigned short root_swap = 0xffff;
  48 static struct block_device *resume_bdev;
  49
  50 /**
  51  *      submit - submit BIO request.
  52  *      @rw:    READ or WRITE.
  53  *      @off    physical offset of page.
  54  *      @page:  page we're reading or writing.
  55  *      @bio_chain: list of pending biod (for async reading)
  56  *
  57  *      Straight from the textbook - allocate and initialize the bio.
  58  *      If we're reading, make sure the page is marked as dirty.
  59  *      Then submit it and, if @bio_chain == NULL, wait.
  60  */
  61 static int submit(int rw, pgoff_t page_off, struct page *page,
  62                         struct bio **bio_chain)
  63 {
  64         struct bio *bio;
  65
  66         bio = bio_alloc(GFP_ATOMIC, 1);
  67         if (!bio)
  68                 return -ENOMEM;
  69         bio->bi_sector = page_off * (PAGE_SIZE >> 9);
  70         bio->bi_bdev = resume_bdev;
  71         bio->bi_end_io = end_swap_bio_read;
  72
  73         if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
  74                 printk("swsusp: ERROR: adding page to bio at %ld\n", page_off);
  75                 bio_put(bio);
  76                 return -EFAULT;
  77         }
  78
  79         lock_page(page);
  80         bio_get(bio);
  81
  82         if (bio_chain == NULL) {
  83                 submit_bio(rw | (1 << BIO_RW_SYNC), bio);
  84                 wait_on_page_locked(page);
  85                 if (rw == READ)
  86                         bio_set_pages_dirty(bio);
  87                 bio_put(bio);
  88         } else {
  89                 if (rw == READ)
  90                         get_page(page); /* These pages are freed later */
  91                 bio->bi_private = *bio_chain;
  92                 *bio_chain = bio;
  93                 submit_bio(rw | (1 << BIO_RW_SYNC), bio);
  94         }
  95         return 0;
  96 }
  97
  98 static int bio_read_page(pgoff_t page_off, void *addr, struct bio **bio_chain)
  99 {
 100         return submit(READ, page_off, virt_to_page(addr), bio_chain);
 101 }
 102
 103 static int bio_write_page(pgoff_t page_off, void *addr)
 104 {
 105         return submit(WRITE, page_off, virt_to_page(addr), NULL);
 106 }
 107
 108 static int wait_on_bio_chain(struct bio **bio_chain)
 109 {
 110         struct bio *bio;
 111         struct bio *next_bio;
 112         int ret = 0;
 113
 114         if (bio_chain == NULL)
 115                 return 0;
 116
 117         bio = *bio_chain;
 118         if (bio == NULL)
 119                 return 0;
 120         while (bio) {
 121                 struct page *page;
 122
 123                 next_bio = bio->bi_private;
 124                 page = bio->bi_io_vec[0].bv_page;
 125                 wait_on_page_locked(page);
 126                 if (!PageUptodate(page) || PageError(page))
 127                         ret = -EIO;
 128                 put_page(page);
 129                 bio_put(bio);
 130                 bio = next_bio;
 131         }
 132         *bio_chain = NULL;
 133         return ret;
 134 }
 135
 136 static void show_speed(struct timeval *start, struct timeval *stop,
 137                         unsigned nr_pages, char *msg)
 138 {
 139         s64 elapsed_centisecs64;
 140         int centisecs;
 141         int k;
 142         int kps;
 143
 144         elapsed_centisecs64 = timeval_to_ns(stop) - timeval_to_ns(start);
 145         do_div(elapsed_centisecs64, NSEC_PER_SEC / 100);
 146         centisecs = elapsed_centisecs64;
 147         if (centisecs == 0)
 148                 centisecs = 1;  /* avoid div-by-zero */
 149         k = nr_pages * (PAGE_SIZE / 1024);
 150         kps = (k * 100) / centisecs;
 151         printk("%s %d kbytes in %d.%02d seconds (%d.%02d MB/s)\n", msg, k,
 152                         centisecs / 100, centisecs % 100,
 153                         kps / 1000, (kps % 1000) / 10);
 154 }
 155
 156 /*
 157  * Saving part
 158  */
 159
 160 static int mark_swapfiles(swp_entry_t start)
 161 {
 162         int error;
 163
 164         rw_swap_page_sync(READ, swp_entry(root_swap, 0),
 165                           virt_to_page((unsigned long)&swsusp_header), NULL);
 166         if (!memcmp("SWAP-SPACE",swsusp_header.sig, 10) ||
 167             !memcmp("SWAPSPACE2",swsusp_header.sig, 10)) {
 168                 memcpy(swsusp_header.orig_sig,swsusp_header.sig, 10);
 169                 memcpy(swsusp_header.sig,SWSUSP_SIG, 10);
 170                 swsusp_header.image = start;
 171                 error = rw_swap_page_sync(WRITE, swp_entry(root_swap, 0),
 172                                 virt_to_page((unsigned long)&swsusp_header),
 173                                 NULL);
 174         } else {
 175                 pr_debug("swsusp: Partition is not swap space.\n");
 176                 error = -ENODEV;
 177         }
 178         return error;
 179 }
 180
 181 /**
 182  *      swsusp_swap_check - check if the resume device is a swap device
 183  *      and get its index (if so)
 184  */
 185
 186 static int swsusp_swap_check(void) /* This is called before saving image */
 187 {
 188         int res = swap_type_of(swsusp_resume_device, 0);
 189
 190         if (res >= 0) {
 191                 root_swap = res;
 192                 return 0;
 193         }
 194         return res;
 195 }
 196
 197 /**
 198  *      write_page - Write one page to given swap location.
 199  *      @buf:           Address we're writing.
 200  *      @offset:        Offset of the swap page we're writing to.
 201  *      @bio_chain:     Link the next write BIO here
 202  */
 203
 204 static int write_page(void *buf, unsigned long offset, struct bio **bio_chain)
 205 {
 206         swp_entry_t entry;
 207         int error = -ENOSPC;
 208
 209         if (offset) {
 210                 struct page *page = virt_to_page(buf);
 211
 212                 if (bio_chain) {
 213                         /*
 214                          * Whether or not we successfully allocated a copy page,
 215                          * we take a ref on the page here.  It gets undone in
 216                          * wait_on_bio_chain().
 217                          */
 218                         struct page *page_copy;
 219                         page_copy = alloc_page(GFP_ATOMIC);
 220                         if (page_copy == NULL) {
 221                                 WARN_ON_ONCE(1);
 222                                 bio_chain = NULL;       /* Go synchronous */
 223                                 get_page(page);
 224                         } else {
 225                                 memcpy(page_address(page_copy),
 226                                         page_address(page), PAGE_SIZE);
 227                                 page = page_copy;
 228                         }
 229                 }
 230                 entry = swp_entry(root_swap, offset);
 231                 error = rw_swap_page_sync(WRITE, entry, page, bio_chain);
 232         }
 233         return error;
 234 }
 235
 236 /*
 237  *      The swap map is a data structure used for keeping track of each page
 238  *      written to a swap partition.  It consists of many swap_map_page
 239  *      structures that contain each an array of MAP_PAGE_SIZE swap entries.
 240  *      These structures are stored on the swap and linked together with the
 241  *      help of the .next_swap member.
 242  *
 243  *      The swap map is created during suspend.  The swap map pages are
 244  *      allocated and populated one at a time, so we only need one memory
 245  *      page to set up the entire structure.
 246  *
 247  *      During resume we also only need to use one swap_map_page structure
 248  *      at a time.
 249  */
 250
 251 #define MAP_PAGE_ENTRIES        (PAGE_SIZE / sizeof(long) - 1)
 252
 253 struct swap_map_page {
 254         unsigned long           entries[MAP_PAGE_ENTRIES];
 255         unsigned long           next_swap;
 256 };
 257
 258 /**
 259  *      The swap_map_handle structure is used for handling swap in
 260  *      a file-alike way
 261  */
 262
 263 struct swap_map_handle {
 264         struct swap_map_page *cur;
 265         unsigned long cur_swap;
 266         struct bitmap_page *bitmap;
 267         unsigned int k;
 268 };
 269
 270 static void release_swap_writer(struct swap_map_handle *handle)
 271 {
 272         if (handle->cur)
 273                 free_page((unsigned long)handle->cur);
 274         handle->cur = NULL;
 275         if (handle->bitmap)
 276                 free_bitmap(handle->bitmap);
 277         handle->bitmap = NULL;
 278 }
 279
 280 static int get_swap_writer(struct swap_map_handle *handle)
 281 {
 282         handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_KERNEL);
 283         if (!handle->cur)
 284                 return -ENOMEM;
 285         handle->bitmap = alloc_bitmap(count_swap_pages(root_swap, 0));
 286         if (!handle->bitmap) {
 287                 release_swap_writer(handle);
 288                 return -ENOMEM;
 289         }
 290         handle->cur_swap = alloc_swap_page(root_swap, handle->bitmap);
 291         if (!handle->cur_swap) {
 292                 release_swap_writer(handle);
 293                 return -ENOSPC;
 294         }
 295         handle->k = 0;
 296         return 0;
 297 }
 298
 299 static int swap_write_page(struct swap_map_handle *handle, void *buf,
 300                                 struct bio **bio_chain)
 301 {
 302         int error = 0;
 303         unsigned long offset;
 304
 305         if (!handle->cur)
 306                 return -EINVAL;
 307         offset = alloc_swap_page(root_swap, handle->bitmap);
 308         error = write_page(buf, offset, bio_chain);
 309         if (error)
 310                 return error;
 311         handle->cur->entries[handle->k++] = offset;
 312         if (handle->k >= MAP_PAGE_ENTRIES) {
 313                 error = wait_on_bio_chain(bio_chain);
 314                 if (error)
 315                         goto out;
 316                 offset = alloc_swap_page(root_swap, handle->bitmap);
 317                 if (!offset)
 318                         return -ENOSPC;
 319                 handle->cur->next_swap = offset;
 320                 error = write_page(handle->cur, handle->cur_swap, NULL);
 321                 if (error)
 322                         goto out;
 323                 memset(handle->cur, 0, PAGE_SIZE);
 324                 handle->cur_swap = offset;
 325                 handle->k = 0;
 326         }
 327 out:
 328         return error;
 329 }
 330
 331 static int flush_swap_writer(struct swap_map_handle *handle)
 332 {
 333         if (handle->cur && handle->cur_swap)
 334                 return write_page(handle->cur, handle->cur_swap, NULL);
 335         else
 336                 return -EINVAL;
 337 }
 338
 339 /**
 340  *      save_image - save the suspend image data
 341  */
 342
 343 static int save_image(struct swap_map_handle *handle,
 344                       struct snapshot_handle *snapshot,
 345                       unsigned int nr_to_write)
 346 {
 347         unsigned int m;
 348         int ret;
 349         int error = 0;
 350         int nr_pages;
 351         int err2;
 352         struct bio *bio;
 353         struct timeval start;
 354         struct timeval stop;
 355
 356         printk("Saving image data pages (%u pages) ...     ", nr_to_write);
 357         m = nr_to_write / 100;
 358         if (!m)
 359                 m = 1;
 360         nr_pages = 0;
 361         bio = NULL;
 362         do_gettimeofday(&start);
 363         do {
 364                 ret = snapshot_read_next(snapshot, PAGE_SIZE);
 365                 if (ret > 0) {
 366                         error = swap_write_page(handle, data_of(*snapshot),
 367                                                 &bio);
 368                         if (error)
 369                                 break;
 370                         if (!(nr_pages % m))
 371                                 printk("\b\b\b\b%3d%%", nr_pages / m);
 372                         nr_pages++;
 373                 }
 374         } while (ret > 0);
 375         err2 = wait_on_bio_chain(&bio);
 376         do_gettimeofday(&stop);
 377         if (!error)
 378                 error = err2;
 379         if (!error)
 380                 printk("\b\b\b\bdone\n");
 381         show_speed(&start, &stop, nr_to_write, "Wrote");
 382         return error;
 383 }
 384
 385 /**
 386  *      enough_swap - Make sure we have enough swap to save the image.
 387  *
 388  *      Returns TRUE or FALSE after checking the total amount of swap
 389  *      space avaiable from the resume partition.
 390  */
 391
 392 static int enough_swap(unsigned int nr_pages)
 393 {
 394         unsigned int free_swap = count_swap_pages(root_swap, 1);
 395
 396         pr_debug("swsusp: free swap pages: %u\n", free_swap);
 397         return free_swap > nr_pages + PAGES_FOR_IO;
 398 }
 399
 400 /**
 401  *      swsusp_write - Write entire image and metadata.
 402  *
 403  *      It is important _NOT_ to umount filesystems at this point. We want
 404  *      them synced (in case something goes wrong) but we DO not want to mark
 405  *      filesystem clean: it is not. (And it does not matter, if we resume
 406  *      correctly, we'll mark system clean, anyway.)
 407  */
 408
 409 int swsusp_write(void)
 410 {
 411         struct swap_map_handle handle;
 412         struct snapshot_handle snapshot;
 413         struct swsusp_info *header;
 414         int error;
 415
 416         if ((error = swsusp_swap_check())) {
 417                 printk(KERN_ERR "swsusp: Cannot find swap device, try "
 418                                 "swapon -a.\n");
 419                 return error;
 420         }
 421         memset(&snapshot, 0, sizeof(struct snapshot_handle));
 422         error = snapshot_read_next(&snapshot, PAGE_SIZE);
 423         if (error < PAGE_SIZE)
 424                 return error < 0 ? error : -EFAULT;
 425         header = (struct swsusp_info *)data_of(snapshot);
 426         if (!enough_swap(header->pages)) {
 427                 printk(KERN_ERR "swsusp: Not enough free swap\n");
 428                 return -ENOSPC;
 429         }
 430         error = get_swap_writer(&handle);
 431         if (!error) {
 432                 unsigned long start = handle.cur_swap;
 433                 error = swap_write_page(&handle, header, NULL);
 434                 if (!error)
 435                         error = save_image(&handle, &snapshot,
 436                                         header->pages - 1);
 437                 if (!error) {
 438                         flush_swap_writer(&handle);
 439                         printk("S");
 440                         error = mark_swapfiles(swp_entry(root_swap, start));
 441                         printk("|\n");
 442                 }
 443         }
 444         if (error)
 445                 free_all_swap_pages(root_swap, handle.bitmap);
 446         release_swap_writer(&handle);
 447         return error;
 448 }
 449
 450 /**
 451  *      The following functions allow us to read data using a swap map
 452  *      in a file-alike way
 453  */
 454
 455 static void release_swap_reader(struct swap_map_handle *handle)
 456 {
 457         if (handle->cur)
 458                 free_page((unsigned long)handle->cur);
 459         handle->cur = NULL;
 460 }
 461
 462 static int get_swap_reader(struct swap_map_handle *handle,
 463                                       swp_entry_t start)
 464 {
 465         int error;
 466
 467         if (!swp_offset(start))
 468                 return -EINVAL;
 469         handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_ATOMIC);
 470         if (!handle->cur)
 471                 return -ENOMEM;
 472         error = bio_read_page(swp_offset(start), handle->cur, NULL);
 473         if (error) {
 474                 release_swap_reader(handle);
 475                 return error;
 476         }
 477         handle->k = 0;
 478         return 0;
 479 }
 480
 481 static int swap_read_page(struct swap_map_handle *handle, void *buf,
 482                                 struct bio **bio_chain)
 483 {
 484         unsigned long offset;
 485         int error;
 486
 487         if (!handle->cur)
 488                 return -EINVAL;
 489         offset = handle->cur->entries[handle->k];
 490         if (!offset)
 491                 return -EFAULT;
 492         error = bio_read_page(offset, buf, bio_chain);
 493         if (error)
 494                 return error;
 495         if (++handle->k >= MAP_PAGE_ENTRIES) {
 496                 error = wait_on_bio_chain(bio_chain);
 497                 handle->k = 0;
 498                 offset = handle->cur->next_swap;
 499                 if (!offset)
 500                         release_swap_reader(handle);
 501                 else if (!error)
 502                         error = bio_read_page(offset, handle->cur, NULL);
 503         }
 504         return error;
 505 }
 506
 507 /**
 508  *      load_image - load the image using the swap map handle
 509  *      @handle and the snapshot handle @snapshot
 510  *      (assume there are @nr_pages pages to load)
 511  */
 512
 513 static int load_image(struct swap_map_handle *handle,
 514                       struct snapshot_handle *snapshot,
 515                       unsigned int nr_to_read)
 516 {
 517         unsigned int m;
 518         int error = 0;
 519         struct timeval start;
 520         struct timeval stop;
 521         struct bio *bio;
 522         int err2;
 523         unsigned nr_pages;
 524
 525         printk("Loading image data pages (%u pages) ...     ", nr_to_read);
 526         m = nr_to_read / 100;
 527         if (!m)
 528                 m = 1;
 529         nr_pages = 0;
 530         bio = NULL;
 531         do_gettimeofday(&start);
 532         for ( ; ; ) {
 533                 error = snapshot_write_next(snapshot, PAGE_SIZE);
 534                 if (error <= 0)
 535                         break;
 536                 error = swap_read_page(handle, data_of(*snapshot), &bio);
 537                 if (error)
 538                         break;
 539                 if (snapshot->sync_read)
 540                         error = wait_on_bio_chain(&bio);
 541                 if (error)
 542                         break;
 543                 if (!(nr_pages % m))
 544                         printk("\b\b\b\b%3d%%", nr_pages / m);
 545                 nr_pages++;
 546         }
 547         err2 = wait_on_bio_chain(&bio);
 548         do_gettimeofday(&stop);
 549         if (!error)
 550                 error = err2;
 551         if (!error) {
 552                 printk("\b\b\b\bdone\n");
 553                 snapshot_free_unused_memory(snapshot);
 554                 if (!snapshot_image_loaded(snapshot))
 555                         error = -ENODATA;
 556         }
 557         show_speed(&start, &stop, nr_to_read, "Read");
 558         return error;
 559 }
 560
 561 int swsusp_read(void)
 562 {
 563         int error;
 564         struct swap_map_handle handle;
 565         struct snapshot_handle snapshot;
 566         struct swsusp_info *header;
 567
 568         if (IS_ERR(resume_bdev)) {
 569                 pr_debug("swsusp: block device not initialised\n");
 570                 return PTR_ERR(resume_bdev);
 571         }
 572
 573         memset(&snapshot, 0, sizeof(struct snapshot_handle));
 574         error = snapshot_write_next(&snapshot, PAGE_SIZE);
 575         if (error < PAGE_SIZE)
 576                 return error < 0 ? error : -EFAULT;
 577         header = (struct swsusp_info *)data_of(snapshot);
 578         error = get_swap_reader(&handle, swsusp_header.image);
 579         if (!error)
 580                 error = swap_read_page(&handle, header, NULL);
 581         if (!error)
 582                 error = load_image(&handle, &snapshot, header->pages - 1);
 583         release_swap_reader(&handle);
 584
 585         blkdev_put(resume_bdev);
 586
 587         if (!error)
 588                 pr_debug("swsusp: Reading resume file was successful\n");
 589         else
 590                 pr_debug("swsusp: Error %d resuming\n", error);
 591         return error;
 592 }
 593
 594 /**
 595  *      swsusp_check - Check for swsusp signature in the resume device
 596  */
 597
 598 int swsusp_check(void)
 599 {
 600         int error;
 601
 602         resume_bdev = open_by_devnum(swsusp_resume_device, FMODE_READ);
 603         if (!IS_ERR(resume_bdev)) {
 604                 set_blocksize(resume_bdev, PAGE_SIZE);
 605                 memset(&swsusp_header, 0, sizeof(swsusp_header));
 606                 if ((error = bio_read_page(0, &swsusp_header, NULL)))
 607                         return error;
 608                 if (!memcmp(SWSUSP_SIG, swsusp_header.sig, 10)) {
 609                         memcpy(swsusp_header.sig, swsusp_header.orig_sig, 10);
 610                         /* Reset swap signature now */
 611                         error = bio_write_page(0, &swsusp_header);
 612                 } else {
 613                         return -EINVAL;
 614                 }
 615                 if (error)
 616                         blkdev_put(resume_bdev);
 617                 else
 618                         pr_debug("swsusp: Signature found, resuming\n");
 619         } else {
 620                 error = PTR_ERR(resume_bdev);
 621         }
 622
 623         if (error)
 624                 pr_debug("swsusp: Error %d check for resume file\n", error);
 625
 626         return error;
 627 }
 628
 629 /**
 630  *      swsusp_close - close swap device.
 631  */
 632
 633 void swsusp_close(void)
 634 {
 635         if (IS_ERR(resume_bdev)) {
 636                 pr_debug("swsusp: block device not initialised\n");
 637                 return;
 638         }
 639
 640         blkdev_put(resume_bdev);
 641 }