mm/vmalloc.c

   1 /*
   2  *  linux/mm/vmalloc.c
   3  *
   4  *  Copyright (C) 1993  Linus Torvalds
   5  *  Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
   6  *  SMP-safe vmalloc/vfree/ioremap, Tigran Aivazian <tigran@veritas.com>, May 2000
   7  *  Major rework to support vmap/vunmap, Christoph Hellwig, SGI, August 2002
   8  *  Numa awareness, Christoph Lameter, SGI, June 2005
   9  */
  10
  11 #include <linux/mm.h>
  12 #include <linux/module.h>
  13 #include <linux/highmem.h>
  14 #include <linux/slab.h>
  15 #include <linux/spinlock.h>
  16 #include <linux/interrupt.h>
  17
  18 #include <linux/vmalloc.h>
  19
  20 #include <asm/uaccess.h>
  21 #include <asm/tlbflush.h>
  22
  23
  24 DEFINE_RWLOCK(vmlist_lock);
  25 struct vm_struct *vmlist;
  26
  27 static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
  28                             int node);
  29
  30 static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end)
  31 {
  32         pte_t *pte;
  33
  34         pte = pte_offset_kernel(pmd, addr);
  35         do {
  36                 pte_t ptent = ptep_get_and_clear(&init_mm, addr, pte);
  37                 WARN_ON(!pte_none(ptent) && !pte_present(ptent));
  38         } while (pte++, addr += PAGE_SIZE, addr != end);
  39 }
  40
  41 static inline void vunmap_pmd_range(pud_t *pud, unsigned long addr,
  42                                                 unsigned long end)
  43 {
  44         pmd_t *pmd;
  45         unsigned long next;
  46
  47         pmd = pmd_offset(pud, addr);
  48         do {
  49                 next = pmd_addr_end(addr, end);
  50                 if (pmd_none_or_clear_bad(pmd))
  51                         continue;
  52                 vunmap_pte_range(pmd, addr, next);
  53         } while (pmd++, addr = next, addr != end);
  54 }
  55
  56 static inline void vunmap_pud_range(pgd_t *pgd, unsigned long addr,
  57                                                 unsigned long end)
  58 {
  59         pud_t *pud;
  60         unsigned long next;
  61
  62         pud = pud_offset(pgd, addr);
  63         do {
  64                 next = pud_addr_end(addr, end);
  65                 if (pud_none_or_clear_bad(pud))
  66                         continue;
  67                 vunmap_pmd_range(pud, addr, next);
  68         } while (pud++, addr = next, addr != end);
  69 }
  70
  71 void unmap_kernel_range(unsigned long addr, unsigned long size)
  72 {
  73         pgd_t *pgd;
  74         unsigned long next;
  75         unsigned long start = addr;
  76         unsigned long end = addr + size;
  77
  78         BUG_ON(addr >= end);
  79         pgd = pgd_offset_k(addr);
  80         flush_cache_vunmap(addr, end);
  81         do {
  82                 next = pgd_addr_end(addr, end);
  83                 if (pgd_none_or_clear_bad(pgd))
  84                         continue;
  85                 vunmap_pud_range(pgd, addr, next);
  86         } while (pgd++, addr = next, addr != end);
  87         flush_tlb_kernel_range(start, end);
  88 }
  89
  90 static void unmap_vm_area(struct vm_struct *area)
  91 {
  92         unmap_kernel_range((unsigned long)area->addr, area->size);
  93 }
  94
  95 static int vmap_pte_range(pmd_t *pmd, unsigned long addr,
  96                         unsigned long end, pgprot_t prot, struct page ***pages)
  97 {
  98         pte_t *pte;
  99
 100         pte = pte_alloc_kernel(pmd, addr);
 101         if (!pte)
 102                 return -ENOMEM;
 103         do {
 104                 struct page *page = **pages;
 105                 WARN_ON(!pte_none(*pte));
 106                 if (!page)
 107                         return -ENOMEM;
 108                 set_pte_at(&init_mm, addr, pte, mk_pte(page, prot));
 109                 (*pages)++;
 110         } while (pte++, addr += PAGE_SIZE, addr != end);
 111         return 0;
 112 }
 113
 114 static inline int vmap_pmd_range(pud_t *pud, unsigned long addr,
 115                         unsigned long end, pgprot_t prot, struct page ***pages)
 116 {
 117         pmd_t *pmd;
 118         unsigned long next;
 119
 120         pmd = pmd_alloc(&init_mm, pud, addr);
 121         if (!pmd)
 122                 return -ENOMEM;
 123         do {
 124                 next = pmd_addr_end(addr, end);
 125                 if (vmap_pte_range(pmd, addr, next, prot, pages))
 126                         return -ENOMEM;
 127         } while (pmd++, addr = next, addr != end);
 128         return 0;
 129 }
 130
 131 static inline int vmap_pud_range(pgd_t *pgd, unsigned long addr,
 132                         unsigned long end, pgprot_t prot, struct page ***pages)
 133 {
 134         pud_t *pud;
 135         unsigned long next;
 136
 137         pud = pud_alloc(&init_mm, pgd, addr);
 138         if (!pud)
 139                 return -ENOMEM;
 140         do {
 141                 next = pud_addr_end(addr, end);
 142                 if (vmap_pmd_range(pud, addr, next, prot, pages))
 143                         return -ENOMEM;
 144         } while (pud++, addr = next, addr != end);
 145         return 0;
 146 }
 147
 148 int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages)
 149 {
 150         pgd_t *pgd;
 151         unsigned long next;
 152         unsigned long addr = (unsigned long) area->addr;
 153         unsigned long end = addr + area->size - PAGE_SIZE;
 154         int err;
 155
 156         BUG_ON(addr >= end);
 157         pgd = pgd_offset_k(addr);
 158         do {
 159                 next = pgd_addr_end(addr, end);
 160                 err = vmap_pud_range(pgd, addr, next, prot, pages);
 161                 if (err)
 162                         break;
 163         } while (pgd++, addr = next, addr != end);
 164         flush_cache_vmap((unsigned long) area->addr, end);
 165         return err;
 166 }
 167 EXPORT_SYMBOL_GPL(map_vm_area);
 168
 169 static struct vm_struct *__get_vm_area_node(unsigned long size, unsigned long flags,
 170                                             unsigned long start, unsigned long end,
 171                                             int node, gfp_t gfp_mask)
 172 {
 173         struct vm_struct **p, *tmp, *area;
 174         unsigned long align = 1;
 175         unsigned long addr;
 176
 177         BUG_ON(in_interrupt());
 178         if (flags & VM_IOREMAP) {
 179                 int bit = fls(size);
 180
 181                 if (bit > IOREMAP_MAX_ORDER)
 182                         bit = IOREMAP_MAX_ORDER;
 183                 else if (bit < PAGE_SHIFT)
 184                         bit = PAGE_SHIFT;
 185
 186                 align = 1ul << bit;
 187         }
 188         addr = ALIGN(start, align);
 189         size = PAGE_ALIGN(size);
 190         if (unlikely(!size))
 191                 return NULL;
 192
 193         area = kmalloc_node(sizeof(*area), gfp_mask & GFP_LEVEL_MASK, node);
 194         if (unlikely(!area))
 195                 return NULL;
 196
 197         /*
 198          * We always allocate a guard page.
 199          */
 200         size += PAGE_SIZE;
 201
 202         write_lock(&vmlist_lock);
 203         for (p = &vmlist; (tmp = *p) != NULL ;p = &tmp->next) {
 204                 if ((unsigned long)tmp->addr < addr) {
 205                         if((unsigned long)tmp->addr + tmp->size >= addr)
 206                                 addr = ALIGN(tmp->size +
 207                                              (unsigned long)tmp->addr, align);
 208                         continue;
 209                 }
 210                 if ((size + addr) < addr)
 211                         goto out;
 212                 if (size + addr <= (unsigned long)tmp->addr)
 213                         goto found;
 214                 addr = ALIGN(tmp->size + (unsigned long)tmp->addr, align);
 215                 if (addr > end - size)
 216                         goto out;
 217         }
 218
 219 found:
 220         area->next = *p;
 221         *p = area;
 222
 223         area->flags = flags;
 224         area->addr = (void *)addr;
 225         area->size = size;
 226         area->pages = NULL;
 227         area->nr_pages = 0;
 228         area->phys_addr = 0;
 229         write_unlock(&vmlist_lock);
 230
 231         return area;
 232
 233 out:
 234         write_unlock(&vmlist_lock);
 235         kfree(area);
 236         if (printk_ratelimit())
 237                 printk(KERN_WARNING "allocation failed: out of vmalloc space - use vmalloc=<size> to increase size.\n");
 238         return NULL;
 239 }
 240
 241 struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags,
 242                                 unsigned long start, unsigned long end)
 243 {
 244         return __get_vm_area_node(size, flags, start, end, -1, GFP_KERNEL);
 245 }
 246 EXPORT_SYMBOL_GPL(__get_vm_area);
 247
 248 /**
 249  *      get_vm_area  -  reserve a contingous kernel virtual area
 250  *      @size:          size of the area
 251  *      @flags:         %VM_IOREMAP for I/O mappings or VM_ALLOC
 252  *
 253  *      Search an area of @size in the kernel virtual mapping area,
 254  *      and reserved it for out purposes.  Returns the area descriptor
 255  *      on success or %NULL on failure.
 256  */
 257 struct vm_struct *get_vm_area(unsigned long size, unsigned long flags)
 258 {
 259         return __get_vm_area(size, flags, VMALLOC_START, VMALLOC_END);
 260 }
 261
 262 struct vm_struct *get_vm_area_node(unsigned long size, unsigned long flags,
 263                                    int node, gfp_t gfp_mask)
 264 {
 265         return __get_vm_area_node(size, flags, VMALLOC_START, VMALLOC_END, node,
 266                                   gfp_mask);
 267 }
 268
 269 /* Caller must hold vmlist_lock */
 270 static struct vm_struct *__find_vm_area(void *addr)
 271 {
 272         struct vm_struct *tmp;
 273
 274         for (tmp = vmlist; tmp != NULL; tmp = tmp->next) {
 275                  if (tmp->addr == addr)
 276                         break;
 277         }
 278
 279         return tmp;
 280 }
 281
 282 /* Caller must hold vmlist_lock */
 283 static struct vm_struct *__remove_vm_area(void *addr)
 284 {
 285         struct vm_struct **p, *tmp;
 286
 287         for (p = &vmlist ; (tmp = *p) != NULL ;p = &tmp->next) {
 288                  if (tmp->addr == addr)
 289                          goto found;
 290         }
 291         return NULL;
 292
 293 found:
 294         unmap_vm_area(tmp);
 295         *p = tmp->next;
 296
 297         /*
 298          * Remove the guard page.
 299          */
 300         tmp->size -= PAGE_SIZE;
 301         return tmp;
 302 }
 303
 304 /**
 305  *      remove_vm_area  -  find and remove a contingous kernel virtual area
 306  *      @addr:          base address
 307  *
 308  *      Search for the kernel VM area starting at @addr, and remove it.
 309  *      This function returns the found VM area, but using it is NOT safe
 310  *      on SMP machines, except for its size or flags.
 311  */
 312 struct vm_struct *remove_vm_area(void *addr)
 313 {
 314         struct vm_struct *v;
 315         write_lock(&vmlist_lock);
 316         v = __remove_vm_area(addr);
 317         write_unlock(&vmlist_lock);
 318         return v;
 319 }
 320
 321 static void __vunmap(void *addr, int deallocate_pages)
 322 {
 323         struct vm_struct *area;
 324
 325         if (!addr)
 326                 return;
 327
 328         if ((PAGE_SIZE-1) & (unsigned long)addr) {
 329                 printk(KERN_ERR "Trying to vfree() bad address (%p)\n", addr);
 330                 WARN_ON(1);
 331                 return;
 332         }
 333
 334         area = remove_vm_area(addr);
 335         if (unlikely(!area)) {
 336                 printk(KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
 337                                 addr);
 338                 WARN_ON(1);
 339                 return;
 340         }
 341
 342         debug_check_no_locks_freed(addr, area->size);
 343
 344         if (deallocate_pages) {
 345                 int i;
 346
 347                 for (i = 0; i < area->nr_pages; i++) {
 348                         BUG_ON(!area->pages[i]);
 349                         __free_page(area->pages[i]);
 350                 }
 351
 352                 if (area->flags & VM_VPAGES)
 353                         vfree(area->pages);
 354                 else
 355                         kfree(area->pages);
 356         }
 357
 358         kfree(area);
 359         return;
 360 }
 361
 362 /**
 363  *      vfree  -  release memory allocated by vmalloc()
 364  *      @addr:          memory base address
 365  *
 366  *      Free the virtually contiguous memory area starting at @addr, as
 367  *      obtained from vmalloc(), vmalloc_32() or __vmalloc(). If @addr is
 368  *      NULL, no operation is performed.
 369  *
 370  *      Must not be called in interrupt context.
 371  */
 372 void vfree(void *addr)
 373 {
 374         BUG_ON(in_interrupt());
 375         __vunmap(addr, 1);
 376 }
 377 EXPORT_SYMBOL(vfree);
 378
 379 /**
 380  *      vunmap  -  release virtual mapping obtained by vmap()
 381  *      @addr:          memory base address
 382  *
 383  *      Free the virtually contiguous memory area starting at @addr,
 384  *      which was created from the page array passed to vmap().
 385  *
 386  *      Must not be called in interrupt context.
 387  */
 388 void vunmap(void *addr)
 389 {
 390         BUG_ON(in_interrupt());
 391         __vunmap(addr, 0);
 392 }
 393 EXPORT_SYMBOL(vunmap);
 394
 395 /**
 396  *      vmap  -  map an array of pages into virtually contiguous space
 397  *      @pages:         array of page pointers
 398  *      @count:         number of pages to map
 399  *      @flags:         vm_area->flags
 400  *      @prot:          page protection for the mapping
 401  *
 402  *      Maps @count pages from @pages into contiguous kernel virtual
 403  *      space.
 404  */
 405 void *vmap(struct page **pages, unsigned int count,
 406                 unsigned long flags, pgprot_t prot)
 407 {
 408         struct vm_struct *area;
 409
 410         if (count > num_physpages)
 411                 return NULL;
 412
 413         area = get_vm_area((count << PAGE_SHIFT), flags);
 414         if (!area)
 415                 return NULL;
 416         if (map_vm_area(area, prot, &pages)) {
 417                 vunmap(area->addr);
 418                 return NULL;
 419         }
 420
 421         return area->addr;
 422 }
 423 EXPORT_SYMBOL(vmap);
 424
 425 void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
 426                                 pgprot_t prot, int node)
 427 {
 428         struct page **pages;
 429         unsigned int nr_pages, array_size, i;
 430
 431         nr_pages = (area->size - PAGE_SIZE) >> PAGE_SHIFT;
 432         array_size = (nr_pages * sizeof(struct page *));
 433
 434         area->nr_pages = nr_pages;
 435         /* Please note that the recursion is strictly bounded. */
 436         if (array_size > PAGE_SIZE) {
 437                 pages = __vmalloc_node(array_size, gfp_mask | __GFP_ZERO,
 438                                         PAGE_KERNEL, node);
 439                 area->flags |= VM_VPAGES;
 440         } else {
 441                 pages = kmalloc_node(array_size,
 442                                 (gfp_mask & GFP_LEVEL_MASK) | __GFP_ZERO,
 443                                 node);
 444         }
 445         area->pages = pages;
 446         if (!area->pages) {
 447                 remove_vm_area(area->addr);
 448                 kfree(area);
 449                 return NULL;
 450         }
 451
 452         for (i = 0; i < area->nr_pages; i++) {
 453                 if (node < 0)
 454                         area->pages[i] = alloc_page(gfp_mask);
 455                 else
 456                         area->pages[i] = alloc_pages_node(node, gfp_mask, 0);
 457                 if (unlikely(!area->pages[i])) {
 458                         /* Successfully allocated i pages, free them in __vunmap() */
 459                         area->nr_pages = i;
 460                         goto fail;
 461                 }
 462         }
 463
 464         if (map_vm_area(area, prot, &pages))
 465                 goto fail;
 466         return area->addr;
 467
 468 fail:
 469         vfree(area->addr);
 470         return NULL;
 471 }
 472
 473 void *__vmalloc_area(struct vm_struct *area, gfp_t gfp_mask, pgprot_t prot)
 474 {
 475         return __vmalloc_area_node(area, gfp_mask, prot, -1);
 476 }
 477
 478 /**
 479  *      __vmalloc_node  -  allocate virtually contiguous memory
 480  *      @size:          allocation size
 481  *      @gfp_mask:      flags for the page level allocator
 482  *      @prot:          protection mask for the allocated pages
 483  *      @node:          node to use for allocation or -1
 484  *
 485  *      Allocate enough pages to cover @size from the page level
 486  *      allocator with @gfp_mask flags.  Map them into contiguous
 487  *      kernel virtual space, using a pagetable protection of @prot.
 488  */
 489 static void *__vmalloc_node(unsigned long size, gfp_t gfp_mask, pgprot_t prot,
 490                             int node)
 491 {
 492         struct vm_struct *area;
 493
 494         size = PAGE_ALIGN(size);
 495         if (!size || (size >> PAGE_SHIFT) > num_physpages)
 496                 return NULL;
 497
 498         area = get_vm_area_node(size, VM_ALLOC, node, gfp_mask);
 499         if (!area)
 500                 return NULL;
 501
 502         return __vmalloc_area_node(area, gfp_mask, prot, node);
 503 }
 504
 505 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot)
 506 {
 507         return __vmalloc_node(size, gfp_mask, prot, -1);
 508 }
 509 EXPORT_SYMBOL(__vmalloc);
 510
 511 /**
 512  *      vmalloc  -  allocate virtually contiguous memory
 513  *      @size:          allocation size
 514  *      Allocate enough pages to cover @size from the page level
 515  *      allocator and map them into contiguous kernel virtual space.
 516  *
 517  *      For tight control over page level allocator and protection flags
 518  *      use __vmalloc() instead.
 519  */
 520 void *vmalloc(unsigned long size)
 521 {
 522         return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL);
 523 }
 524 EXPORT_SYMBOL(vmalloc);
 525
 526 /**
 527  * vmalloc_user - allocate zeroed virtually contiguous memory for userspace
 528  * @size: allocation size
 529  *
 530  * The resulting memory area is zeroed so it can be mapped to userspace
 531  * without leaking data.
 532  */
 533 void *vmalloc_user(unsigned long size)
 534 {
 535         struct vm_struct *area;
 536         void *ret;
 537
 538         ret = __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL);
 539         if (ret) {
 540                 write_lock(&vmlist_lock);
 541                 area = __find_vm_area(ret);
 542                 area->flags |= VM_USERMAP;
 543                 write_unlock(&vmlist_lock);
 544         }
 545         return ret;
 546 }
 547 EXPORT_SYMBOL(vmalloc_user);
 548
 549 /**
 550  *      vmalloc_node  -  allocate memory on a specific node
 551  *      @size:          allocation size
 552  *      @node:          numa node
 553  *
 554  *      Allocate enough pages to cover @size from the page level
 555  *      allocator and map them into contiguous kernel virtual space.
 556  *
 557  *      For tight control over page level allocator and protection flags
 558  *      use __vmalloc() instead.
 559  */
 560 void *vmalloc_node(unsigned long size, int node)
 561 {
 562         return __vmalloc_node(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL, node);
 563 }
 564 EXPORT_SYMBOL(vmalloc_node);
 565
 566 #ifndef PAGE_KERNEL_EXEC
 567 # define PAGE_KERNEL_EXEC PAGE_KERNEL
 568 #endif
 569
 570 /**
 571  *      vmalloc_exec  -  allocate virtually contiguous, executable memory
 572  *      @size:          allocation size
 573  *
 574  *      Kernel-internal function to allocate enough pages to cover @size
 575  *      the page level allocator and map them into contiguous and
 576  *      executable kernel virtual space.
 577  *
 578  *      For tight control over page level allocator and protection flags
 579  *      use __vmalloc() instead.
 580  */
 581
 582 void *vmalloc_exec(unsigned long size)
 583 {
 584         return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC);
 585 }
 586
 587 #if defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA32)
 588 #define GFP_VMALLOC32 GFP_DMA32 | GFP_KERNEL
 589 #elif defined(CONFIG_64BIT) && defined(CONFIG_ZONE_DMA)
 590 #define GFP_VMALLOC32 GFP_DMA | GFP_KERNEL
 591 #else
 592 #define GFP_VMALLOC32 GFP_KERNEL
 593 #endif
 594
 595 /**
 596  *      vmalloc_32  -  allocate virtually contiguous memory (32bit addressable)
 597  *      @size:          allocation size
 598  *
 599  *      Allocate enough 32bit PA addressable pages to cover @size from the
 600  *      page level allocator and map them into contiguous kernel virtual space.
 601  */
 602 void *vmalloc_32(unsigned long size)
 603 {
 604         return __vmalloc(size, GFP_VMALLOC32, PAGE_KERNEL);
 605 }
 606 EXPORT_SYMBOL(vmalloc_32);
 607
 608 /**
 609  * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory
 610  *      @size:          allocation size
 611  *
 612  * The resulting memory area is 32bit addressable and zeroed so it can be
 613  * mapped to userspace without leaking data.
 614  */
 615 void *vmalloc_32_user(unsigned long size)
 616 {
 617         struct vm_struct *area;
 618         void *ret;
 619
 620         ret = __vmalloc(size, GFP_VMALLOC32 | __GFP_ZERO, PAGE_KERNEL);
 621         if (ret) {
 622                 write_lock(&vmlist_lock);
 623                 area = __find_vm_area(ret);
 624                 area->flags |= VM_USERMAP;
 625                 write_unlock(&vmlist_lock);
 626         }
 627         return ret;
 628 }
 629 EXPORT_SYMBOL(vmalloc_32_user);
 630
 631 long vread(char *buf, char *addr, unsigned long count)
 632 {
 633         struct vm_struct *tmp;
 634         char *vaddr, *buf_start = buf;
 635         unsigned long n;
 636
 637         /* Don't allow overflow */
 638         if ((unsigned long) addr + count < count)
 639                 count = -(unsigned long) addr;
 640
 641         read_lock(&vmlist_lock);
 642         for (tmp = vmlist; tmp; tmp = tmp->next) {
 643                 vaddr = (char *) tmp->addr;
 644                 if (addr >= vaddr + tmp->size - PAGE_SIZE)
 645                         continue;
 646                 while (addr < vaddr) {
 647                         if (count == 0)
 648                                 goto finished;
 649                         *buf = '\0';
 650                         buf++;
 651                         addr++;
 652                         count--;
 653                 }
 654                 n = vaddr + tmp->size - PAGE_SIZE - addr;
 655                 do {
 656                         if (count == 0)
 657                                 goto finished;
 658                         *buf = *addr;
 659                         buf++;
 660                         addr++;
 661                         count--;
 662                 } while (--n > 0);
 663         }
 664 finished:
 665         read_unlock(&vmlist_lock);
 666         return buf - buf_start;
 667 }
 668
 669 long vwrite(char *buf, char *addr, unsigned long count)
 670 {
 671         struct vm_struct *tmp;
 672         char *vaddr, *buf_start = buf;
 673         unsigned long n;
 674
 675         /* Don't allow overflow */
 676         if ((unsigned long) addr + count < count)
 677                 count = -(unsigned long) addr;
 678
 679         read_lock(&vmlist_lock);
 680         for (tmp = vmlist; tmp; tmp = tmp->next) {
 681                 vaddr = (char *) tmp->addr;
 682                 if (addr >= vaddr + tmp->size - PAGE_SIZE)
 683                         continue;
 684                 while (addr < vaddr) {
 685                         if (count == 0)
 686                                 goto finished;
 687                         buf++;
 688                         addr++;
 689                         count--;
 690                 }
 691                 n = vaddr + tmp->size - PAGE_SIZE - addr;
 692                 do {
 693                         if (count == 0)
 694                                 goto finished;
 695                         *addr = *buf;
 696                         buf++;
 697                         addr++;
 698                         count--;
 699                 } while (--n > 0);
 700         }
 701 finished:
 702         read_unlock(&vmlist_lock);
 703         return buf - buf_start;
 704 }
 705
 706 /**
 707  *      remap_vmalloc_range  -  map vmalloc pages to userspace
 708  *      @vma:           vma to cover (map full range of vma)
 709  *      @addr:          vmalloc memory
 710  *      @pgoff:         number of pages into addr before first page to map
 711  *      @returns:       0 for success, -Exxx on failure
 712  *
 713  *      This function checks that addr is a valid vmalloc'ed area, and
 714  *      that it is big enough to cover the vma. Will return failure if
 715  *      that criteria isn't met.
 716  *
 717  *      Similar to remap_pfn_range() (see mm/memory.c)
 718  */
 719 int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
 720                                                 unsigned long pgoff)
 721 {
 722         struct vm_struct *area;
 723         unsigned long uaddr = vma->vm_start;
 724         unsigned long usize = vma->vm_end - vma->vm_start;
 725         int ret;
 726
 727         if ((PAGE_SIZE-1) & (unsigned long)addr)
 728                 return -EINVAL;
 729
 730         read_lock(&vmlist_lock);
 731         area = __find_vm_area(addr);
 732         if (!area)
 733                 goto out_einval_locked;
 734
 735         if (!(area->flags & VM_USERMAP))
 736                 goto out_einval_locked;
 737
 738         if (usize + (pgoff << PAGE_SHIFT) > area->size - PAGE_SIZE)
 739                 goto out_einval_locked;
 740         read_unlock(&vmlist_lock);
 741
 742         addr += pgoff << PAGE_SHIFT;
 743         do {
 744                 struct page *page = vmalloc_to_page(addr);
 745                 ret = vm_insert_page(vma, uaddr, page);
 746                 if (ret)
 747                         return ret;
 748
 749                 uaddr += PAGE_SIZE;
 750                 addr += PAGE_SIZE;
 751                 usize -= PAGE_SIZE;
 752         } while (usize > 0);
 753
 754         /* Prevent "things" like memory migration? VM_flags need a cleanup... */
 755         vma->vm_flags |= VM_RESERVED;
 756
 757         return ret;
 758
 759 out_einval_locked:
 760         read_unlock(&vmlist_lock);
 761         return -EINVAL;
 762 }
 763 EXPORT_SYMBOL(remap_vmalloc_range);
 764
 765 /*
 766  * Implement a stub for vmalloc_sync_all() if the architecture chose not to
 767  * have one.
 768  */
 769 void  __attribute__((weak)) vmalloc_sync_all(void)
 770 {
 771 }
 772
 773
 774 static int f(pte_t *pte, struct page *pmd_page, unsigned long addr, void *data)
 775 {
 776         /* apply_to_page_range() does all the hard work. */
 777         return 0;
 778 }
 779
 780 /**
 781  *      alloc_vm_area - allocate a range of kernel address space
 782  *      @size:          size of the area
 783  *      @returns:       NULL on failure, vm_struct on success
 784  *
 785  *      This function reserves a range of kernel address space, and
 786  *      allocates pagetables to map that range.  No actual mappings
 787  *      are created.  If the kernel address space is not shared
 788  *      between processes, it syncs the pagetable across all
 789  *      processes.
 790  */
 791 struct vm_struct *alloc_vm_area(size_t size)
 792 {
 793         struct vm_struct *area;
 794
 795         area = get_vm_area(size, VM_IOREMAP);
 796         if (area == NULL)
 797                 return NULL;
 798
 799         /*
 800          * This ensures that page tables are constructed for this region
 801          * of kernel virtual address space and mapped into init_mm.
 802          */
 803         if (apply_to_page_range(&init_mm, (unsigned long)area->addr,
 804                                 area->size, f, NULL)) {
 805                 free_vm_area(area);
 806                 return NULL;
 807         }
 808
 809         /* Make sure the pagetables are constructed in process kernel
 810            mappings */
 811         vmalloc_sync_all();
 812
 813         return area;
 814 }
 815 EXPORT_SYMBOL_GPL(alloc_vm_area);
 816
 817 void free_vm_area(struct vm_struct *area)
 818 {
 819         struct vm_struct *ret;
 820         ret = remove_vm_area(area->addr);
 821         BUG_ON(ret != area);
 822         kfree(area);
 823 }
 824 EXPORT_SYMBOL_GPL(free_vm_area);