mm/util.c

   1 #include <linux/mm.h>
   2 #include <linux/slab.h>
   3 #include <linux/string.h>
   4 #include <linux/export.h>
   5 #include <linux/err.h>
   6 #include <linux/sched.h>
   7 #include <linux/security.h>
   8 #include <asm/uaccess.h>
   9
  10 #include "internal.h"
  11
  12 #define CREATE_TRACE_POINTS
  13 #include <trace/events/kmem.h>
  14
  15 /**
  16  * kstrdup - allocate space for and copy an existing string
  17  * @s: the string to duplicate
  18  * @gfp: the GFP mask used in the kmalloc() call when allocating memory
  19  */
  20 char *kstrdup(const char *s, gfp_t gfp)
  21 {
  22         size_t len;
  23         char *buf;
  24
  25         if (!s)
  26                 return NULL;
  27
  28         len = strlen(s) + 1;
  29         buf = kmalloc_track_caller(len, gfp);
  30         if (buf)
  31                 memcpy(buf, s, len);
  32         return buf;
  33 }
  34 EXPORT_SYMBOL(kstrdup);
  35
  36 /**
  37  * kstrndup - allocate space for and copy an existing string
  38  * @s: the string to duplicate
  39  * @max: read at most @max chars from @s
  40  * @gfp: the GFP mask used in the kmalloc() call when allocating memory
  41  */
  42 char *kstrndup(const char *s, size_t max, gfp_t gfp)
  43 {
  44         size_t len;
  45         char *buf;
  46
  47         if (!s)
  48                 return NULL;
  49
  50         len = strnlen(s, max);
  51         buf = kmalloc_track_caller(len+1, gfp);
  52         if (buf) {
  53                 memcpy(buf, s, len);
  54                 buf[len] = '\0';
  55         }
  56         return buf;
  57 }
  58 EXPORT_SYMBOL(kstrndup);
  59
  60 /**
  61  * kmemdup - duplicate region of memory
  62  *
  63  * @src: memory region to duplicate
  64  * @len: memory region length
  65  * @gfp: GFP mask to use
  66  */
  67 void *kmemdup(const void *src, size_t len, gfp_t gfp)
  68 {
  69         void *p;
  70
  71         p = kmalloc_track_caller(len, gfp);
  72         if (p)
  73                 memcpy(p, src, len);
  74         return p;
  75 }
  76 EXPORT_SYMBOL(kmemdup);
  77
  78 /**
  79  * memdup_user - duplicate memory region from user space
  80  *
  81  * @src: source address in user space
  82  * @len: number of bytes to copy
  83  *
  84  * Returns an ERR_PTR() on failure.
  85  */
  86 void *memdup_user(const void __user *src, size_t len)
  87 {
  88         void *p;
  89
  90         /*
  91          * Always use GFP_KERNEL, since copy_from_user() can sleep and
  92          * cause pagefault, which makes it pointless to use GFP_NOFS
  93          * or GFP_ATOMIC.
  94          */
  95         p = kmalloc_track_caller(len, GFP_KERNEL);
  96         if (!p)
  97                 return ERR_PTR(-ENOMEM);
  98
  99         if (copy_from_user(p, src, len)) {
 100                 kfree(p);
 101                 return ERR_PTR(-EFAULT);
 102         }
 103
 104         return p;
 105 }
 106 EXPORT_SYMBOL(memdup_user);
 107
 108 /**
 109  * __krealloc - like krealloc() but don't free @p.
 110  * @p: object to reallocate memory for.
 111  * @new_size: how many bytes of memory are required.
 112  * @flags: the type of memory to allocate.
 113  *
 114  * This function is like krealloc() except it never frees the originally
 115  * allocated buffer. Use this if you don't want to free the buffer immediately
 116  * like, for example, with RCU.
 117  */
 118 void *__krealloc(const void *p, size_t new_size, gfp_t flags)
 119 {
 120         void *ret;
 121         size_t ks = 0;
 122
 123         if (unlikely(!new_size))
 124                 return ZERO_SIZE_PTR;
 125
 126         if (p)
 127                 ks = ksize(p);
 128
 129         if (ks >= new_size)
 130                 return (void *)p;
 131
 132         ret = kmalloc_track_caller(new_size, flags);
 133         if (ret && p)
 134                 memcpy(ret, p, ks);
 135
 136         return ret;
 137 }
 138 EXPORT_SYMBOL(__krealloc);
 139
 140 /**
 141  * krealloc - reallocate memory. The contents will remain unchanged.
 142  * @p: object to reallocate memory for.
 143  * @new_size: how many bytes of memory are required.
 144  * @flags: the type of memory to allocate.
 145  *
 146  * The contents of the object pointed to are preserved up to the
 147  * lesser of the new and old sizes.  If @p is %NULL, krealloc()
 148  * behaves exactly like kmalloc().  If @size is 0 and @p is not a
 149  * %NULL pointer, the object pointed to is freed.
 150  */
 151 void *krealloc(const void *p, size_t new_size, gfp_t flags)
 152 {
 153         void *ret;
 154
 155         if (unlikely(!new_size)) {
 156                 kfree(p);
 157                 return ZERO_SIZE_PTR;
 158         }
 159
 160         ret = __krealloc(p, new_size, flags);
 161         if (ret && p != ret)
 162                 kfree(p);
 163
 164         return ret;
 165 }
 166 EXPORT_SYMBOL(krealloc);
 167
 168 /**
 169  * kzfree - like kfree but zero memory
 170  * @p: object to free memory of
 171  *
 172  * The memory of the object @p points to is zeroed before freed.
 173  * If @p is %NULL, kzfree() does nothing.
 174  *
 175  * Note: this function zeroes the whole allocated buffer which can be a good
 176  * deal bigger than the requested buffer size passed to kmalloc(). So be
 177  * careful when using this function in performance sensitive code.
 178  */
 179 void kzfree(const void *p)
 180 {
 181         size_t ks;
 182         void *mem = (void *)p;
 183
 184         if (unlikely(ZERO_OR_NULL_PTR(mem)))
 185                 return;
 186         ks = ksize(mem);
 187         memset(mem, 0, ks);
 188         kfree(mem);
 189 }
 190 EXPORT_SYMBOL(kzfree);
 191
 192 /*
 193  * strndup_user - duplicate an existing string from user space
 194  * @s: The string to duplicate
 195  * @n: Maximum number of bytes to copy, including the trailing NUL.
 196  */
 197 char *strndup_user(const char __user *s, long n)
 198 {
 199         char *p;
 200         long length;
 201
 202         length = strnlen_user(s, n);
 203
 204         if (!length)
 205                 return ERR_PTR(-EFAULT);
 206
 207         if (length > n)
 208                 return ERR_PTR(-EINVAL);
 209
 210         p = memdup_user(s, length);
 211
 212         if (IS_ERR(p))
 213                 return p;
 214
 215         p[length - 1] = '\0';
 216
 217         return p;
 218 }
 219 EXPORT_SYMBOL(strndup_user);
 220
 221 void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
 222                 struct vm_area_struct *prev, struct rb_node *rb_parent)
 223 {
 224         struct vm_area_struct *next;
 225
 226         vma->vm_prev = prev;
 227         if (prev) {
 228                 next = prev->vm_next;
 229                 prev->vm_next = vma;
 230         } else {
 231                 mm->mmap = vma;
 232                 if (rb_parent)
 233                         next = rb_entry(rb_parent,
 234                                         struct vm_area_struct, vm_rb);
 235                 else
 236                         next = NULL;
 237         }
 238         vma->vm_next = next;
 239         if (next)
 240                 next->vm_prev = vma;
 241 }
 242
 243 /* Check if the vma is being used as a stack by this task */
 244 static int vm_is_stack_for_task(struct task_struct *t,
 245                                 struct vm_area_struct *vma)
 246 {
 247         return (vma->vm_start <= KSTK_ESP(t) && vma->vm_end >= KSTK_ESP(t));
 248 }
 249
 250 /*
 251  * Check if the vma is being used as a stack.
 252  * If is_group is non-zero, check in the entire thread group or else
 253  * just check in the current task. Returns the pid of the task that
 254  * the vma is stack for.
 255  */
 256 pid_t vm_is_stack(struct task_struct *task,
 257                   struct vm_area_struct *vma, int in_group)
 258 {
 259         pid_t ret = 0;
 260
 261         if (vm_is_stack_for_task(task, vma))
 262                 return task->pid;
 263
 264         if (in_group) {
 265                 struct task_struct *t;
 266                 rcu_read_lock();
 267                 if (!pid_alive(task))
 268                         goto done;
 269
 270                 t = task;
 271                 do {
 272                         if (vm_is_stack_for_task(t, vma)) {
 273                                 ret = t->pid;
 274                                 goto done;
 275                         }
 276                 } while_each_thread(task, t);
 277 done:
 278                 rcu_read_unlock();
 279         }
 280
 281         return ret;
 282 }
 283
 284 #if defined(CONFIG_MMU) && !defined(HAVE_ARCH_PICK_MMAP_LAYOUT)
 285 void arch_pick_mmap_layout(struct mm_struct *mm)
 286 {
 287         mm->mmap_base = TASK_UNMAPPED_BASE;
 288         mm->get_unmapped_area = arch_get_unmapped_area;
 289         mm->unmap_area = arch_unmap_area;
 290 }
 291 #endif
 292
 293 /*
 294  * Like get_user_pages_fast() except its IRQ-safe in that it won't fall
 295  * back to the regular GUP.
 296  * If the architecture not support this function, simply return with no
 297  * page pinned
 298  */
 299 int __attribute__((weak)) __get_user_pages_fast(unsigned long start,
 300                                  int nr_pages, int write, struct page **pages)
 301 {
 302         return 0;
 303 }
 304 EXPORT_SYMBOL_GPL(__get_user_pages_fast);
 305
 306 /**
 307  * get_user_pages_fast() - pin user pages in memory
 308  * @start:      starting user address
 309  * @nr_pages:   number of pages from start to pin
 310  * @write:      whether pages will be written to
 311  * @pages:      array that receives pointers to the pages pinned.
 312  *              Should be at least nr_pages long.
 313  *
 314  * Returns number of pages pinned. This may be fewer than the number
 315  * requested. If nr_pages is 0 or negative, returns 0. If no pages
 316  * were pinned, returns -errno.
 317  *
 318  * get_user_pages_fast provides equivalent functionality to get_user_pages,
 319  * operating on current and current->mm, with force=0 and vma=NULL. However
 320  * unlike get_user_pages, it must be called without mmap_sem held.
 321  *
 322  * get_user_pages_fast may take mmap_sem and page table locks, so no
 323  * assumptions can be made about lack of locking. get_user_pages_fast is to be
 324  * implemented in a way that is advantageous (vs get_user_pages()) when the
 325  * user memory area is already faulted in and present in ptes. However if the
 326  * pages have to be faulted in, it may turn out to be slightly slower so
 327  * callers need to carefully consider what to use. On many architectures,
 328  * get_user_pages_fast simply falls back to get_user_pages.
 329  */
 330 int __attribute__((weak)) get_user_pages_fast(unsigned long start,
 331                                 int nr_pages, int write, struct page **pages)
 332 {
 333         struct mm_struct *mm = current->mm;
 334         int ret;
 335
 336         down_read(&mm->mmap_sem);
 337         ret = get_user_pages(current, mm, start, nr_pages,
 338                                         write, 0, pages, NULL);
 339         up_read(&mm->mmap_sem);
 340
 341         return ret;
 342 }
 343 EXPORT_SYMBOL_GPL(get_user_pages_fast);
 344
 345 unsigned long vm_mmap_pgoff(struct file *file, unsigned long addr,
 346         unsigned long len, unsigned long prot,
 347         unsigned long flag, unsigned long pgoff)
 348 {
 349         unsigned long ret;
 350         struct mm_struct *mm = current->mm;
 351
 352         ret = security_mmap_file(file, prot, flag);
 353         if (!ret) {
 354                 down_write(&mm->mmap_sem);
 355                 ret = do_mmap_pgoff(file, addr, len, prot, flag, pgoff);
 356                 up_write(&mm->mmap_sem);
 357         }
 358         return ret;
 359 }
 360
 361 unsigned long vm_mmap(struct file *file, unsigned long addr,
 362         unsigned long len, unsigned long prot,
 363         unsigned long flag, unsigned long offset)
 364 {
 365         if (unlikely(offset + PAGE_ALIGN(len) < offset))
 366                 return -EINVAL;
 367         if (unlikely(offset & ~PAGE_MASK))
 368                 return -EINVAL;
 369
 370         return vm_mmap_pgoff(file, addr, len, prot, flag, offset >> PAGE_SHIFT);
 371 }
 372 EXPORT_SYMBOL(vm_mmap);
 373
 374 /* Tracepoints definitions. */
 375 EXPORT_TRACEPOINT_SYMBOL(kmalloc);
 376 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
 377 EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
 378 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
 379 EXPORT_TRACEPOINT_SYMBOL(kfree);
 380 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);