mm/util.c

   1 #include <linux/mm.h>
   2 #include <linux/slab.h>
   3 #include <linux/string.h>
   4 #include <linux/module.h>
   5 #include <linux/err.h>
   6 #include <linux/sched.h>
   7 #include <linux/hugetlb.h>
   8 #include <linux/syscalls.h>
   9 #include <linux/mman.h>
  10 #include <linux/file.h>
  11 #include <asm/uaccess.h>
  12
  13 #define CREATE_TRACE_POINTS
  14 #include <trace/events/kmem.h>
  15
  16 /**
  17  * kstrdup - allocate space for and copy an existing string
  18  * @s: the string to duplicate
  19  * @gfp: the GFP mask used in the kmalloc() call when allocating memory
  20  */
  21 char *kstrdup(const char *s, gfp_t gfp)
  22 {
  23         size_t len;
  24         char *buf;
  25
  26         if (!s)
  27                 return NULL;
  28
  29         len = strlen(s) + 1;
  30         buf = kmalloc_track_caller(len, gfp);
  31         if (buf)
  32                 memcpy(buf, s, len);
  33         return buf;
  34 }
  35 EXPORT_SYMBOL(kstrdup);
  36
  37 /**
  38  * kstrndup - allocate space for and copy an existing string
  39  * @s: the string to duplicate
  40  * @max: read at most @max chars from @s
  41  * @gfp: the GFP mask used in the kmalloc() call when allocating memory
  42  */
  43 char *kstrndup(const char *s, size_t max, gfp_t gfp)
  44 {
  45         size_t len;
  46         char *buf;
  47
  48         if (!s)
  49                 return NULL;
  50
  51         len = strnlen(s, max);
  52         buf = kmalloc_track_caller(len+1, gfp);
  53         if (buf) {
  54                 memcpy(buf, s, len);
  55                 buf[len] = '\0';
  56         }
  57         return buf;
  58 }
  59 EXPORT_SYMBOL(kstrndup);
  60
  61 /**
  62  * kmemdup - duplicate region of memory
  63  *
  64  * @src: memory region to duplicate
  65  * @len: memory region length
  66  * @gfp: GFP mask to use
  67  */
  68 void *kmemdup(const void *src, size_t len, gfp_t gfp)
  69 {
  70         void *p;
  71
  72         p = kmalloc_track_caller(len, gfp);
  73         if (p)
  74                 memcpy(p, src, len);
  75         return p;
  76 }
  77 EXPORT_SYMBOL(kmemdup);
  78
  79 /**
  80  * memdup_user - duplicate memory region from user space
  81  *
  82  * @src: source address in user space
  83  * @len: number of bytes to copy
  84  *
  85  * Returns an ERR_PTR() on failure.
  86  */
  87 void *memdup_user(const void __user *src, size_t len)
  88 {
  89         void *p;
  90
  91         /*
  92          * Always use GFP_KERNEL, since copy_from_user() can sleep and
  93          * cause pagefault, which makes it pointless to use GFP_NOFS
  94          * or GFP_ATOMIC.
  95          */
  96         p = kmalloc_track_caller(len, GFP_KERNEL);
  97         if (!p)
  98                 return ERR_PTR(-ENOMEM);
  99
 100         if (copy_from_user(p, src, len)) {
 101                 kfree(p);
 102                 return ERR_PTR(-EFAULT);
 103         }
 104
 105         return p;
 106 }
 107 EXPORT_SYMBOL(memdup_user);
 108
 109 /**
 110  * __krealloc - like krealloc() but don't free @p.
 111  * @p: object to reallocate memory for.
 112  * @new_size: how many bytes of memory are required.
 113  * @flags: the type of memory to allocate.
 114  *
 115  * This function is like krealloc() except it never frees the originally
 116  * allocated buffer. Use this if you don't want to free the buffer immediately
 117  * like, for example, with RCU.
 118  */
 119 void *__krealloc(const void *p, size_t new_size, gfp_t flags)
 120 {
 121         void *ret;
 122         size_t ks = 0;
 123
 124         if (unlikely(!new_size))
 125                 return ZERO_SIZE_PTR;
 126
 127         if (p)
 128                 ks = ksize(p);
 129
 130         if (ks >= new_size)
 131                 return (void *)p;
 132
 133         ret = kmalloc_track_caller(new_size, flags);
 134         if (ret && p)
 135                 memcpy(ret, p, ks);
 136
 137         return ret;
 138 }
 139 EXPORT_SYMBOL(__krealloc);
 140
 141 /**
 142  * krealloc - reallocate memory. The contents will remain unchanged.
 143  * @p: object to reallocate memory for.
 144  * @new_size: how many bytes of memory are required.
 145  * @flags: the type of memory to allocate.
 146  *
 147  * The contents of the object pointed to are preserved up to the
 148  * lesser of the new and old sizes.  If @p is %NULL, krealloc()
 149  * behaves exactly like kmalloc().  If @size is 0 and @p is not a
 150  * %NULL pointer, the object pointed to is freed.
 151  */
 152 void *krealloc(const void *p, size_t new_size, gfp_t flags)
 153 {
 154         void *ret;
 155
 156         if (unlikely(!new_size)) {
 157                 kfree(p);
 158                 return ZERO_SIZE_PTR;
 159         }
 160
 161         ret = __krealloc(p, new_size, flags);
 162         if (ret && p != ret)
 163                 kfree(p);
 164
 165         return ret;
 166 }
 167 EXPORT_SYMBOL(krealloc);
 168
 169 /**
 170  * kzfree - like kfree but zero memory
 171  * @p: object to free memory of
 172  *
 173  * The memory of the object @p points to is zeroed before freed.
 174  * If @p is %NULL, kzfree() does nothing.
 175  *
 176  * Note: this function zeroes the whole allocated buffer which can be a good
 177  * deal bigger than the requested buffer size passed to kmalloc(). So be
 178  * careful when using this function in performance sensitive code.
 179  */
 180 void kzfree(const void *p)
 181 {
 182         size_t ks;
 183         void *mem = (void *)p;
 184
 185         if (unlikely(ZERO_OR_NULL_PTR(mem)))
 186                 return;
 187         ks = ksize(mem);
 188         memset(mem, 0, ks);
 189         kfree(mem);
 190 }
 191 EXPORT_SYMBOL(kzfree);
 192
 193 /*
 194  * strndup_user - duplicate an existing string from user space
 195  * @s: The string to duplicate
 196  * @n: Maximum number of bytes to copy, including the trailing NUL.
 197  */
 198 char *strndup_user(const char __user *s, long n)
 199 {
 200         char *p;
 201         long length;
 202
 203         length = strnlen_user(s, n);
 204
 205         if (!length)
 206                 return ERR_PTR(-EFAULT);
 207
 208         if (length > n)
 209                 return ERR_PTR(-EINVAL);
 210
 211         p = kmalloc(length, GFP_KERNEL);
 212
 213         if (!p)
 214                 return ERR_PTR(-ENOMEM);
 215
 216         if (copy_from_user(p, s, length)) {
 217                 kfree(p);
 218                 return ERR_PTR(-EFAULT);
 219         }
 220
 221         p[length - 1] = '\0';
 222
 223         return p;
 224 }
 225 EXPORT_SYMBOL(strndup_user);
 226
 227 #ifndef HAVE_ARCH_PICK_MMAP_LAYOUT
 228 void arch_pick_mmap_layout(struct mm_struct *mm)
 229 {
 230         mm->mmap_base = TASK_UNMAPPED_BASE;
 231         mm->get_unmapped_area = arch_get_unmapped_area;
 232         mm->unmap_area = arch_unmap_area;
 233 }
 234 #endif
 235
 236 /*
 237  * Like get_user_pages_fast() except its IRQ-safe in that it won't fall
 238  * back to the regular GUP.
 239  * If the architecture not support this fucntion, simply return with no
 240  * page pinned
 241  */
 242 int __attribute__((weak)) __get_user_pages_fast(unsigned long start,
 243                                  int nr_pages, int write, struct page **pages)
 244 {
 245         return 0;
 246 }
 247 EXPORT_SYMBOL_GPL(__get_user_pages_fast);
 248
 249 /**
 250  * get_user_pages_fast() - pin user pages in memory
 251  * @start:      starting user address
 252  * @nr_pages:   number of pages from start to pin
 253  * @write:      whether pages will be written to
 254  * @pages:      array that receives pointers to the pages pinned.
 255  *              Should be at least nr_pages long.
 256  *
 257  * Returns number of pages pinned. This may be fewer than the number
 258  * requested. If nr_pages is 0 or negative, returns 0. If no pages
 259  * were pinned, returns -errno.
 260  *
 261  * get_user_pages_fast provides equivalent functionality to get_user_pages,
 262  * operating on current and current->mm, with force=0 and vma=NULL. However
 263  * unlike get_user_pages, it must be called without mmap_sem held.
 264  *
 265  * get_user_pages_fast may take mmap_sem and page table locks, so no
 266  * assumptions can be made about lack of locking. get_user_pages_fast is to be
 267  * implemented in a way that is advantageous (vs get_user_pages()) when the
 268  * user memory area is already faulted in and present in ptes. However if the
 269  * pages have to be faulted in, it may turn out to be slightly slower so
 270  * callers need to carefully consider what to use. On many architectures,
 271  * get_user_pages_fast simply falls back to get_user_pages.
 272  */
 273 int __attribute__((weak)) get_user_pages_fast(unsigned long start,
 274                                 int nr_pages, int write, struct page **pages)
 275 {
 276         struct mm_struct *mm = current->mm;
 277         int ret;
 278
 279         down_read(&mm->mmap_sem);
 280         ret = get_user_pages(current, mm, start, nr_pages,
 281                                         write, 0, pages, NULL);
 282         up_read(&mm->mmap_sem);
 283
 284         return ret;
 285 }
 286 EXPORT_SYMBOL_GPL(get_user_pages_fast);
 287
 288 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len,
 289                 unsigned long, prot, unsigned long, flags,
 290                 unsigned long, fd, unsigned long, pgoff)
 291 {
 292         struct file * file = NULL;
 293         unsigned long retval = -EBADF;
 294
 295         if (!(flags & MAP_ANONYMOUS)) {
 296                 if (unlikely(flags & MAP_HUGETLB))
 297                         return -EINVAL;
 298                 file = fget(fd);
 299                 if (!file)
 300                         goto out;
 301         } else if (flags & MAP_HUGETLB) {
 302                 struct user_struct *user = NULL;
 303                 /*
 304                  * VM_NORESERVE is used because the reservations will be
 305                  * taken when vm_ops->mmap() is called
 306                  * A dummy user value is used because we are not locking
 307                  * memory so no accounting is necessary
 308                  */
 309                 len = ALIGN(len, huge_page_size(&default_hstate));
 310                 file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, VM_NORESERVE,
 311                                                 &user, HUGETLB_ANONHUGE_INODE);
 312                 if (IS_ERR(file))
 313                         return PTR_ERR(file);
 314         }
 315
 316         flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE);
 317
 318         down_write(&current->mm->mmap_sem);
 319         retval = do_mmap_pgoff(file, addr, len, prot, flags, pgoff);
 320         up_write(&current->mm->mmap_sem);
 321
 322         if (file)
 323                 fput(file);
 324 out:
 325         return retval;
 326 }
 327
 328 /* Tracepoints definitions. */
 329 EXPORT_TRACEPOINT_SYMBOL(kmalloc);
 330 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
 331 EXPORT_TRACEPOINT_SYMBOL(kmalloc_node);
 332 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node);
 333 EXPORT_TRACEPOINT_SYMBOL(kfree);
 334 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);