1 #include <linux/mm.h> 2 #include <linux/slab.h> 3 #include <linux/string.h> 4 #include <linux/module.h> 5 #include <linux/err.h> 6 #include <linux/sched.h> 7 #include <linux/hugetlb.h> 8 #include <linux/syscalls.h> 9 #include <linux/mman.h> 10 #include <linux/file.h> 11 #include <asm/uaccess.h> 12 13 #define CREATE_TRACE_POINTS 14 #include <trace/events/kmem.h> 15 16 /** 17 * kstrdup - allocate space for and copy an existing string 18 * @s: the string to duplicate 19 * @gfp: the GFP mask used in the kmalloc() call when allocating memory 20 */ 21 char *kstrdup(const char *s, gfp_t gfp) 22 { 23 size_t len; 24 char *buf; 25 26 if (!s) 27 return NULL; 28 29 len = strlen(s) + 1; 30 buf = kmalloc_track_caller(len, gfp); 31 if (buf) 32 memcpy(buf, s, len); 33 return buf; 34 } 35 EXPORT_SYMBOL(kstrdup); 36 37 /** 38 * kstrndup - allocate space for and copy an existing string 39 * @s: the string to duplicate 40 * @max: read at most @max chars from @s 41 * @gfp: the GFP mask used in the kmalloc() call when allocating memory 42 */ 43 char *kstrndup(const char *s, size_t max, gfp_t gfp) 44 { 45 size_t len; 46 char *buf; 47 48 if (!s) 49 return NULL; 50 51 len = strnlen(s, max); 52 buf = kmalloc_track_caller(len+1, gfp); 53 if (buf) { 54 memcpy(buf, s, len); 55 buf[len] = '\0'; 56 } 57 return buf; 58 } 59 EXPORT_SYMBOL(kstrndup); 60 61 /** 62 * kmemdup - duplicate region of memory 63 * 64 * @src: memory region to duplicate 65 * @len: memory region length 66 * @gfp: GFP mask to use 67 */ 68 void *kmemdup(const void *src, size_t len, gfp_t gfp) 69 { 70 void *p; 71 72 p = kmalloc_track_caller(len, gfp); 73 if (p) 74 memcpy(p, src, len); 75 return p; 76 } 77 EXPORT_SYMBOL(kmemdup); 78 79 /** 80 * memdup_user - duplicate memory region from user space 81 * 82 * @src: source address in user space 83 * @len: number of bytes to copy 84 * 85 * Returns an ERR_PTR() on failure. 86 */ 87 void *memdup_user(const void __user *src, size_t len) 88 { 89 void *p; 90 91 /* 92 * Always use GFP_KERNEL, since copy_from_user() can sleep and 93 * cause pagefault, which makes it pointless to use GFP_NOFS 94 * or GFP_ATOMIC. 95 */ 96 p = kmalloc_track_caller(len, GFP_KERNEL); 97 if (!p) 98 return ERR_PTR(-ENOMEM); 99 100 if (copy_from_user(p, src, len)) { 101 kfree(p); 102 return ERR_PTR(-EFAULT); 103 } 104 105 return p; 106 } 107 EXPORT_SYMBOL(memdup_user); 108 109 /** 110 * __krealloc - like krealloc() but don't free @p. 111 * @p: object to reallocate memory for. 112 * @new_size: how many bytes of memory are required. 113 * @flags: the type of memory to allocate. 114 * 115 * This function is like krealloc() except it never frees the originally 116 * allocated buffer. Use this if you don't want to free the buffer immediately 117 * like, for example, with RCU. 118 */ 119 void *__krealloc(const void *p, size_t new_size, gfp_t flags) 120 { 121 void *ret; 122 size_t ks = 0; 123 124 if (unlikely(!new_size)) 125 return ZERO_SIZE_PTR; 126 127 if (p) 128 ks = ksize(p); 129 130 if (ks >= new_size) 131 return (void *)p; 132 133 ret = kmalloc_track_caller(new_size, flags); 134 if (ret && p) 135 memcpy(ret, p, ks); 136 137 return ret; 138 } 139 EXPORT_SYMBOL(__krealloc); 140 141 /** 142 * krealloc - reallocate memory. The contents will remain unchanged. 143 * @p: object to reallocate memory for. 144 * @new_size: how many bytes of memory are required. 145 * @flags: the type of memory to allocate. 146 * 147 * The contents of the object pointed to are preserved up to the 148 * lesser of the new and old sizes. If @p is %NULL, krealloc() 149 * behaves exactly like kmalloc(). If @size is 0 and @p is not a 150 * %NULL pointer, the object pointed to is freed. 151 */ 152 void *krealloc(const void *p, size_t new_size, gfp_t flags) 153 { 154 void *ret; 155 156 if (unlikely(!new_size)) { 157 kfree(p); 158 return ZERO_SIZE_PTR; 159 } 160 161 ret = __krealloc(p, new_size, flags); 162 if (ret && p != ret) 163 kfree(p); 164 165 return ret; 166 } 167 EXPORT_SYMBOL(krealloc); 168 169 /** 170 * kzfree - like kfree but zero memory 171 * @p: object to free memory of 172 * 173 * The memory of the object @p points to is zeroed before freed. 174 * If @p is %NULL, kzfree() does nothing. 175 * 176 * Note: this function zeroes the whole allocated buffer which can be a good 177 * deal bigger than the requested buffer size passed to kmalloc(). So be 178 * careful when using this function in performance sensitive code. 179 */ 180 void kzfree(const void *p) 181 { 182 size_t ks; 183 void *mem = (void *)p; 184 185 if (unlikely(ZERO_OR_NULL_PTR(mem))) 186 return; 187 ks = ksize(mem); 188 memset(mem, 0, ks); 189 kfree(mem); 190 } 191 EXPORT_SYMBOL(kzfree); 192 193 /* 194 * strndup_user - duplicate an existing string from user space 195 * @s: The string to duplicate 196 * @n: Maximum number of bytes to copy, including the trailing NUL. 197 */ 198 char *strndup_user(const char __user *s, long n) 199 { 200 char *p; 201 long length; 202 203 length = strnlen_user(s, n); 204 205 if (!length) 206 return ERR_PTR(-EFAULT); 207 208 if (length > n) 209 return ERR_PTR(-EINVAL); 210 211 p = kmalloc(length, GFP_KERNEL); 212 213 if (!p) 214 return ERR_PTR(-ENOMEM); 215 216 if (copy_from_user(p, s, length)) { 217 kfree(p); 218 return ERR_PTR(-EFAULT); 219 } 220 221 p[length - 1] = '\0'; 222 223 return p; 224 } 225 EXPORT_SYMBOL(strndup_user); 226 227 #ifndef HAVE_ARCH_PICK_MMAP_LAYOUT 228 void arch_pick_mmap_layout(struct mm_struct *mm) 229 { 230 mm->mmap_base = TASK_UNMAPPED_BASE; 231 mm->get_unmapped_area = arch_get_unmapped_area; 232 mm->unmap_area = arch_unmap_area; 233 } 234 #endif 235 236 /** 237 * get_user_pages_fast() - pin user pages in memory 238 * @start: starting user address 239 * @nr_pages: number of pages from start to pin 240 * @write: whether pages will be written to 241 * @pages: array that receives pointers to the pages pinned. 242 * Should be at least nr_pages long. 243 * 244 * Returns number of pages pinned. This may be fewer than the number 245 * requested. If nr_pages is 0 or negative, returns 0. If no pages 246 * were pinned, returns -errno. 247 * 248 * get_user_pages_fast provides equivalent functionality to get_user_pages, 249 * operating on current and current->mm, with force=0 and vma=NULL. However 250 * unlike get_user_pages, it must be called without mmap_sem held. 251 * 252 * get_user_pages_fast may take mmap_sem and page table locks, so no 253 * assumptions can be made about lack of locking. get_user_pages_fast is to be 254 * implemented in a way that is advantageous (vs get_user_pages()) when the 255 * user memory area is already faulted in and present in ptes. However if the 256 * pages have to be faulted in, it may turn out to be slightly slower so 257 * callers need to carefully consider what to use. On many architectures, 258 * get_user_pages_fast simply falls back to get_user_pages. 259 */ 260 int __attribute__((weak)) get_user_pages_fast(unsigned long start, 261 int nr_pages, int write, struct page **pages) 262 { 263 struct mm_struct *mm = current->mm; 264 int ret; 265 266 down_read(&mm->mmap_sem); 267 ret = get_user_pages(current, mm, start, nr_pages, 268 write, 0, pages, NULL); 269 up_read(&mm->mmap_sem); 270 271 return ret; 272 } 273 EXPORT_SYMBOL_GPL(get_user_pages_fast); 274 275 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, 276 unsigned long, prot, unsigned long, flags, 277 unsigned long, fd, unsigned long, pgoff) 278 { 279 struct file * file = NULL; 280 unsigned long retval = -EBADF; 281 282 if (!(flags & MAP_ANONYMOUS)) { 283 if (unlikely(flags & MAP_HUGETLB)) 284 return -EINVAL; 285 file = fget(fd); 286 if (!file) 287 goto out; 288 } else if (flags & MAP_HUGETLB) { 289 struct user_struct *user = NULL; 290 /* 291 * VM_NORESERVE is used because the reservations will be 292 * taken when vm_ops->mmap() is called 293 * A dummy user value is used because we are not locking 294 * memory so no accounting is necessary 295 */ 296 len = ALIGN(len, huge_page_size(&default_hstate)); 297 file = hugetlb_file_setup(HUGETLB_ANON_FILE, len, VM_NORESERVE, 298 &user, HUGETLB_ANONHUGE_INODE); 299 if (IS_ERR(file)) 300 return PTR_ERR(file); 301 } 302 303 flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE); 304 305 down_write(¤t->mm->mmap_sem); 306 retval = do_mmap_pgoff(file, addr, len, prot, flags, pgoff); 307 up_write(¤t->mm->mmap_sem); 308 309 if (file) 310 fput(file); 311 out: 312 return retval; 313 } 314 315 /* Tracepoints definitions. */ 316 EXPORT_TRACEPOINT_SYMBOL(kmalloc); 317 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc); 318 EXPORT_TRACEPOINT_SYMBOL(kmalloc_node); 319 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc_node); 320 EXPORT_TRACEPOINT_SYMBOL(kfree); 321 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free); 322