1 /* 2 * linux/mm/nommu.c 3 * 4 * Replacement code for mm functions to support CPU's that don't 5 * have any form of memory management unit (thus no virtual memory). 6 * 7 * See Documentation/nommu-mmap.txt 8 * 9 * Copyright (c) 2004-2008 David Howells <dhowells@redhat.com> 10 * Copyright (c) 2000-2003 David McCullough <davidm@snapgear.com> 11 * Copyright (c) 2000-2001 D Jeff Dionne <jeff@uClinux.org> 12 * Copyright (c) 2002 Greg Ungerer <gerg@snapgear.com> 13 * Copyright (c) 2007-2010 Paul Mundt <lethal@linux-sh.org> 14 */ 15 16 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 17 18 #include <linux/export.h> 19 #include <linux/mm.h> 20 #include <linux/sched/mm.h> 21 #include <linux/vmacache.h> 22 #include <linux/mman.h> 23 #include <linux/swap.h> 24 #include <linux/file.h> 25 #include <linux/highmem.h> 26 #include <linux/pagemap.h> 27 #include <linux/slab.h> 28 #include <linux/vmalloc.h> 29 #include <linux/blkdev.h> 30 #include <linux/backing-dev.h> 31 #include <linux/compiler.h> 32 #include <linux/mount.h> 33 #include <linux/personality.h> 34 #include <linux/security.h> 35 #include <linux/syscalls.h> 36 #include <linux/audit.h> 37 #include <linux/printk.h> 38 39 #include <linux/uaccess.h> 40 #include <asm/tlb.h> 41 #include <asm/tlbflush.h> 42 #include <asm/mmu_context.h> 43 #include "internal.h" 44 45 void *high_memory; 46 EXPORT_SYMBOL(high_memory); 47 struct page *mem_map; 48 unsigned long max_mapnr; 49 EXPORT_SYMBOL(max_mapnr); 50 unsigned long highest_memmap_pfn; 51 int sysctl_nr_trim_pages = CONFIG_NOMMU_INITIAL_TRIM_EXCESS; 52 int heap_stack_gap = 0; 53 54 atomic_long_t mmap_pages_allocated; 55 56 EXPORT_SYMBOL(mem_map); 57 58 /* list of mapped, potentially shareable regions */ 59 static struct kmem_cache *vm_region_jar; 60 struct rb_root nommu_region_tree = RB_ROOT; 61 DECLARE_RWSEM(nommu_region_sem); 62 63 const struct vm_operations_struct generic_file_vm_ops = { 64 }; 65 66 /* 67 * Return the total memory allocated for this pointer, not 68 * just what the caller asked for. 69 * 70 * Doesn't have to be accurate, i.e. may have races. 71 */ 72 unsigned int kobjsize(const void *objp) 73 { 74 struct page *page; 75 76 /* 77 * If the object we have should not have ksize performed on it, 78 * return size of 0 79 */ 80 if (!objp || !virt_addr_valid(objp)) 81 return 0; 82 83 page = virt_to_head_page(objp); 84 85 /* 86 * If the allocator sets PageSlab, we know the pointer came from 87 * kmalloc(). 88 */ 89 if (PageSlab(page)) 90 return ksize(objp); 91 92 /* 93 * If it's not a compound page, see if we have a matching VMA 94 * region. This test is intentionally done in reverse order, 95 * so if there's no VMA, we still fall through and hand back 96 * PAGE_SIZE for 0-order pages. 97 */ 98 if (!PageCompound(page)) { 99 struct vm_area_struct *vma; 100 101 vma = find_vma(current->mm, (unsigned long)objp); 102 if (vma) 103 return vma->vm_end - vma->vm_start; 104 } 105 106 /* 107 * The ksize() function is only guaranteed to work for pointers 108 * returned by kmalloc(). So handle arbitrary pointers here. 109 */ 110 return PAGE_SIZE << compound_order(page); 111 } 112 113 static long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 114 unsigned long start, unsigned long nr_pages, 115 unsigned int foll_flags, struct page **pages, 116 struct vm_area_struct **vmas, int *nonblocking) 117 { 118 struct vm_area_struct *vma; 119 unsigned long vm_flags; 120 int i; 121 122 /* calculate required read or write permissions. 123 * If FOLL_FORCE is set, we only require the "MAY" flags. 124 */ 125 vm_flags = (foll_flags & FOLL_WRITE) ? 126 (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); 127 vm_flags &= (foll_flags & FOLL_FORCE) ? 128 (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); 129 130 for (i = 0; i < nr_pages; i++) { 131 vma = find_vma(mm, start); 132 if (!vma) 133 goto finish_or_fault; 134 135 /* protect what we can, including chardevs */ 136 if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) || 137 !(vm_flags & vma->vm_flags)) 138 goto finish_or_fault; 139 140 if (pages) { 141 pages[i] = virt_to_page(start); 142 if (pages[i]) 143 get_page(pages[i]); 144 } 145 if (vmas) 146 vmas[i] = vma; 147 start = (start + PAGE_SIZE) & PAGE_MASK; 148 } 149 150 return i; 151 152 finish_or_fault: 153 return i ? : -EFAULT; 154 } 155 156 /* 157 * get a list of pages in an address range belonging to the specified process 158 * and indicate the VMA that covers each page 159 * - this is potentially dodgy as we may end incrementing the page count of a 160 * slab page or a secondary page from a compound page 161 * - don't permit access to VMAs that don't support it, such as I/O mappings 162 */ 163 long get_user_pages(unsigned long start, unsigned long nr_pages, 164 unsigned int gup_flags, struct page **pages, 165 struct vm_area_struct **vmas) 166 { 167 return __get_user_pages(current, current->mm, start, nr_pages, 168 gup_flags, pages, vmas, NULL); 169 } 170 EXPORT_SYMBOL(get_user_pages); 171 172 long get_user_pages_locked(unsigned long start, unsigned long nr_pages, 173 unsigned int gup_flags, struct page **pages, 174 int *locked) 175 { 176 return get_user_pages(start, nr_pages, gup_flags, pages, NULL); 177 } 178 EXPORT_SYMBOL(get_user_pages_locked); 179 180 static long __get_user_pages_unlocked(struct task_struct *tsk, 181 struct mm_struct *mm, unsigned long start, 182 unsigned long nr_pages, struct page **pages, 183 unsigned int gup_flags) 184 { 185 long ret; 186 down_read(&mm->mmap_sem); 187 ret = __get_user_pages(tsk, mm, start, nr_pages, gup_flags, pages, 188 NULL, NULL); 189 up_read(&mm->mmap_sem); 190 return ret; 191 } 192 193 long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages, 194 struct page **pages, unsigned int gup_flags) 195 { 196 return __get_user_pages_unlocked(current, current->mm, start, nr_pages, 197 pages, gup_flags); 198 } 199 EXPORT_SYMBOL(get_user_pages_unlocked); 200 201 /** 202 * follow_pfn - look up PFN at a user virtual address 203 * @vma: memory mapping 204 * @address: user virtual address 205 * @pfn: location to store found PFN 206 * 207 * Only IO mappings and raw PFN mappings are allowed. 208 * 209 * Returns zero and the pfn at @pfn on success, -ve otherwise. 210 */ 211 int follow_pfn(struct vm_area_struct *vma, unsigned long address, 212 unsigned long *pfn) 213 { 214 if (!(vma->vm_flags & (VM_IO | VM_PFNMAP))) 215 return -EINVAL; 216 217 *pfn = address >> PAGE_SHIFT; 218 return 0; 219 } 220 EXPORT_SYMBOL(follow_pfn); 221 222 LIST_HEAD(vmap_area_list); 223 224 void vfree(const void *addr) 225 { 226 kfree(addr); 227 } 228 EXPORT_SYMBOL(vfree); 229 230 void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) 231 { 232 /* 233 * You can't specify __GFP_HIGHMEM with kmalloc() since kmalloc() 234 * returns only a logical address. 235 */ 236 return kmalloc(size, (gfp_mask | __GFP_COMP) & ~__GFP_HIGHMEM); 237 } 238 EXPORT_SYMBOL(__vmalloc); 239 240 void *__vmalloc_node_flags(unsigned long size, int node, gfp_t flags) 241 { 242 return __vmalloc(size, flags, PAGE_KERNEL); 243 } 244 245 void *vmalloc_user(unsigned long size) 246 { 247 void *ret; 248 249 ret = __vmalloc(size, GFP_KERNEL | __GFP_ZERO, PAGE_KERNEL); 250 if (ret) { 251 struct vm_area_struct *vma; 252 253 down_write(¤t->mm->mmap_sem); 254 vma = find_vma(current->mm, (unsigned long)ret); 255 if (vma) 256 vma->vm_flags |= VM_USERMAP; 257 up_write(¤t->mm->mmap_sem); 258 } 259 260 return ret; 261 } 262 EXPORT_SYMBOL(vmalloc_user); 263 264 struct page *vmalloc_to_page(const void *addr) 265 { 266 return virt_to_page(addr); 267 } 268 EXPORT_SYMBOL(vmalloc_to_page); 269 270 unsigned long vmalloc_to_pfn(const void *addr) 271 { 272 return page_to_pfn(virt_to_page(addr)); 273 } 274 EXPORT_SYMBOL(vmalloc_to_pfn); 275 276 long vread(char *buf, char *addr, unsigned long count) 277 { 278 /* Don't allow overflow */ 279 if ((unsigned long) buf + count < count) 280 count = -(unsigned long) buf; 281 282 memcpy(buf, addr, count); 283 return count; 284 } 285 286 long vwrite(char *buf, char *addr, unsigned long count) 287 { 288 /* Don't allow overflow */ 289 if ((unsigned long) addr + count < count) 290 count = -(unsigned long) addr; 291 292 memcpy(addr, buf, count); 293 return count; 294 } 295 296 /* 297 * vmalloc - allocate virtually contiguous memory 298 * 299 * @size: allocation size 300 * 301 * Allocate enough pages to cover @size from the page level 302 * allocator and map them into contiguous kernel virtual space. 303 * 304 * For tight control over page level allocator and protection flags 305 * use __vmalloc() instead. 306 */ 307 void *vmalloc(unsigned long size) 308 { 309 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL); 310 } 311 EXPORT_SYMBOL(vmalloc); 312 313 /* 314 * vzalloc - allocate virtually contiguous memory with zero fill 315 * 316 * @size: allocation size 317 * 318 * Allocate enough pages to cover @size from the page level 319 * allocator and map them into contiguous kernel virtual space. 320 * The memory allocated is set to zero. 321 * 322 * For tight control over page level allocator and protection flags 323 * use __vmalloc() instead. 324 */ 325 void *vzalloc(unsigned long size) 326 { 327 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, 328 PAGE_KERNEL); 329 } 330 EXPORT_SYMBOL(vzalloc); 331 332 /** 333 * vmalloc_node - allocate memory on a specific node 334 * @size: allocation size 335 * @node: numa node 336 * 337 * Allocate enough pages to cover @size from the page level 338 * allocator and map them into contiguous kernel virtual space. 339 * 340 * For tight control over page level allocator and protection flags 341 * use __vmalloc() instead. 342 */ 343 void *vmalloc_node(unsigned long size, int node) 344 { 345 return vmalloc(size); 346 } 347 EXPORT_SYMBOL(vmalloc_node); 348 349 /** 350 * vzalloc_node - allocate memory on a specific node with zero fill 351 * @size: allocation size 352 * @node: numa node 353 * 354 * Allocate enough pages to cover @size from the page level 355 * allocator and map them into contiguous kernel virtual space. 356 * The memory allocated is set to zero. 357 * 358 * For tight control over page level allocator and protection flags 359 * use __vmalloc() instead. 360 */ 361 void *vzalloc_node(unsigned long size, int node) 362 { 363 return vzalloc(size); 364 } 365 EXPORT_SYMBOL(vzalloc_node); 366 367 #ifndef PAGE_KERNEL_EXEC 368 # define PAGE_KERNEL_EXEC PAGE_KERNEL 369 #endif 370 371 /** 372 * vmalloc_exec - allocate virtually contiguous, executable memory 373 * @size: allocation size 374 * 375 * Kernel-internal function to allocate enough pages to cover @size 376 * the page level allocator and map them into contiguous and 377 * executable kernel virtual space. 378 * 379 * For tight control over page level allocator and protection flags 380 * use __vmalloc() instead. 381 */ 382 383 void *vmalloc_exec(unsigned long size) 384 { 385 return __vmalloc(size, GFP_KERNEL | __GFP_HIGHMEM, PAGE_KERNEL_EXEC); 386 } 387 388 /** 389 * vmalloc_32 - allocate virtually contiguous memory (32bit addressable) 390 * @size: allocation size 391 * 392 * Allocate enough 32bit PA addressable pages to cover @size from the 393 * page level allocator and map them into contiguous kernel virtual space. 394 */ 395 void *vmalloc_32(unsigned long size) 396 { 397 return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL); 398 } 399 EXPORT_SYMBOL(vmalloc_32); 400 401 /** 402 * vmalloc_32_user - allocate zeroed virtually contiguous 32bit memory 403 * @size: allocation size 404 * 405 * The resulting memory area is 32bit addressable and zeroed so it can be 406 * mapped to userspace without leaking data. 407 * 408 * VM_USERMAP is set on the corresponding VMA so that subsequent calls to 409 * remap_vmalloc_range() are permissible. 410 */ 411 void *vmalloc_32_user(unsigned long size) 412 { 413 /* 414 * We'll have to sort out the ZONE_DMA bits for 64-bit, 415 * but for now this can simply use vmalloc_user() directly. 416 */ 417 return vmalloc_user(size); 418 } 419 EXPORT_SYMBOL(vmalloc_32_user); 420 421 void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot) 422 { 423 BUG(); 424 return NULL; 425 } 426 EXPORT_SYMBOL(vmap); 427 428 void vunmap(const void *addr) 429 { 430 BUG(); 431 } 432 EXPORT_SYMBOL(vunmap); 433 434 void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot) 435 { 436 BUG(); 437 return NULL; 438 } 439 EXPORT_SYMBOL(vm_map_ram); 440 441 void vm_unmap_ram(const void *mem, unsigned int count) 442 { 443 BUG(); 444 } 445 EXPORT_SYMBOL(vm_unmap_ram); 446 447 void vm_unmap_aliases(void) 448 { 449 } 450 EXPORT_SYMBOL_GPL(vm_unmap_aliases); 451 452 /* 453 * Implement a stub for vmalloc_sync_all() if the architecture chose not to 454 * have one. 455 */ 456 void __weak vmalloc_sync_all(void) 457 { 458 } 459 460 /** 461 * alloc_vm_area - allocate a range of kernel address space 462 * @size: size of the area 463 * 464 * Returns: NULL on failure, vm_struct on success 465 * 466 * This function reserves a range of kernel address space, and 467 * allocates pagetables to map that range. No actual mappings 468 * are created. If the kernel address space is not shared 469 * between processes, it syncs the pagetable across all 470 * processes. 471 */ 472 struct vm_struct *alloc_vm_area(size_t size, pte_t **ptes) 473 { 474 BUG(); 475 return NULL; 476 } 477 EXPORT_SYMBOL_GPL(alloc_vm_area); 478 479 void free_vm_area(struct vm_struct *area) 480 { 481 BUG(); 482 } 483 EXPORT_SYMBOL_GPL(free_vm_area); 484 485 int vm_insert_page(struct vm_area_struct *vma, unsigned long addr, 486 struct page *page) 487 { 488 return -EINVAL; 489 } 490 EXPORT_SYMBOL(vm_insert_page); 491 492 /* 493 * sys_brk() for the most part doesn't need the global kernel 494 * lock, except when an application is doing something nasty 495 * like trying to un-brk an area that has already been mapped 496 * to a regular file. in this case, the unmapping will need 497 * to invoke file system routines that need the global lock. 498 */ 499 SYSCALL_DEFINE1(brk, unsigned long, brk) 500 { 501 struct mm_struct *mm = current->mm; 502 503 if (brk < mm->start_brk || brk > mm->context.end_brk) 504 return mm->brk; 505 506 if (mm->brk == brk) 507 return mm->brk; 508 509 /* 510 * Always allow shrinking brk 511 */ 512 if (brk <= mm->brk) { 513 mm->brk = brk; 514 return brk; 515 } 516 517 /* 518 * Ok, looks good - let it rip. 519 */ 520 flush_icache_range(mm->brk, brk); 521 return mm->brk = brk; 522 } 523 524 /* 525 * initialise the percpu counter for VM and region record slabs 526 */ 527 void __init mmap_init(void) 528 { 529 int ret; 530 531 ret = percpu_counter_init(&vm_committed_as, 0, GFP_KERNEL); 532 VM_BUG_ON(ret); 533 vm_region_jar = KMEM_CACHE(vm_region, SLAB_PANIC|SLAB_ACCOUNT); 534 } 535 536 /* 537 * validate the region tree 538 * - the caller must hold the region lock 539 */ 540 #ifdef CONFIG_DEBUG_NOMMU_REGIONS 541 static noinline void validate_nommu_regions(void) 542 { 543 struct vm_region *region, *last; 544 struct rb_node *p, *lastp; 545 546 lastp = rb_first(&nommu_region_tree); 547 if (!lastp) 548 return; 549 550 last = rb_entry(lastp, struct vm_region, vm_rb); 551 BUG_ON(last->vm_end <= last->vm_start); 552 BUG_ON(last->vm_top < last->vm_end); 553 554 while ((p = rb_next(lastp))) { 555 region = rb_entry(p, struct vm_region, vm_rb); 556 last = rb_entry(lastp, struct vm_region, vm_rb); 557 558 BUG_ON(region->vm_end <= region->vm_start); 559 BUG_ON(region->vm_top < region->vm_end); 560 BUG_ON(region->vm_start < last->vm_top); 561 562 lastp = p; 563 } 564 } 565 #else 566 static void validate_nommu_regions(void) 567 { 568 } 569 #endif 570 571 /* 572 * add a region into the global tree 573 */ 574 static void add_nommu_region(struct vm_region *region) 575 { 576 struct vm_region *pregion; 577 struct rb_node **p, *parent; 578 579 validate_nommu_regions(); 580 581 parent = NULL; 582 p = &nommu_region_tree.rb_node; 583 while (*p) { 584 parent = *p; 585 pregion = rb_entry(parent, struct vm_region, vm_rb); 586 if (region->vm_start < pregion->vm_start) 587 p = &(*p)->rb_left; 588 else if (region->vm_start > pregion->vm_start) 589 p = &(*p)->rb_right; 590 else if (pregion == region) 591 return; 592 else 593 BUG(); 594 } 595 596 rb_link_node(®ion->vm_rb, parent, p); 597 rb_insert_color(®ion->vm_rb, &nommu_region_tree); 598 599 validate_nommu_regions(); 600 } 601 602 /* 603 * delete a region from the global tree 604 */ 605 static void delete_nommu_region(struct vm_region *region) 606 { 607 BUG_ON(!nommu_region_tree.rb_node); 608 609 validate_nommu_regions(); 610 rb_erase(®ion->vm_rb, &nommu_region_tree); 611 validate_nommu_regions(); 612 } 613 614 /* 615 * free a contiguous series of pages 616 */ 617 static void free_page_series(unsigned long from, unsigned long to) 618 { 619 for (; from < to; from += PAGE_SIZE) { 620 struct page *page = virt_to_page(from); 621 622 atomic_long_dec(&mmap_pages_allocated); 623 put_page(page); 624 } 625 } 626 627 /* 628 * release a reference to a region 629 * - the caller must hold the region semaphore for writing, which this releases 630 * - the region may not have been added to the tree yet, in which case vm_top 631 * will equal vm_start 632 */ 633 static void __put_nommu_region(struct vm_region *region) 634 __releases(nommu_region_sem) 635 { 636 BUG_ON(!nommu_region_tree.rb_node); 637 638 if (--region->vm_usage == 0) { 639 if (region->vm_top > region->vm_start) 640 delete_nommu_region(region); 641 up_write(&nommu_region_sem); 642 643 if (region->vm_file) 644 fput(region->vm_file); 645 646 /* IO memory and memory shared directly out of the pagecache 647 * from ramfs/tmpfs mustn't be released here */ 648 if (region->vm_flags & VM_MAPPED_COPY) 649 free_page_series(region->vm_start, region->vm_top); 650 kmem_cache_free(vm_region_jar, region); 651 } else { 652 up_write(&nommu_region_sem); 653 } 654 } 655 656 /* 657 * release a reference to a region 658 */ 659 static void put_nommu_region(struct vm_region *region) 660 { 661 down_write(&nommu_region_sem); 662 __put_nommu_region(region); 663 } 664 665 /* 666 * update protection on a vma 667 */ 668 static void protect_vma(struct vm_area_struct *vma, unsigned long flags) 669 { 670 #ifdef CONFIG_MPU 671 struct mm_struct *mm = vma->vm_mm; 672 long start = vma->vm_start & PAGE_MASK; 673 while (start < vma->vm_end) { 674 protect_page(mm, start, flags); 675 start += PAGE_SIZE; 676 } 677 update_protections(mm); 678 #endif 679 } 680 681 /* 682 * add a VMA into a process's mm_struct in the appropriate place in the list 683 * and tree and add to the address space's page tree also if not an anonymous 684 * page 685 * - should be called with mm->mmap_sem held writelocked 686 */ 687 static void add_vma_to_mm(struct mm_struct *mm, struct vm_area_struct *vma) 688 { 689 struct vm_area_struct *pvma, *prev; 690 struct address_space *mapping; 691 struct rb_node **p, *parent, *rb_prev; 692 693 BUG_ON(!vma->vm_region); 694 695 mm->map_count++; 696 vma->vm_mm = mm; 697 698 protect_vma(vma, vma->vm_flags); 699 700 /* add the VMA to the mapping */ 701 if (vma->vm_file) { 702 mapping = vma->vm_file->f_mapping; 703 704 i_mmap_lock_write(mapping); 705 flush_dcache_mmap_lock(mapping); 706 vma_interval_tree_insert(vma, &mapping->i_mmap); 707 flush_dcache_mmap_unlock(mapping); 708 i_mmap_unlock_write(mapping); 709 } 710 711 /* add the VMA to the tree */ 712 parent = rb_prev = NULL; 713 p = &mm->mm_rb.rb_node; 714 while (*p) { 715 parent = *p; 716 pvma = rb_entry(parent, struct vm_area_struct, vm_rb); 717 718 /* sort by: start addr, end addr, VMA struct addr in that order 719 * (the latter is necessary as we may get identical VMAs) */ 720 if (vma->vm_start < pvma->vm_start) 721 p = &(*p)->rb_left; 722 else if (vma->vm_start > pvma->vm_start) { 723 rb_prev = parent; 724 p = &(*p)->rb_right; 725 } else if (vma->vm_end < pvma->vm_end) 726 p = &(*p)->rb_left; 727 else if (vma->vm_end > pvma->vm_end) { 728 rb_prev = parent; 729 p = &(*p)->rb_right; 730 } else if (vma < pvma) 731 p = &(*p)->rb_left; 732 else if (vma > pvma) { 733 rb_prev = parent; 734 p = &(*p)->rb_right; 735 } else 736 BUG(); 737 } 738 739 rb_link_node(&vma->vm_rb, parent, p); 740 rb_insert_color(&vma->vm_rb, &mm->mm_rb); 741 742 /* add VMA to the VMA list also */ 743 prev = NULL; 744 if (rb_prev) 745 prev = rb_entry(rb_prev, struct vm_area_struct, vm_rb); 746 747 __vma_link_list(mm, vma, prev, parent); 748 } 749 750 /* 751 * delete a VMA from its owning mm_struct and address space 752 */ 753 static void delete_vma_from_mm(struct vm_area_struct *vma) 754 { 755 int i; 756 struct address_space *mapping; 757 struct mm_struct *mm = vma->vm_mm; 758 struct task_struct *curr = current; 759 760 protect_vma(vma, 0); 761 762 mm->map_count--; 763 for (i = 0; i < VMACACHE_SIZE; i++) { 764 /* if the vma is cached, invalidate the entire cache */ 765 if (curr->vmacache.vmas[i] == vma) { 766 vmacache_invalidate(mm); 767 break; 768 } 769 } 770 771 /* remove the VMA from the mapping */ 772 if (vma->vm_file) { 773 mapping = vma->vm_file->f_mapping; 774 775 i_mmap_lock_write(mapping); 776 flush_dcache_mmap_lock(mapping); 777 vma_interval_tree_remove(vma, &mapping->i_mmap); 778 flush_dcache_mmap_unlock(mapping); 779 i_mmap_unlock_write(mapping); 780 } 781 782 /* remove from the MM's tree and list */ 783 rb_erase(&vma->vm_rb, &mm->mm_rb); 784 785 if (vma->vm_prev) 786 vma->vm_prev->vm_next = vma->vm_next; 787 else 788 mm->mmap = vma->vm_next; 789 790 if (vma->vm_next) 791 vma->vm_next->vm_prev = vma->vm_prev; 792 } 793 794 /* 795 * destroy a VMA record 796 */ 797 static void delete_vma(struct mm_struct *mm, struct vm_area_struct *vma) 798 { 799 if (vma->vm_ops && vma->vm_ops->close) 800 vma->vm_ops->close(vma); 801 if (vma->vm_file) 802 fput(vma->vm_file); 803 put_nommu_region(vma->vm_region); 804 kmem_cache_free(vm_area_cachep, vma); 805 } 806 807 /* 808 * look up the first VMA in which addr resides, NULL if none 809 * - should be called with mm->mmap_sem at least held readlocked 810 */ 811 struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr) 812 { 813 struct vm_area_struct *vma; 814 815 /* check the cache first */ 816 vma = vmacache_find(mm, addr); 817 if (likely(vma)) 818 return vma; 819 820 /* trawl the list (there may be multiple mappings in which addr 821 * resides) */ 822 for (vma = mm->mmap; vma; vma = vma->vm_next) { 823 if (vma->vm_start > addr) 824 return NULL; 825 if (vma->vm_end > addr) { 826 vmacache_update(addr, vma); 827 return vma; 828 } 829 } 830 831 return NULL; 832 } 833 EXPORT_SYMBOL(find_vma); 834 835 /* 836 * find a VMA 837 * - we don't extend stack VMAs under NOMMU conditions 838 */ 839 struct vm_area_struct *find_extend_vma(struct mm_struct *mm, unsigned long addr) 840 { 841 return find_vma(mm, addr); 842 } 843 844 /* 845 * expand a stack to a given address 846 * - not supported under NOMMU conditions 847 */ 848 int expand_stack(struct vm_area_struct *vma, unsigned long address) 849 { 850 return -ENOMEM; 851 } 852 853 /* 854 * look up the first VMA exactly that exactly matches addr 855 * - should be called with mm->mmap_sem at least held readlocked 856 */ 857 static struct vm_area_struct *find_vma_exact(struct mm_struct *mm, 858 unsigned long addr, 859 unsigned long len) 860 { 861 struct vm_area_struct *vma; 862 unsigned long end = addr + len; 863 864 /* check the cache first */ 865 vma = vmacache_find_exact(mm, addr, end); 866 if (vma) 867 return vma; 868 869 /* trawl the list (there may be multiple mappings in which addr 870 * resides) */ 871 for (vma = mm->mmap; vma; vma = vma->vm_next) { 872 if (vma->vm_start < addr) 873 continue; 874 if (vma->vm_start > addr) 875 return NULL; 876 if (vma->vm_end == end) { 877 vmacache_update(addr, vma); 878 return vma; 879 } 880 } 881 882 return NULL; 883 } 884 885 /* 886 * determine whether a mapping should be permitted and, if so, what sort of 887 * mapping we're capable of supporting 888 */ 889 static int validate_mmap_request(struct file *file, 890 unsigned long addr, 891 unsigned long len, 892 unsigned long prot, 893 unsigned long flags, 894 unsigned long pgoff, 895 unsigned long *_capabilities) 896 { 897 unsigned long capabilities, rlen; 898 int ret; 899 900 /* do the simple checks first */ 901 if (flags & MAP_FIXED) 902 return -EINVAL; 903 904 if ((flags & MAP_TYPE) != MAP_PRIVATE && 905 (flags & MAP_TYPE) != MAP_SHARED) 906 return -EINVAL; 907 908 if (!len) 909 return -EINVAL; 910 911 /* Careful about overflows.. */ 912 rlen = PAGE_ALIGN(len); 913 if (!rlen || rlen > TASK_SIZE) 914 return -ENOMEM; 915 916 /* offset overflow? */ 917 if ((pgoff + (rlen >> PAGE_SHIFT)) < pgoff) 918 return -EOVERFLOW; 919 920 if (file) { 921 /* files must support mmap */ 922 if (!file->f_op->mmap) 923 return -ENODEV; 924 925 /* work out if what we've got could possibly be shared 926 * - we support chardevs that provide their own "memory" 927 * - we support files/blockdevs that are memory backed 928 */ 929 if (file->f_op->mmap_capabilities) { 930 capabilities = file->f_op->mmap_capabilities(file); 931 } else { 932 /* no explicit capabilities set, so assume some 933 * defaults */ 934 switch (file_inode(file)->i_mode & S_IFMT) { 935 case S_IFREG: 936 case S_IFBLK: 937 capabilities = NOMMU_MAP_COPY; 938 break; 939 940 case S_IFCHR: 941 capabilities = 942 NOMMU_MAP_DIRECT | 943 NOMMU_MAP_READ | 944 NOMMU_MAP_WRITE; 945 break; 946 947 default: 948 return -EINVAL; 949 } 950 } 951 952 /* eliminate any capabilities that we can't support on this 953 * device */ 954 if (!file->f_op->get_unmapped_area) 955 capabilities &= ~NOMMU_MAP_DIRECT; 956 if (!(file->f_mode & FMODE_CAN_READ)) 957 capabilities &= ~NOMMU_MAP_COPY; 958 959 /* The file shall have been opened with read permission. */ 960 if (!(file->f_mode & FMODE_READ)) 961 return -EACCES; 962 963 if (flags & MAP_SHARED) { 964 /* do checks for writing, appending and locking */ 965 if ((prot & PROT_WRITE) && 966 !(file->f_mode & FMODE_WRITE)) 967 return -EACCES; 968 969 if (IS_APPEND(file_inode(file)) && 970 (file->f_mode & FMODE_WRITE)) 971 return -EACCES; 972 973 if (locks_verify_locked(file)) 974 return -EAGAIN; 975 976 if (!(capabilities & NOMMU_MAP_DIRECT)) 977 return -ENODEV; 978 979 /* we mustn't privatise shared mappings */ 980 capabilities &= ~NOMMU_MAP_COPY; 981 } else { 982 /* we're going to read the file into private memory we 983 * allocate */ 984 if (!(capabilities & NOMMU_MAP_COPY)) 985 return -ENODEV; 986 987 /* we don't permit a private writable mapping to be 988 * shared with the backing device */ 989 if (prot & PROT_WRITE) 990 capabilities &= ~NOMMU_MAP_DIRECT; 991 } 992 993 if (capabilities & NOMMU_MAP_DIRECT) { 994 if (((prot & PROT_READ) && !(capabilities & NOMMU_MAP_READ)) || 995 ((prot & PROT_WRITE) && !(capabilities & NOMMU_MAP_WRITE)) || 996 ((prot & PROT_EXEC) && !(capabilities & NOMMU_MAP_EXEC)) 997 ) { 998 capabilities &= ~NOMMU_MAP_DIRECT; 999 if (flags & MAP_SHARED) { 1000 pr_warn("MAP_SHARED not completely supported on !MMU\n"); 1001 return -EINVAL; 1002 } 1003 } 1004 } 1005 1006 /* handle executable mappings and implied executable 1007 * mappings */ 1008 if (path_noexec(&file->f_path)) { 1009 if (prot & PROT_EXEC) 1010 return -EPERM; 1011 } else if ((prot & PROT_READ) && !(prot & PROT_EXEC)) { 1012 /* handle implication of PROT_EXEC by PROT_READ */ 1013 if (current->personality & READ_IMPLIES_EXEC) { 1014 if (capabilities & NOMMU_MAP_EXEC) 1015 prot |= PROT_EXEC; 1016 } 1017 } else if ((prot & PROT_READ) && 1018 (prot & PROT_EXEC) && 1019 !(capabilities & NOMMU_MAP_EXEC) 1020 ) { 1021 /* backing file is not executable, try to copy */ 1022 capabilities &= ~NOMMU_MAP_DIRECT; 1023 } 1024 } else { 1025 /* anonymous mappings are always memory backed and can be 1026 * privately mapped 1027 */ 1028 capabilities = NOMMU_MAP_COPY; 1029 1030 /* handle PROT_EXEC implication by PROT_READ */ 1031 if ((prot & PROT_READ) && 1032 (current->personality & READ_IMPLIES_EXEC)) 1033 prot |= PROT_EXEC; 1034 } 1035 1036 /* allow the security API to have its say */ 1037 ret = security_mmap_addr(addr); 1038 if (ret < 0) 1039 return ret; 1040 1041 /* looks okay */ 1042 *_capabilities = capabilities; 1043 return 0; 1044 } 1045 1046 /* 1047 * we've determined that we can make the mapping, now translate what we 1048 * now know into VMA flags 1049 */ 1050 static unsigned long determine_vm_flags(struct file *file, 1051 unsigned long prot, 1052 unsigned long flags, 1053 unsigned long capabilities) 1054 { 1055 unsigned long vm_flags; 1056 1057 vm_flags = calc_vm_prot_bits(prot, 0) | calc_vm_flag_bits(flags); 1058 /* vm_flags |= mm->def_flags; */ 1059 1060 if (!(capabilities & NOMMU_MAP_DIRECT)) { 1061 /* attempt to share read-only copies of mapped file chunks */ 1062 vm_flags |= VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC; 1063 if (file && !(prot & PROT_WRITE)) 1064 vm_flags |= VM_MAYSHARE; 1065 } else { 1066 /* overlay a shareable mapping on the backing device or inode 1067 * if possible - used for chardevs, ramfs/tmpfs/shmfs and 1068 * romfs/cramfs */ 1069 vm_flags |= VM_MAYSHARE | (capabilities & NOMMU_VMFLAGS); 1070 if (flags & MAP_SHARED) 1071 vm_flags |= VM_SHARED; 1072 } 1073 1074 /* refuse to let anyone share private mappings with this process if 1075 * it's being traced - otherwise breakpoints set in it may interfere 1076 * with another untraced process 1077 */ 1078 if ((flags & MAP_PRIVATE) && current->ptrace) 1079 vm_flags &= ~VM_MAYSHARE; 1080 1081 return vm_flags; 1082 } 1083 1084 /* 1085 * set up a shared mapping on a file (the driver or filesystem provides and 1086 * pins the storage) 1087 */ 1088 static int do_mmap_shared_file(struct vm_area_struct *vma) 1089 { 1090 int ret; 1091 1092 ret = call_mmap(vma->vm_file, vma); 1093 if (ret == 0) { 1094 vma->vm_region->vm_top = vma->vm_region->vm_end; 1095 return 0; 1096 } 1097 if (ret != -ENOSYS) 1098 return ret; 1099 1100 /* getting -ENOSYS indicates that direct mmap isn't possible (as 1101 * opposed to tried but failed) so we can only give a suitable error as 1102 * it's not possible to make a private copy if MAP_SHARED was given */ 1103 return -ENODEV; 1104 } 1105 1106 /* 1107 * set up a private mapping or an anonymous shared mapping 1108 */ 1109 static int do_mmap_private(struct vm_area_struct *vma, 1110 struct vm_region *region, 1111 unsigned long len, 1112 unsigned long capabilities) 1113 { 1114 unsigned long total, point; 1115 void *base; 1116 int ret, order; 1117 1118 /* invoke the file's mapping function so that it can keep track of 1119 * shared mappings on devices or memory 1120 * - VM_MAYSHARE will be set if it may attempt to share 1121 */ 1122 if (capabilities & NOMMU_MAP_DIRECT) { 1123 ret = call_mmap(vma->vm_file, vma); 1124 if (ret == 0) { 1125 /* shouldn't return success if we're not sharing */ 1126 BUG_ON(!(vma->vm_flags & VM_MAYSHARE)); 1127 vma->vm_region->vm_top = vma->vm_region->vm_end; 1128 return 0; 1129 } 1130 if (ret != -ENOSYS) 1131 return ret; 1132 1133 /* getting an ENOSYS error indicates that direct mmap isn't 1134 * possible (as opposed to tried but failed) so we'll try to 1135 * make a private copy of the data and map that instead */ 1136 } 1137 1138 1139 /* allocate some memory to hold the mapping 1140 * - note that this may not return a page-aligned address if the object 1141 * we're allocating is smaller than a page 1142 */ 1143 order = get_order(len); 1144 total = 1 << order; 1145 point = len >> PAGE_SHIFT; 1146 1147 /* we don't want to allocate a power-of-2 sized page set */ 1148 if (sysctl_nr_trim_pages && total - point >= sysctl_nr_trim_pages) 1149 total = point; 1150 1151 base = alloc_pages_exact(total << PAGE_SHIFT, GFP_KERNEL); 1152 if (!base) 1153 goto enomem; 1154 1155 atomic_long_add(total, &mmap_pages_allocated); 1156 1157 region->vm_flags = vma->vm_flags |= VM_MAPPED_COPY; 1158 region->vm_start = (unsigned long) base; 1159 region->vm_end = region->vm_start + len; 1160 region->vm_top = region->vm_start + (total << PAGE_SHIFT); 1161 1162 vma->vm_start = region->vm_start; 1163 vma->vm_end = region->vm_start + len; 1164 1165 if (vma->vm_file) { 1166 /* read the contents of a file into the copy */ 1167 loff_t fpos; 1168 1169 fpos = vma->vm_pgoff; 1170 fpos <<= PAGE_SHIFT; 1171 1172 ret = kernel_read(vma->vm_file, base, len, &fpos); 1173 if (ret < 0) 1174 goto error_free; 1175 1176 /* clear the last little bit */ 1177 if (ret < len) 1178 memset(base + ret, 0, len - ret); 1179 1180 } 1181 1182 return 0; 1183 1184 error_free: 1185 free_page_series(region->vm_start, region->vm_top); 1186 region->vm_start = vma->vm_start = 0; 1187 region->vm_end = vma->vm_end = 0; 1188 region->vm_top = 0; 1189 return ret; 1190 1191 enomem: 1192 pr_err("Allocation of length %lu from process %d (%s) failed\n", 1193 len, current->pid, current->comm); 1194 show_free_areas(0, NULL); 1195 return -ENOMEM; 1196 } 1197 1198 /* 1199 * handle mapping creation for uClinux 1200 */ 1201 unsigned long do_mmap(struct file *file, 1202 unsigned long addr, 1203 unsigned long len, 1204 unsigned long prot, 1205 unsigned long flags, 1206 vm_flags_t vm_flags, 1207 unsigned long pgoff, 1208 unsigned long *populate, 1209 struct list_head *uf) 1210 { 1211 struct vm_area_struct *vma; 1212 struct vm_region *region; 1213 struct rb_node *rb; 1214 unsigned long capabilities, result; 1215 int ret; 1216 1217 *populate = 0; 1218 1219 /* decide whether we should attempt the mapping, and if so what sort of 1220 * mapping */ 1221 ret = validate_mmap_request(file, addr, len, prot, flags, pgoff, 1222 &capabilities); 1223 if (ret < 0) 1224 return ret; 1225 1226 /* we ignore the address hint */ 1227 addr = 0; 1228 len = PAGE_ALIGN(len); 1229 1230 /* we've determined that we can make the mapping, now translate what we 1231 * now know into VMA flags */ 1232 vm_flags |= determine_vm_flags(file, prot, flags, capabilities); 1233 1234 /* we're going to need to record the mapping */ 1235 region = kmem_cache_zalloc(vm_region_jar, GFP_KERNEL); 1236 if (!region) 1237 goto error_getting_region; 1238 1239 vma = kmem_cache_zalloc(vm_area_cachep, GFP_KERNEL); 1240 if (!vma) 1241 goto error_getting_vma; 1242 1243 region->vm_usage = 1; 1244 region->vm_flags = vm_flags; 1245 region->vm_pgoff = pgoff; 1246 1247 INIT_LIST_HEAD(&vma->anon_vma_chain); 1248 vma->vm_flags = vm_flags; 1249 vma->vm_pgoff = pgoff; 1250 1251 if (file) { 1252 region->vm_file = get_file(file); 1253 vma->vm_file = get_file(file); 1254 } 1255 1256 down_write(&nommu_region_sem); 1257 1258 /* if we want to share, we need to check for regions created by other 1259 * mmap() calls that overlap with our proposed mapping 1260 * - we can only share with a superset match on most regular files 1261 * - shared mappings on character devices and memory backed files are 1262 * permitted to overlap inexactly as far as we are concerned for in 1263 * these cases, sharing is handled in the driver or filesystem rather 1264 * than here 1265 */ 1266 if (vm_flags & VM_MAYSHARE) { 1267 struct vm_region *pregion; 1268 unsigned long pglen, rpglen, pgend, rpgend, start; 1269 1270 pglen = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; 1271 pgend = pgoff + pglen; 1272 1273 for (rb = rb_first(&nommu_region_tree); rb; rb = rb_next(rb)) { 1274 pregion = rb_entry(rb, struct vm_region, vm_rb); 1275 1276 if (!(pregion->vm_flags & VM_MAYSHARE)) 1277 continue; 1278 1279 /* search for overlapping mappings on the same file */ 1280 if (file_inode(pregion->vm_file) != 1281 file_inode(file)) 1282 continue; 1283 1284 if (pregion->vm_pgoff >= pgend) 1285 continue; 1286 1287 rpglen = pregion->vm_end - pregion->vm_start; 1288 rpglen = (rpglen + PAGE_SIZE - 1) >> PAGE_SHIFT; 1289 rpgend = pregion->vm_pgoff + rpglen; 1290 if (pgoff >= rpgend) 1291 continue; 1292 1293 /* handle inexactly overlapping matches between 1294 * mappings */ 1295 if ((pregion->vm_pgoff != pgoff || rpglen != pglen) && 1296 !(pgoff >= pregion->vm_pgoff && pgend <= rpgend)) { 1297 /* new mapping is not a subset of the region */ 1298 if (!(capabilities & NOMMU_MAP_DIRECT)) 1299 goto sharing_violation; 1300 continue; 1301 } 1302 1303 /* we've found a region we can share */ 1304 pregion->vm_usage++; 1305 vma->vm_region = pregion; 1306 start = pregion->vm_start; 1307 start += (pgoff - pregion->vm_pgoff) << PAGE_SHIFT; 1308 vma->vm_start = start; 1309 vma->vm_end = start + len; 1310 1311 if (pregion->vm_flags & VM_MAPPED_COPY) 1312 vma->vm_flags |= VM_MAPPED_COPY; 1313 else { 1314 ret = do_mmap_shared_file(vma); 1315 if (ret < 0) { 1316 vma->vm_region = NULL; 1317 vma->vm_start = 0; 1318 vma->vm_end = 0; 1319 pregion->vm_usage--; 1320 pregion = NULL; 1321 goto error_just_free; 1322 } 1323 } 1324 fput(region->vm_file); 1325 kmem_cache_free(vm_region_jar, region); 1326 region = pregion; 1327 result = start; 1328 goto share; 1329 } 1330 1331 /* obtain the address at which to make a shared mapping 1332 * - this is the hook for quasi-memory character devices to 1333 * tell us the location of a shared mapping 1334 */ 1335 if (capabilities & NOMMU_MAP_DIRECT) { 1336 addr = file->f_op->get_unmapped_area(file, addr, len, 1337 pgoff, flags); 1338 if (IS_ERR_VALUE(addr)) { 1339 ret = addr; 1340 if (ret != -ENOSYS) 1341 goto error_just_free; 1342 1343 /* the driver refused to tell us where to site 1344 * the mapping so we'll have to attempt to copy 1345 * it */ 1346 ret = -ENODEV; 1347 if (!(capabilities & NOMMU_MAP_COPY)) 1348 goto error_just_free; 1349 1350 capabilities &= ~NOMMU_MAP_DIRECT; 1351 } else { 1352 vma->vm_start = region->vm_start = addr; 1353 vma->vm_end = region->vm_end = addr + len; 1354 } 1355 } 1356 } 1357 1358 vma->vm_region = region; 1359 1360 /* set up the mapping 1361 * - the region is filled in if NOMMU_MAP_DIRECT is still set 1362 */ 1363 if (file && vma->vm_flags & VM_SHARED) 1364 ret = do_mmap_shared_file(vma); 1365 else 1366 ret = do_mmap_private(vma, region, len, capabilities); 1367 if (ret < 0) 1368 goto error_just_free; 1369 add_nommu_region(region); 1370 1371 /* clear anonymous mappings that don't ask for uninitialized data */ 1372 if (!vma->vm_file && !(flags & MAP_UNINITIALIZED)) 1373 memset((void *)region->vm_start, 0, 1374 region->vm_end - region->vm_start); 1375 1376 /* okay... we have a mapping; now we have to register it */ 1377 result = vma->vm_start; 1378 1379 current->mm->total_vm += len >> PAGE_SHIFT; 1380 1381 share: 1382 add_vma_to_mm(current->mm, vma); 1383 1384 /* we flush the region from the icache only when the first executable 1385 * mapping of it is made */ 1386 if (vma->vm_flags & VM_EXEC && !region->vm_icache_flushed) { 1387 flush_icache_range(region->vm_start, region->vm_end); 1388 region->vm_icache_flushed = true; 1389 } 1390 1391 up_write(&nommu_region_sem); 1392 1393 return result; 1394 1395 error_just_free: 1396 up_write(&nommu_region_sem); 1397 error: 1398 if (region->vm_file) 1399 fput(region->vm_file); 1400 kmem_cache_free(vm_region_jar, region); 1401 if (vma->vm_file) 1402 fput(vma->vm_file); 1403 kmem_cache_free(vm_area_cachep, vma); 1404 return ret; 1405 1406 sharing_violation: 1407 up_write(&nommu_region_sem); 1408 pr_warn("Attempt to share mismatched mappings\n"); 1409 ret = -EINVAL; 1410 goto error; 1411 1412 error_getting_vma: 1413 kmem_cache_free(vm_region_jar, region); 1414 pr_warn("Allocation of vma for %lu byte allocation from process %d failed\n", 1415 len, current->pid); 1416 show_free_areas(0, NULL); 1417 return -ENOMEM; 1418 1419 error_getting_region: 1420 pr_warn("Allocation of vm region for %lu byte allocation from process %d failed\n", 1421 len, current->pid); 1422 show_free_areas(0, NULL); 1423 return -ENOMEM; 1424 } 1425 1426 SYSCALL_DEFINE6(mmap_pgoff, unsigned long, addr, unsigned long, len, 1427 unsigned long, prot, unsigned long, flags, 1428 unsigned long, fd, unsigned long, pgoff) 1429 { 1430 struct file *file = NULL; 1431 unsigned long retval = -EBADF; 1432 1433 audit_mmap_fd(fd, flags); 1434 if (!(flags & MAP_ANONYMOUS)) { 1435 file = fget(fd); 1436 if (!file) 1437 goto out; 1438 } 1439 1440 flags &= ~(MAP_EXECUTABLE | MAP_DENYWRITE); 1441 1442 retval = vm_mmap_pgoff(file, addr, len, prot, flags, pgoff); 1443 1444 if (file) 1445 fput(file); 1446 out: 1447 return retval; 1448 } 1449 1450 #ifdef __ARCH_WANT_SYS_OLD_MMAP 1451 struct mmap_arg_struct { 1452 unsigned long addr; 1453 unsigned long len; 1454 unsigned long prot; 1455 unsigned long flags; 1456 unsigned long fd; 1457 unsigned long offset; 1458 }; 1459 1460 SYSCALL_DEFINE1(old_mmap, struct mmap_arg_struct __user *, arg) 1461 { 1462 struct mmap_arg_struct a; 1463 1464 if (copy_from_user(&a, arg, sizeof(a))) 1465 return -EFAULT; 1466 if (offset_in_page(a.offset)) 1467 return -EINVAL; 1468 1469 return sys_mmap_pgoff(a.addr, a.len, a.prot, a.flags, a.fd, 1470 a.offset >> PAGE_SHIFT); 1471 } 1472 #endif /* __ARCH_WANT_SYS_OLD_MMAP */ 1473 1474 /* 1475 * split a vma into two pieces at address 'addr', a new vma is allocated either 1476 * for the first part or the tail. 1477 */ 1478 int split_vma(struct mm_struct *mm, struct vm_area_struct *vma, 1479 unsigned long addr, int new_below) 1480 { 1481 struct vm_area_struct *new; 1482 struct vm_region *region; 1483 unsigned long npages; 1484 1485 /* we're only permitted to split anonymous regions (these should have 1486 * only a single usage on the region) */ 1487 if (vma->vm_file) 1488 return -ENOMEM; 1489 1490 if (mm->map_count >= sysctl_max_map_count) 1491 return -ENOMEM; 1492 1493 region = kmem_cache_alloc(vm_region_jar, GFP_KERNEL); 1494 if (!region) 1495 return -ENOMEM; 1496 1497 new = kmem_cache_alloc(vm_area_cachep, GFP_KERNEL); 1498 if (!new) { 1499 kmem_cache_free(vm_region_jar, region); 1500 return -ENOMEM; 1501 } 1502 1503 /* most fields are the same, copy all, and then fixup */ 1504 *new = *vma; 1505 *region = *vma->vm_region; 1506 new->vm_region = region; 1507 1508 npages = (addr - vma->vm_start) >> PAGE_SHIFT; 1509 1510 if (new_below) { 1511 region->vm_top = region->vm_end = new->vm_end = addr; 1512 } else { 1513 region->vm_start = new->vm_start = addr; 1514 region->vm_pgoff = new->vm_pgoff += npages; 1515 } 1516 1517 if (new->vm_ops && new->vm_ops->open) 1518 new->vm_ops->open(new); 1519 1520 delete_vma_from_mm(vma); 1521 down_write(&nommu_region_sem); 1522 delete_nommu_region(vma->vm_region); 1523 if (new_below) { 1524 vma->vm_region->vm_start = vma->vm_start = addr; 1525 vma->vm_region->vm_pgoff = vma->vm_pgoff += npages; 1526 } else { 1527 vma->vm_region->vm_end = vma->vm_end = addr; 1528 vma->vm_region->vm_top = addr; 1529 } 1530 add_nommu_region(vma->vm_region); 1531 add_nommu_region(new->vm_region); 1532 up_write(&nommu_region_sem); 1533 add_vma_to_mm(mm, vma); 1534 add_vma_to_mm(mm, new); 1535 return 0; 1536 } 1537 1538 /* 1539 * shrink a VMA by removing the specified chunk from either the beginning or 1540 * the end 1541 */ 1542 static int shrink_vma(struct mm_struct *mm, 1543 struct vm_area_struct *vma, 1544 unsigned long from, unsigned long to) 1545 { 1546 struct vm_region *region; 1547 1548 /* adjust the VMA's pointers, which may reposition it in the MM's tree 1549 * and list */ 1550 delete_vma_from_mm(vma); 1551 if (from > vma->vm_start) 1552 vma->vm_end = from; 1553 else 1554 vma->vm_start = to; 1555 add_vma_to_mm(mm, vma); 1556 1557 /* cut the backing region down to size */ 1558 region = vma->vm_region; 1559 BUG_ON(region->vm_usage != 1); 1560 1561 down_write(&nommu_region_sem); 1562 delete_nommu_region(region); 1563 if (from > region->vm_start) { 1564 to = region->vm_top; 1565 region->vm_top = region->vm_end = from; 1566 } else { 1567 region->vm_start = to; 1568 } 1569 add_nommu_region(region); 1570 up_write(&nommu_region_sem); 1571 1572 free_page_series(from, to); 1573 return 0; 1574 } 1575 1576 /* 1577 * release a mapping 1578 * - under NOMMU conditions the chunk to be unmapped must be backed by a single 1579 * VMA, though it need not cover the whole VMA 1580 */ 1581 int do_munmap(struct mm_struct *mm, unsigned long start, size_t len, struct list_head *uf) 1582 { 1583 struct vm_area_struct *vma; 1584 unsigned long end; 1585 int ret; 1586 1587 len = PAGE_ALIGN(len); 1588 if (len == 0) 1589 return -EINVAL; 1590 1591 end = start + len; 1592 1593 /* find the first potentially overlapping VMA */ 1594 vma = find_vma(mm, start); 1595 if (!vma) { 1596 static int limit; 1597 if (limit < 5) { 1598 pr_warn("munmap of memory not mmapped by process %d (%s): 0x%lx-0x%lx\n", 1599 current->pid, current->comm, 1600 start, start + len - 1); 1601 limit++; 1602 } 1603 return -EINVAL; 1604 } 1605 1606 /* we're allowed to split an anonymous VMA but not a file-backed one */ 1607 if (vma->vm_file) { 1608 do { 1609 if (start > vma->vm_start) 1610 return -EINVAL; 1611 if (end == vma->vm_end) 1612 goto erase_whole_vma; 1613 vma = vma->vm_next; 1614 } while (vma); 1615 return -EINVAL; 1616 } else { 1617 /* the chunk must be a subset of the VMA found */ 1618 if (start == vma->vm_start && end == vma->vm_end) 1619 goto erase_whole_vma; 1620 if (start < vma->vm_start || end > vma->vm_end) 1621 return -EINVAL; 1622 if (offset_in_page(start)) 1623 return -EINVAL; 1624 if (end != vma->vm_end && offset_in_page(end)) 1625 return -EINVAL; 1626 if (start != vma->vm_start && end != vma->vm_end) { 1627 ret = split_vma(mm, vma, start, 1); 1628 if (ret < 0) 1629 return ret; 1630 } 1631 return shrink_vma(mm, vma, start, end); 1632 } 1633 1634 erase_whole_vma: 1635 delete_vma_from_mm(vma); 1636 delete_vma(mm, vma); 1637 return 0; 1638 } 1639 EXPORT_SYMBOL(do_munmap); 1640 1641 int vm_munmap(unsigned long addr, size_t len) 1642 { 1643 struct mm_struct *mm = current->mm; 1644 int ret; 1645 1646 down_write(&mm->mmap_sem); 1647 ret = do_munmap(mm, addr, len, NULL); 1648 up_write(&mm->mmap_sem); 1649 return ret; 1650 } 1651 EXPORT_SYMBOL(vm_munmap); 1652 1653 SYSCALL_DEFINE2(munmap, unsigned long, addr, size_t, len) 1654 { 1655 return vm_munmap(addr, len); 1656 } 1657 1658 /* 1659 * release all the mappings made in a process's VM space 1660 */ 1661 void exit_mmap(struct mm_struct *mm) 1662 { 1663 struct vm_area_struct *vma; 1664 1665 if (!mm) 1666 return; 1667 1668 mm->total_vm = 0; 1669 1670 while ((vma = mm->mmap)) { 1671 mm->mmap = vma->vm_next; 1672 delete_vma_from_mm(vma); 1673 delete_vma(mm, vma); 1674 cond_resched(); 1675 } 1676 } 1677 1678 int vm_brk(unsigned long addr, unsigned long len) 1679 { 1680 return -ENOMEM; 1681 } 1682 1683 /* 1684 * expand (or shrink) an existing mapping, potentially moving it at the same 1685 * time (controlled by the MREMAP_MAYMOVE flag and available VM space) 1686 * 1687 * under NOMMU conditions, we only permit changing a mapping's size, and only 1688 * as long as it stays within the region allocated by do_mmap_private() and the 1689 * block is not shareable 1690 * 1691 * MREMAP_FIXED is not supported under NOMMU conditions 1692 */ 1693 static unsigned long do_mremap(unsigned long addr, 1694 unsigned long old_len, unsigned long new_len, 1695 unsigned long flags, unsigned long new_addr) 1696 { 1697 struct vm_area_struct *vma; 1698 1699 /* insanity checks first */ 1700 old_len = PAGE_ALIGN(old_len); 1701 new_len = PAGE_ALIGN(new_len); 1702 if (old_len == 0 || new_len == 0) 1703 return (unsigned long) -EINVAL; 1704 1705 if (offset_in_page(addr)) 1706 return -EINVAL; 1707 1708 if (flags & MREMAP_FIXED && new_addr != addr) 1709 return (unsigned long) -EINVAL; 1710 1711 vma = find_vma_exact(current->mm, addr, old_len); 1712 if (!vma) 1713 return (unsigned long) -EINVAL; 1714 1715 if (vma->vm_end != vma->vm_start + old_len) 1716 return (unsigned long) -EFAULT; 1717 1718 if (vma->vm_flags & VM_MAYSHARE) 1719 return (unsigned long) -EPERM; 1720 1721 if (new_len > vma->vm_region->vm_end - vma->vm_region->vm_start) 1722 return (unsigned long) -ENOMEM; 1723 1724 /* all checks complete - do it */ 1725 vma->vm_end = vma->vm_start + new_len; 1726 return vma->vm_start; 1727 } 1728 1729 SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len, 1730 unsigned long, new_len, unsigned long, flags, 1731 unsigned long, new_addr) 1732 { 1733 unsigned long ret; 1734 1735 down_write(¤t->mm->mmap_sem); 1736 ret = do_mremap(addr, old_len, new_len, flags, new_addr); 1737 up_write(¤t->mm->mmap_sem); 1738 return ret; 1739 } 1740 1741 struct page *follow_page_mask(struct vm_area_struct *vma, 1742 unsigned long address, unsigned int flags, 1743 unsigned int *page_mask) 1744 { 1745 *page_mask = 0; 1746 return NULL; 1747 } 1748 1749 int remap_pfn_range(struct vm_area_struct *vma, unsigned long addr, 1750 unsigned long pfn, unsigned long size, pgprot_t prot) 1751 { 1752 if (addr != (pfn << PAGE_SHIFT)) 1753 return -EINVAL; 1754 1755 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; 1756 return 0; 1757 } 1758 EXPORT_SYMBOL(remap_pfn_range); 1759 1760 int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len) 1761 { 1762 unsigned long pfn = start >> PAGE_SHIFT; 1763 unsigned long vm_len = vma->vm_end - vma->vm_start; 1764 1765 pfn += vma->vm_pgoff; 1766 return io_remap_pfn_range(vma, vma->vm_start, pfn, vm_len, vma->vm_page_prot); 1767 } 1768 EXPORT_SYMBOL(vm_iomap_memory); 1769 1770 int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, 1771 unsigned long pgoff) 1772 { 1773 unsigned int size = vma->vm_end - vma->vm_start; 1774 1775 if (!(vma->vm_flags & VM_USERMAP)) 1776 return -EINVAL; 1777 1778 vma->vm_start = (unsigned long)(addr + (pgoff << PAGE_SHIFT)); 1779 vma->vm_end = vma->vm_start + size; 1780 1781 return 0; 1782 } 1783 EXPORT_SYMBOL(remap_vmalloc_range); 1784 1785 unsigned long arch_get_unmapped_area(struct file *file, unsigned long addr, 1786 unsigned long len, unsigned long pgoff, unsigned long flags) 1787 { 1788 return -ENOMEM; 1789 } 1790 1791 void unmap_mapping_range(struct address_space *mapping, 1792 loff_t const holebegin, loff_t const holelen, 1793 int even_cows) 1794 { 1795 } 1796 EXPORT_SYMBOL(unmap_mapping_range); 1797 1798 int filemap_fault(struct vm_fault *vmf) 1799 { 1800 BUG(); 1801 return 0; 1802 } 1803 EXPORT_SYMBOL(filemap_fault); 1804 1805 void filemap_map_pages(struct vm_fault *vmf, 1806 pgoff_t start_pgoff, pgoff_t end_pgoff) 1807 { 1808 BUG(); 1809 } 1810 EXPORT_SYMBOL(filemap_map_pages); 1811 1812 int __access_remote_vm(struct task_struct *tsk, struct mm_struct *mm, 1813 unsigned long addr, void *buf, int len, unsigned int gup_flags) 1814 { 1815 struct vm_area_struct *vma; 1816 int write = gup_flags & FOLL_WRITE; 1817 1818 down_read(&mm->mmap_sem); 1819 1820 /* the access must start within one of the target process's mappings */ 1821 vma = find_vma(mm, addr); 1822 if (vma) { 1823 /* don't overrun this mapping */ 1824 if (addr + len >= vma->vm_end) 1825 len = vma->vm_end - addr; 1826 1827 /* only read or write mappings where it is permitted */ 1828 if (write && vma->vm_flags & VM_MAYWRITE) 1829 copy_to_user_page(vma, NULL, addr, 1830 (void *) addr, buf, len); 1831 else if (!write && vma->vm_flags & VM_MAYREAD) 1832 copy_from_user_page(vma, NULL, addr, 1833 buf, (void *) addr, len); 1834 else 1835 len = 0; 1836 } else { 1837 len = 0; 1838 } 1839 1840 up_read(&mm->mmap_sem); 1841 1842 return len; 1843 } 1844 1845 /** 1846 * @access_remote_vm - access another process' address space 1847 * @mm: the mm_struct of the target address space 1848 * @addr: start address to access 1849 * @buf: source or destination buffer 1850 * @len: number of bytes to transfer 1851 * @gup_flags: flags modifying lookup behaviour 1852 * 1853 * The caller must hold a reference on @mm. 1854 */ 1855 int access_remote_vm(struct mm_struct *mm, unsigned long addr, 1856 void *buf, int len, unsigned int gup_flags) 1857 { 1858 return __access_remote_vm(NULL, mm, addr, buf, len, gup_flags); 1859 } 1860 1861 /* 1862 * Access another process' address space. 1863 * - source/target buffer must be kernel space 1864 */ 1865 int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, 1866 unsigned int gup_flags) 1867 { 1868 struct mm_struct *mm; 1869 1870 if (addr + len < addr) 1871 return 0; 1872 1873 mm = get_task_mm(tsk); 1874 if (!mm) 1875 return 0; 1876 1877 len = __access_remote_vm(tsk, mm, addr, buf, len, gup_flags); 1878 1879 mmput(mm); 1880 return len; 1881 } 1882 EXPORT_SYMBOL_GPL(access_process_vm); 1883 1884 /** 1885 * nommu_shrink_inode_mappings - Shrink the shared mappings on an inode 1886 * @inode: The inode to check 1887 * @size: The current filesize of the inode 1888 * @newsize: The proposed filesize of the inode 1889 * 1890 * Check the shared mappings on an inode on behalf of a shrinking truncate to 1891 * make sure that that any outstanding VMAs aren't broken and then shrink the 1892 * vm_regions that extend that beyond so that do_mmap_pgoff() doesn't 1893 * automatically grant mappings that are too large. 1894 */ 1895 int nommu_shrink_inode_mappings(struct inode *inode, size_t size, 1896 size_t newsize) 1897 { 1898 struct vm_area_struct *vma; 1899 struct vm_region *region; 1900 pgoff_t low, high; 1901 size_t r_size, r_top; 1902 1903 low = newsize >> PAGE_SHIFT; 1904 high = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1905 1906 down_write(&nommu_region_sem); 1907 i_mmap_lock_read(inode->i_mapping); 1908 1909 /* search for VMAs that fall within the dead zone */ 1910 vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, low, high) { 1911 /* found one - only interested if it's shared out of the page 1912 * cache */ 1913 if (vma->vm_flags & VM_SHARED) { 1914 i_mmap_unlock_read(inode->i_mapping); 1915 up_write(&nommu_region_sem); 1916 return -ETXTBSY; /* not quite true, but near enough */ 1917 } 1918 } 1919 1920 /* reduce any regions that overlap the dead zone - if in existence, 1921 * these will be pointed to by VMAs that don't overlap the dead zone 1922 * 1923 * we don't check for any regions that start beyond the EOF as there 1924 * shouldn't be any 1925 */ 1926 vma_interval_tree_foreach(vma, &inode->i_mapping->i_mmap, 0, ULONG_MAX) { 1927 if (!(vma->vm_flags & VM_SHARED)) 1928 continue; 1929 1930 region = vma->vm_region; 1931 r_size = region->vm_top - region->vm_start; 1932 r_top = (region->vm_pgoff << PAGE_SHIFT) + r_size; 1933 1934 if (r_top > newsize) { 1935 region->vm_top -= r_top - newsize; 1936 if (region->vm_end > region->vm_top) 1937 region->vm_end = region->vm_top; 1938 } 1939 } 1940 1941 i_mmap_unlock_read(inode->i_mapping); 1942 up_write(&nommu_region_sem); 1943 return 0; 1944 } 1945 1946 /* 1947 * Initialise sysctl_user_reserve_kbytes. 1948 * 1949 * This is intended to prevent a user from starting a single memory hogging 1950 * process, such that they cannot recover (kill the hog) in OVERCOMMIT_NEVER 1951 * mode. 1952 * 1953 * The default value is min(3% of free memory, 128MB) 1954 * 128MB is enough to recover with sshd/login, bash, and top/kill. 1955 */ 1956 static int __meminit init_user_reserve(void) 1957 { 1958 unsigned long free_kbytes; 1959 1960 free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); 1961 1962 sysctl_user_reserve_kbytes = min(free_kbytes / 32, 1UL << 17); 1963 return 0; 1964 } 1965 subsys_initcall(init_user_reserve); 1966 1967 /* 1968 * Initialise sysctl_admin_reserve_kbytes. 1969 * 1970 * The purpose of sysctl_admin_reserve_kbytes is to allow the sys admin 1971 * to log in and kill a memory hogging process. 1972 * 1973 * Systems with more than 256MB will reserve 8MB, enough to recover 1974 * with sshd, bash, and top in OVERCOMMIT_GUESS. Smaller systems will 1975 * only reserve 3% of free pages by default. 1976 */ 1977 static int __meminit init_admin_reserve(void) 1978 { 1979 unsigned long free_kbytes; 1980 1981 free_kbytes = global_zone_page_state(NR_FREE_PAGES) << (PAGE_SHIFT - 10); 1982 1983 sysctl_admin_reserve_kbytes = min(free_kbytes / 32, 1UL << 13); 1984 return 0; 1985 } 1986 subsys_initcall(init_admin_reserve); 1987