1 /* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2011 Hugh Dickins. 10 * Copyright (C) 2011 Google Inc. 11 * Copyright (C) 2002-2005 VERITAS Software Corporation. 12 * Copyright (C) 2004 Andi Kleen, SuSE Labs 13 * 14 * Extended attribute support for tmpfs: 15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 17 * 18 * tiny-shmem: 19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 20 * 21 * This file is released under the GPL. 22 */ 23 24 #include <linux/fs.h> 25 #include <linux/init.h> 26 #include <linux/vfs.h> 27 #include <linux/mount.h> 28 #include <linux/ramfs.h> 29 #include <linux/pagemap.h> 30 #include <linux/file.h> 31 #include <linux/mm.h> 32 #include <linux/random.h> 33 #include <linux/sched/signal.h> 34 #include <linux/export.h> 35 #include <linux/swap.h> 36 #include <linux/uio.h> 37 #include <linux/khugepaged.h> 38 #include <linux/hugetlb.h> 39 40 #include <asm/tlbflush.h> /* for arch/microblaze update_mmu_cache() */ 41 42 static struct vfsmount *shm_mnt; 43 44 #ifdef CONFIG_SHMEM 45 /* 46 * This virtual memory filesystem is heavily based on the ramfs. It 47 * extends ramfs by the ability to use swap and honor resource limits 48 * which makes it a completely usable filesystem. 49 */ 50 51 #include <linux/xattr.h> 52 #include <linux/exportfs.h> 53 #include <linux/posix_acl.h> 54 #include <linux/posix_acl_xattr.h> 55 #include <linux/mman.h> 56 #include <linux/string.h> 57 #include <linux/slab.h> 58 #include <linux/backing-dev.h> 59 #include <linux/shmem_fs.h> 60 #include <linux/writeback.h> 61 #include <linux/blkdev.h> 62 #include <linux/pagevec.h> 63 #include <linux/percpu_counter.h> 64 #include <linux/falloc.h> 65 #include <linux/splice.h> 66 #include <linux/security.h> 67 #include <linux/swapops.h> 68 #include <linux/mempolicy.h> 69 #include <linux/namei.h> 70 #include <linux/ctype.h> 71 #include <linux/migrate.h> 72 #include <linux/highmem.h> 73 #include <linux/seq_file.h> 74 #include <linux/magic.h> 75 #include <linux/syscalls.h> 76 #include <linux/fcntl.h> 77 #include <uapi/linux/memfd.h> 78 #include <linux/userfaultfd_k.h> 79 #include <linux/rmap.h> 80 #include <linux/uuid.h> 81 82 #include <linux/uaccess.h> 83 #include <asm/pgtable.h> 84 85 #include "internal.h" 86 87 #define BLOCKS_PER_PAGE (PAGE_SIZE/512) 88 #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT) 89 90 /* Pretend that each entry is of this size in directory's i_size */ 91 #define BOGO_DIRENT_SIZE 20 92 93 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */ 94 #define SHORT_SYMLINK_LEN 128 95 96 /* 97 * shmem_fallocate communicates with shmem_fault or shmem_writepage via 98 * inode->i_private (with i_mutex making sure that it has only one user at 99 * a time): we would prefer not to enlarge the shmem inode just for that. 100 */ 101 struct shmem_falloc { 102 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */ 103 pgoff_t start; /* start of range currently being fallocated */ 104 pgoff_t next; /* the next page offset to be fallocated */ 105 pgoff_t nr_falloced; /* how many new pages have been fallocated */ 106 pgoff_t nr_unswapped; /* how often writepage refused to swap out */ 107 }; 108 109 #ifdef CONFIG_TMPFS 110 static unsigned long shmem_default_max_blocks(void) 111 { 112 return totalram_pages() / 2; 113 } 114 115 static unsigned long shmem_default_max_inodes(void) 116 { 117 unsigned long nr_pages = totalram_pages(); 118 119 return min(nr_pages - totalhigh_pages(), nr_pages / 2); 120 } 121 #endif 122 123 static bool shmem_should_replace_page(struct page *page, gfp_t gfp); 124 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 125 struct shmem_inode_info *info, pgoff_t index); 126 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 127 struct page **pagep, enum sgp_type sgp, 128 gfp_t gfp, struct vm_area_struct *vma, 129 struct vm_fault *vmf, vm_fault_t *fault_type); 130 131 int shmem_getpage(struct inode *inode, pgoff_t index, 132 struct page **pagep, enum sgp_type sgp) 133 { 134 return shmem_getpage_gfp(inode, index, pagep, sgp, 135 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL); 136 } 137 138 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 139 { 140 return sb->s_fs_info; 141 } 142 143 /* 144 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 145 * for shared memory and for shared anonymous (/dev/zero) mappings 146 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 147 * consistent with the pre-accounting of private mappings ... 148 */ 149 static inline int shmem_acct_size(unsigned long flags, loff_t size) 150 { 151 return (flags & VM_NORESERVE) ? 152 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); 153 } 154 155 static inline void shmem_unacct_size(unsigned long flags, loff_t size) 156 { 157 if (!(flags & VM_NORESERVE)) 158 vm_unacct_memory(VM_ACCT(size)); 159 } 160 161 static inline int shmem_reacct_size(unsigned long flags, 162 loff_t oldsize, loff_t newsize) 163 { 164 if (!(flags & VM_NORESERVE)) { 165 if (VM_ACCT(newsize) > VM_ACCT(oldsize)) 166 return security_vm_enough_memory_mm(current->mm, 167 VM_ACCT(newsize) - VM_ACCT(oldsize)); 168 else if (VM_ACCT(newsize) < VM_ACCT(oldsize)) 169 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize)); 170 } 171 return 0; 172 } 173 174 /* 175 * ... whereas tmpfs objects are accounted incrementally as 176 * pages are allocated, in order to allow large sparse files. 177 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, 178 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 179 */ 180 static inline int shmem_acct_block(unsigned long flags, long pages) 181 { 182 if (!(flags & VM_NORESERVE)) 183 return 0; 184 185 return security_vm_enough_memory_mm(current->mm, 186 pages * VM_ACCT(PAGE_SIZE)); 187 } 188 189 static inline void shmem_unacct_blocks(unsigned long flags, long pages) 190 { 191 if (flags & VM_NORESERVE) 192 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE)); 193 } 194 195 static inline bool shmem_inode_acct_block(struct inode *inode, long pages) 196 { 197 struct shmem_inode_info *info = SHMEM_I(inode); 198 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 199 200 if (shmem_acct_block(info->flags, pages)) 201 return false; 202 203 if (sbinfo->max_blocks) { 204 if (percpu_counter_compare(&sbinfo->used_blocks, 205 sbinfo->max_blocks - pages) > 0) 206 goto unacct; 207 percpu_counter_add(&sbinfo->used_blocks, pages); 208 } 209 210 return true; 211 212 unacct: 213 shmem_unacct_blocks(info->flags, pages); 214 return false; 215 } 216 217 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages) 218 { 219 struct shmem_inode_info *info = SHMEM_I(inode); 220 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 221 222 if (sbinfo->max_blocks) 223 percpu_counter_sub(&sbinfo->used_blocks, pages); 224 shmem_unacct_blocks(info->flags, pages); 225 } 226 227 static const struct super_operations shmem_ops; 228 static const struct address_space_operations shmem_aops; 229 static const struct file_operations shmem_file_operations; 230 static const struct inode_operations shmem_inode_operations; 231 static const struct inode_operations shmem_dir_inode_operations; 232 static const struct inode_operations shmem_special_inode_operations; 233 static const struct vm_operations_struct shmem_vm_ops; 234 static struct file_system_type shmem_fs_type; 235 236 bool vma_is_shmem(struct vm_area_struct *vma) 237 { 238 return vma->vm_ops == &shmem_vm_ops; 239 } 240 241 static LIST_HEAD(shmem_swaplist); 242 static DEFINE_MUTEX(shmem_swaplist_mutex); 243 244 static int shmem_reserve_inode(struct super_block *sb) 245 { 246 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 247 if (sbinfo->max_inodes) { 248 spin_lock(&sbinfo->stat_lock); 249 if (!sbinfo->free_inodes) { 250 spin_unlock(&sbinfo->stat_lock); 251 return -ENOSPC; 252 } 253 sbinfo->free_inodes--; 254 spin_unlock(&sbinfo->stat_lock); 255 } 256 return 0; 257 } 258 259 static void shmem_free_inode(struct super_block *sb) 260 { 261 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 262 if (sbinfo->max_inodes) { 263 spin_lock(&sbinfo->stat_lock); 264 sbinfo->free_inodes++; 265 spin_unlock(&sbinfo->stat_lock); 266 } 267 } 268 269 /** 270 * shmem_recalc_inode - recalculate the block usage of an inode 271 * @inode: inode to recalc 272 * 273 * We have to calculate the free blocks since the mm can drop 274 * undirtied hole pages behind our back. 275 * 276 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 277 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 278 * 279 * It has to be called with the spinlock held. 280 */ 281 static void shmem_recalc_inode(struct inode *inode) 282 { 283 struct shmem_inode_info *info = SHMEM_I(inode); 284 long freed; 285 286 freed = info->alloced - info->swapped - inode->i_mapping->nrpages; 287 if (freed > 0) { 288 info->alloced -= freed; 289 inode->i_blocks -= freed * BLOCKS_PER_PAGE; 290 shmem_inode_unacct_blocks(inode, freed); 291 } 292 } 293 294 bool shmem_charge(struct inode *inode, long pages) 295 { 296 struct shmem_inode_info *info = SHMEM_I(inode); 297 unsigned long flags; 298 299 if (!shmem_inode_acct_block(inode, pages)) 300 return false; 301 302 /* nrpages adjustment first, then shmem_recalc_inode() when balanced */ 303 inode->i_mapping->nrpages += pages; 304 305 spin_lock_irqsave(&info->lock, flags); 306 info->alloced += pages; 307 inode->i_blocks += pages * BLOCKS_PER_PAGE; 308 shmem_recalc_inode(inode); 309 spin_unlock_irqrestore(&info->lock, flags); 310 311 return true; 312 } 313 314 void shmem_uncharge(struct inode *inode, long pages) 315 { 316 struct shmem_inode_info *info = SHMEM_I(inode); 317 unsigned long flags; 318 319 /* nrpages adjustment done by __delete_from_page_cache() or caller */ 320 321 spin_lock_irqsave(&info->lock, flags); 322 info->alloced -= pages; 323 inode->i_blocks -= pages * BLOCKS_PER_PAGE; 324 shmem_recalc_inode(inode); 325 spin_unlock_irqrestore(&info->lock, flags); 326 327 shmem_inode_unacct_blocks(inode, pages); 328 } 329 330 /* 331 * Replace item expected in xarray by a new item, while holding xa_lock. 332 */ 333 static int shmem_replace_entry(struct address_space *mapping, 334 pgoff_t index, void *expected, void *replacement) 335 { 336 XA_STATE(xas, &mapping->i_pages, index); 337 void *item; 338 339 VM_BUG_ON(!expected); 340 VM_BUG_ON(!replacement); 341 item = xas_load(&xas); 342 if (item != expected) 343 return -ENOENT; 344 xas_store(&xas, replacement); 345 return 0; 346 } 347 348 /* 349 * Sometimes, before we decide whether to proceed or to fail, we must check 350 * that an entry was not already brought back from swap by a racing thread. 351 * 352 * Checking page is not enough: by the time a SwapCache page is locked, it 353 * might be reused, and again be SwapCache, using the same swap as before. 354 */ 355 static bool shmem_confirm_swap(struct address_space *mapping, 356 pgoff_t index, swp_entry_t swap) 357 { 358 return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap); 359 } 360 361 /* 362 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option 363 * 364 * SHMEM_HUGE_NEVER: 365 * disables huge pages for the mount; 366 * SHMEM_HUGE_ALWAYS: 367 * enables huge pages for the mount; 368 * SHMEM_HUGE_WITHIN_SIZE: 369 * only allocate huge pages if the page will be fully within i_size, 370 * also respect fadvise()/madvise() hints; 371 * SHMEM_HUGE_ADVISE: 372 * only allocate huge pages if requested with fadvise()/madvise(); 373 */ 374 375 #define SHMEM_HUGE_NEVER 0 376 #define SHMEM_HUGE_ALWAYS 1 377 #define SHMEM_HUGE_WITHIN_SIZE 2 378 #define SHMEM_HUGE_ADVISE 3 379 380 /* 381 * Special values. 382 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled: 383 * 384 * SHMEM_HUGE_DENY: 385 * disables huge on shm_mnt and all mounts, for emergency use; 386 * SHMEM_HUGE_FORCE: 387 * enables huge on shm_mnt and all mounts, w/o needing option, for testing; 388 * 389 */ 390 #define SHMEM_HUGE_DENY (-1) 391 #define SHMEM_HUGE_FORCE (-2) 392 393 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 394 /* ifdef here to avoid bloating shmem.o when not necessary */ 395 396 static int shmem_huge __read_mostly; 397 398 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS) 399 static int shmem_parse_huge(const char *str) 400 { 401 if (!strcmp(str, "never")) 402 return SHMEM_HUGE_NEVER; 403 if (!strcmp(str, "always")) 404 return SHMEM_HUGE_ALWAYS; 405 if (!strcmp(str, "within_size")) 406 return SHMEM_HUGE_WITHIN_SIZE; 407 if (!strcmp(str, "advise")) 408 return SHMEM_HUGE_ADVISE; 409 if (!strcmp(str, "deny")) 410 return SHMEM_HUGE_DENY; 411 if (!strcmp(str, "force")) 412 return SHMEM_HUGE_FORCE; 413 return -EINVAL; 414 } 415 416 static const char *shmem_format_huge(int huge) 417 { 418 switch (huge) { 419 case SHMEM_HUGE_NEVER: 420 return "never"; 421 case SHMEM_HUGE_ALWAYS: 422 return "always"; 423 case SHMEM_HUGE_WITHIN_SIZE: 424 return "within_size"; 425 case SHMEM_HUGE_ADVISE: 426 return "advise"; 427 case SHMEM_HUGE_DENY: 428 return "deny"; 429 case SHMEM_HUGE_FORCE: 430 return "force"; 431 default: 432 VM_BUG_ON(1); 433 return "bad_val"; 434 } 435 } 436 #endif 437 438 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 439 struct shrink_control *sc, unsigned long nr_to_split) 440 { 441 LIST_HEAD(list), *pos, *next; 442 LIST_HEAD(to_remove); 443 struct inode *inode; 444 struct shmem_inode_info *info; 445 struct page *page; 446 unsigned long batch = sc ? sc->nr_to_scan : 128; 447 int removed = 0, split = 0; 448 449 if (list_empty(&sbinfo->shrinklist)) 450 return SHRINK_STOP; 451 452 spin_lock(&sbinfo->shrinklist_lock); 453 list_for_each_safe(pos, next, &sbinfo->shrinklist) { 454 info = list_entry(pos, struct shmem_inode_info, shrinklist); 455 456 /* pin the inode */ 457 inode = igrab(&info->vfs_inode); 458 459 /* inode is about to be evicted */ 460 if (!inode) { 461 list_del_init(&info->shrinklist); 462 removed++; 463 goto next; 464 } 465 466 /* Check if there's anything to gain */ 467 if (round_up(inode->i_size, PAGE_SIZE) == 468 round_up(inode->i_size, HPAGE_PMD_SIZE)) { 469 list_move(&info->shrinklist, &to_remove); 470 removed++; 471 goto next; 472 } 473 474 list_move(&info->shrinklist, &list); 475 next: 476 if (!--batch) 477 break; 478 } 479 spin_unlock(&sbinfo->shrinklist_lock); 480 481 list_for_each_safe(pos, next, &to_remove) { 482 info = list_entry(pos, struct shmem_inode_info, shrinklist); 483 inode = &info->vfs_inode; 484 list_del_init(&info->shrinklist); 485 iput(inode); 486 } 487 488 list_for_each_safe(pos, next, &list) { 489 int ret; 490 491 info = list_entry(pos, struct shmem_inode_info, shrinklist); 492 inode = &info->vfs_inode; 493 494 if (nr_to_split && split >= nr_to_split) 495 goto leave; 496 497 page = find_get_page(inode->i_mapping, 498 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT); 499 if (!page) 500 goto drop; 501 502 /* No huge page at the end of the file: nothing to split */ 503 if (!PageTransHuge(page)) { 504 put_page(page); 505 goto drop; 506 } 507 508 /* 509 * Leave the inode on the list if we failed to lock 510 * the page at this time. 511 * 512 * Waiting for the lock may lead to deadlock in the 513 * reclaim path. 514 */ 515 if (!trylock_page(page)) { 516 put_page(page); 517 goto leave; 518 } 519 520 ret = split_huge_page(page); 521 unlock_page(page); 522 put_page(page); 523 524 /* If split failed leave the inode on the list */ 525 if (ret) 526 goto leave; 527 528 split++; 529 drop: 530 list_del_init(&info->shrinklist); 531 removed++; 532 leave: 533 iput(inode); 534 } 535 536 spin_lock(&sbinfo->shrinklist_lock); 537 list_splice_tail(&list, &sbinfo->shrinklist); 538 sbinfo->shrinklist_len -= removed; 539 spin_unlock(&sbinfo->shrinklist_lock); 540 541 return split; 542 } 543 544 static long shmem_unused_huge_scan(struct super_block *sb, 545 struct shrink_control *sc) 546 { 547 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 548 549 if (!READ_ONCE(sbinfo->shrinklist_len)) 550 return SHRINK_STOP; 551 552 return shmem_unused_huge_shrink(sbinfo, sc, 0); 553 } 554 555 static long shmem_unused_huge_count(struct super_block *sb, 556 struct shrink_control *sc) 557 { 558 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 559 return READ_ONCE(sbinfo->shrinklist_len); 560 } 561 #else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 562 563 #define shmem_huge SHMEM_HUGE_DENY 564 565 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 566 struct shrink_control *sc, unsigned long nr_to_split) 567 { 568 return 0; 569 } 570 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 571 572 static inline bool is_huge_enabled(struct shmem_sb_info *sbinfo) 573 { 574 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && 575 (shmem_huge == SHMEM_HUGE_FORCE || sbinfo->huge) && 576 shmem_huge != SHMEM_HUGE_DENY) 577 return true; 578 return false; 579 } 580 581 /* 582 * Like add_to_page_cache_locked, but error if expected item has gone. 583 */ 584 static int shmem_add_to_page_cache(struct page *page, 585 struct address_space *mapping, 586 pgoff_t index, void *expected, gfp_t gfp) 587 { 588 XA_STATE_ORDER(xas, &mapping->i_pages, index, compound_order(page)); 589 unsigned long i = 0; 590 unsigned long nr = 1UL << compound_order(page); 591 592 VM_BUG_ON_PAGE(PageTail(page), page); 593 VM_BUG_ON_PAGE(index != round_down(index, nr), page); 594 VM_BUG_ON_PAGE(!PageLocked(page), page); 595 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 596 VM_BUG_ON(expected && PageTransHuge(page)); 597 598 page_ref_add(page, nr); 599 page->mapping = mapping; 600 page->index = index; 601 602 do { 603 void *entry; 604 xas_lock_irq(&xas); 605 entry = xas_find_conflict(&xas); 606 if (entry != expected) 607 xas_set_err(&xas, -EEXIST); 608 xas_create_range(&xas); 609 if (xas_error(&xas)) 610 goto unlock; 611 next: 612 xas_store(&xas, page + i); 613 if (++i < nr) { 614 xas_next(&xas); 615 goto next; 616 } 617 if (PageTransHuge(page)) { 618 count_vm_event(THP_FILE_ALLOC); 619 __inc_node_page_state(page, NR_SHMEM_THPS); 620 } 621 mapping->nrpages += nr; 622 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); 623 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr); 624 unlock: 625 xas_unlock_irq(&xas); 626 } while (xas_nomem(&xas, gfp)); 627 628 if (xas_error(&xas)) { 629 page->mapping = NULL; 630 page_ref_sub(page, nr); 631 return xas_error(&xas); 632 } 633 634 return 0; 635 } 636 637 /* 638 * Like delete_from_page_cache, but substitutes swap for page. 639 */ 640 static void shmem_delete_from_page_cache(struct page *page, void *radswap) 641 { 642 struct address_space *mapping = page->mapping; 643 int error; 644 645 VM_BUG_ON_PAGE(PageCompound(page), page); 646 647 xa_lock_irq(&mapping->i_pages); 648 error = shmem_replace_entry(mapping, page->index, page, radswap); 649 page->mapping = NULL; 650 mapping->nrpages--; 651 __dec_node_page_state(page, NR_FILE_PAGES); 652 __dec_node_page_state(page, NR_SHMEM); 653 xa_unlock_irq(&mapping->i_pages); 654 put_page(page); 655 BUG_ON(error); 656 } 657 658 /* 659 * Remove swap entry from page cache, free the swap and its page cache. 660 */ 661 static int shmem_free_swap(struct address_space *mapping, 662 pgoff_t index, void *radswap) 663 { 664 void *old; 665 666 old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0); 667 if (old != radswap) 668 return -ENOENT; 669 free_swap_and_cache(radix_to_swp_entry(radswap)); 670 return 0; 671 } 672 673 /* 674 * Determine (in bytes) how many of the shmem object's pages mapped by the 675 * given offsets are swapped out. 676 * 677 * This is safe to call without i_mutex or the i_pages lock thanks to RCU, 678 * as long as the inode doesn't go away and racy results are not a problem. 679 */ 680 unsigned long shmem_partial_swap_usage(struct address_space *mapping, 681 pgoff_t start, pgoff_t end) 682 { 683 XA_STATE(xas, &mapping->i_pages, start); 684 struct page *page; 685 unsigned long swapped = 0; 686 687 rcu_read_lock(); 688 xas_for_each(&xas, page, end - 1) { 689 if (xas_retry(&xas, page)) 690 continue; 691 if (xa_is_value(page)) 692 swapped++; 693 694 if (need_resched()) { 695 xas_pause(&xas); 696 cond_resched_rcu(); 697 } 698 } 699 700 rcu_read_unlock(); 701 702 return swapped << PAGE_SHIFT; 703 } 704 705 /* 706 * Determine (in bytes) how many of the shmem object's pages mapped by the 707 * given vma is swapped out. 708 * 709 * This is safe to call without i_mutex or the i_pages lock thanks to RCU, 710 * as long as the inode doesn't go away and racy results are not a problem. 711 */ 712 unsigned long shmem_swap_usage(struct vm_area_struct *vma) 713 { 714 struct inode *inode = file_inode(vma->vm_file); 715 struct shmem_inode_info *info = SHMEM_I(inode); 716 struct address_space *mapping = inode->i_mapping; 717 unsigned long swapped; 718 719 /* Be careful as we don't hold info->lock */ 720 swapped = READ_ONCE(info->swapped); 721 722 /* 723 * The easier cases are when the shmem object has nothing in swap, or 724 * the vma maps it whole. Then we can simply use the stats that we 725 * already track. 726 */ 727 if (!swapped) 728 return 0; 729 730 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) 731 return swapped << PAGE_SHIFT; 732 733 /* Here comes the more involved part */ 734 return shmem_partial_swap_usage(mapping, 735 linear_page_index(vma, vma->vm_start), 736 linear_page_index(vma, vma->vm_end)); 737 } 738 739 /* 740 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists. 741 */ 742 void shmem_unlock_mapping(struct address_space *mapping) 743 { 744 struct pagevec pvec; 745 pgoff_t indices[PAGEVEC_SIZE]; 746 pgoff_t index = 0; 747 748 pagevec_init(&pvec); 749 /* 750 * Minor point, but we might as well stop if someone else SHM_LOCKs it. 751 */ 752 while (!mapping_unevictable(mapping)) { 753 /* 754 * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it 755 * has finished, if it hits a row of PAGEVEC_SIZE swap entries. 756 */ 757 pvec.nr = find_get_entries(mapping, index, 758 PAGEVEC_SIZE, pvec.pages, indices); 759 if (!pvec.nr) 760 break; 761 index = indices[pvec.nr - 1] + 1; 762 pagevec_remove_exceptionals(&pvec); 763 check_move_unevictable_pages(&pvec); 764 pagevec_release(&pvec); 765 cond_resched(); 766 } 767 } 768 769 /* 770 * Remove range of pages and swap entries from page cache, and free them. 771 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. 772 */ 773 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, 774 bool unfalloc) 775 { 776 struct address_space *mapping = inode->i_mapping; 777 struct shmem_inode_info *info = SHMEM_I(inode); 778 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT; 779 pgoff_t end = (lend + 1) >> PAGE_SHIFT; 780 unsigned int partial_start = lstart & (PAGE_SIZE - 1); 781 unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1); 782 struct pagevec pvec; 783 pgoff_t indices[PAGEVEC_SIZE]; 784 long nr_swaps_freed = 0; 785 pgoff_t index; 786 int i; 787 788 if (lend == -1) 789 end = -1; /* unsigned, so actually very big */ 790 791 pagevec_init(&pvec); 792 index = start; 793 while (index < end) { 794 pvec.nr = find_get_entries(mapping, index, 795 min(end - index, (pgoff_t)PAGEVEC_SIZE), 796 pvec.pages, indices); 797 if (!pvec.nr) 798 break; 799 for (i = 0; i < pagevec_count(&pvec); i++) { 800 struct page *page = pvec.pages[i]; 801 802 index = indices[i]; 803 if (index >= end) 804 break; 805 806 if (xa_is_value(page)) { 807 if (unfalloc) 808 continue; 809 nr_swaps_freed += !shmem_free_swap(mapping, 810 index, page); 811 continue; 812 } 813 814 VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page); 815 816 if (!trylock_page(page)) 817 continue; 818 819 if (PageTransTail(page)) { 820 /* Middle of THP: zero out the page */ 821 clear_highpage(page); 822 unlock_page(page); 823 continue; 824 } else if (PageTransHuge(page)) { 825 if (index == round_down(end, HPAGE_PMD_NR)) { 826 /* 827 * Range ends in the middle of THP: 828 * zero out the page 829 */ 830 clear_highpage(page); 831 unlock_page(page); 832 continue; 833 } 834 index += HPAGE_PMD_NR - 1; 835 i += HPAGE_PMD_NR - 1; 836 } 837 838 if (!unfalloc || !PageUptodate(page)) { 839 VM_BUG_ON_PAGE(PageTail(page), page); 840 if (page_mapping(page) == mapping) { 841 VM_BUG_ON_PAGE(PageWriteback(page), page); 842 truncate_inode_page(mapping, page); 843 } 844 } 845 unlock_page(page); 846 } 847 pagevec_remove_exceptionals(&pvec); 848 pagevec_release(&pvec); 849 cond_resched(); 850 index++; 851 } 852 853 if (partial_start) { 854 struct page *page = NULL; 855 shmem_getpage(inode, start - 1, &page, SGP_READ); 856 if (page) { 857 unsigned int top = PAGE_SIZE; 858 if (start > end) { 859 top = partial_end; 860 partial_end = 0; 861 } 862 zero_user_segment(page, partial_start, top); 863 set_page_dirty(page); 864 unlock_page(page); 865 put_page(page); 866 } 867 } 868 if (partial_end) { 869 struct page *page = NULL; 870 shmem_getpage(inode, end, &page, SGP_READ); 871 if (page) { 872 zero_user_segment(page, 0, partial_end); 873 set_page_dirty(page); 874 unlock_page(page); 875 put_page(page); 876 } 877 } 878 if (start >= end) 879 return; 880 881 index = start; 882 while (index < end) { 883 cond_resched(); 884 885 pvec.nr = find_get_entries(mapping, index, 886 min(end - index, (pgoff_t)PAGEVEC_SIZE), 887 pvec.pages, indices); 888 if (!pvec.nr) { 889 /* If all gone or hole-punch or unfalloc, we're done */ 890 if (index == start || end != -1) 891 break; 892 /* But if truncating, restart to make sure all gone */ 893 index = start; 894 continue; 895 } 896 for (i = 0; i < pagevec_count(&pvec); i++) { 897 struct page *page = pvec.pages[i]; 898 899 index = indices[i]; 900 if (index >= end) 901 break; 902 903 if (xa_is_value(page)) { 904 if (unfalloc) 905 continue; 906 if (shmem_free_swap(mapping, index, page)) { 907 /* Swap was replaced by page: retry */ 908 index--; 909 break; 910 } 911 nr_swaps_freed++; 912 continue; 913 } 914 915 lock_page(page); 916 917 if (PageTransTail(page)) { 918 /* Middle of THP: zero out the page */ 919 clear_highpage(page); 920 unlock_page(page); 921 /* 922 * Partial thp truncate due 'start' in middle 923 * of THP: don't need to look on these pages 924 * again on !pvec.nr restart. 925 */ 926 if (index != round_down(end, HPAGE_PMD_NR)) 927 start++; 928 continue; 929 } else if (PageTransHuge(page)) { 930 if (index == round_down(end, HPAGE_PMD_NR)) { 931 /* 932 * Range ends in the middle of THP: 933 * zero out the page 934 */ 935 clear_highpage(page); 936 unlock_page(page); 937 continue; 938 } 939 index += HPAGE_PMD_NR - 1; 940 i += HPAGE_PMD_NR - 1; 941 } 942 943 if (!unfalloc || !PageUptodate(page)) { 944 VM_BUG_ON_PAGE(PageTail(page), page); 945 if (page_mapping(page) == mapping) { 946 VM_BUG_ON_PAGE(PageWriteback(page), page); 947 truncate_inode_page(mapping, page); 948 } else { 949 /* Page was replaced by swap: retry */ 950 unlock_page(page); 951 index--; 952 break; 953 } 954 } 955 unlock_page(page); 956 } 957 pagevec_remove_exceptionals(&pvec); 958 pagevec_release(&pvec); 959 index++; 960 } 961 962 spin_lock_irq(&info->lock); 963 info->swapped -= nr_swaps_freed; 964 shmem_recalc_inode(inode); 965 spin_unlock_irq(&info->lock); 966 } 967 968 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 969 { 970 shmem_undo_range(inode, lstart, lend, false); 971 inode->i_ctime = inode->i_mtime = current_time(inode); 972 } 973 EXPORT_SYMBOL_GPL(shmem_truncate_range); 974 975 static int shmem_getattr(const struct path *path, struct kstat *stat, 976 u32 request_mask, unsigned int query_flags) 977 { 978 struct inode *inode = path->dentry->d_inode; 979 struct shmem_inode_info *info = SHMEM_I(inode); 980 struct shmem_sb_info *sb_info = SHMEM_SB(inode->i_sb); 981 982 if (info->alloced - info->swapped != inode->i_mapping->nrpages) { 983 spin_lock_irq(&info->lock); 984 shmem_recalc_inode(inode); 985 spin_unlock_irq(&info->lock); 986 } 987 generic_fillattr(inode, stat); 988 989 if (is_huge_enabled(sb_info)) 990 stat->blksize = HPAGE_PMD_SIZE; 991 992 return 0; 993 } 994 995 static int shmem_setattr(struct dentry *dentry, struct iattr *attr) 996 { 997 struct inode *inode = d_inode(dentry); 998 struct shmem_inode_info *info = SHMEM_I(inode); 999 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1000 int error; 1001 1002 error = setattr_prepare(dentry, attr); 1003 if (error) 1004 return error; 1005 1006 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 1007 loff_t oldsize = inode->i_size; 1008 loff_t newsize = attr->ia_size; 1009 1010 /* protected by i_mutex */ 1011 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || 1012 (newsize > oldsize && (info->seals & F_SEAL_GROW))) 1013 return -EPERM; 1014 1015 if (newsize != oldsize) { 1016 error = shmem_reacct_size(SHMEM_I(inode)->flags, 1017 oldsize, newsize); 1018 if (error) 1019 return error; 1020 i_size_write(inode, newsize); 1021 inode->i_ctime = inode->i_mtime = current_time(inode); 1022 } 1023 if (newsize <= oldsize) { 1024 loff_t holebegin = round_up(newsize, PAGE_SIZE); 1025 if (oldsize > holebegin) 1026 unmap_mapping_range(inode->i_mapping, 1027 holebegin, 0, 1); 1028 if (info->alloced) 1029 shmem_truncate_range(inode, 1030 newsize, (loff_t)-1); 1031 /* unmap again to remove racily COWed private pages */ 1032 if (oldsize > holebegin) 1033 unmap_mapping_range(inode->i_mapping, 1034 holebegin, 0, 1); 1035 1036 /* 1037 * Part of the huge page can be beyond i_size: subject 1038 * to shrink under memory pressure. 1039 */ 1040 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) { 1041 spin_lock(&sbinfo->shrinklist_lock); 1042 /* 1043 * _careful to defend against unlocked access to 1044 * ->shrink_list in shmem_unused_huge_shrink() 1045 */ 1046 if (list_empty_careful(&info->shrinklist)) { 1047 list_add_tail(&info->shrinklist, 1048 &sbinfo->shrinklist); 1049 sbinfo->shrinklist_len++; 1050 } 1051 spin_unlock(&sbinfo->shrinklist_lock); 1052 } 1053 } 1054 } 1055 1056 setattr_copy(inode, attr); 1057 if (attr->ia_valid & ATTR_MODE) 1058 error = posix_acl_chmod(inode, inode->i_mode); 1059 return error; 1060 } 1061 1062 static void shmem_evict_inode(struct inode *inode) 1063 { 1064 struct shmem_inode_info *info = SHMEM_I(inode); 1065 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1066 1067 if (inode->i_mapping->a_ops == &shmem_aops) { 1068 shmem_unacct_size(info->flags, inode->i_size); 1069 inode->i_size = 0; 1070 shmem_truncate_range(inode, 0, (loff_t)-1); 1071 if (!list_empty(&info->shrinklist)) { 1072 spin_lock(&sbinfo->shrinklist_lock); 1073 if (!list_empty(&info->shrinklist)) { 1074 list_del_init(&info->shrinklist); 1075 sbinfo->shrinklist_len--; 1076 } 1077 spin_unlock(&sbinfo->shrinklist_lock); 1078 } 1079 if (!list_empty(&info->swaplist)) { 1080 mutex_lock(&shmem_swaplist_mutex); 1081 list_del_init(&info->swaplist); 1082 mutex_unlock(&shmem_swaplist_mutex); 1083 } 1084 } 1085 1086 simple_xattrs_free(&info->xattrs); 1087 WARN_ON(inode->i_blocks); 1088 shmem_free_inode(inode->i_sb); 1089 clear_inode(inode); 1090 } 1091 1092 static unsigned long find_swap_entry(struct xarray *xa, void *item) 1093 { 1094 XA_STATE(xas, xa, 0); 1095 unsigned int checked = 0; 1096 void *entry; 1097 1098 rcu_read_lock(); 1099 xas_for_each(&xas, entry, ULONG_MAX) { 1100 if (xas_retry(&xas, entry)) 1101 continue; 1102 if (entry == item) 1103 break; 1104 checked++; 1105 if ((checked % XA_CHECK_SCHED) != 0) 1106 continue; 1107 xas_pause(&xas); 1108 cond_resched_rcu(); 1109 } 1110 rcu_read_unlock(); 1111 1112 return entry ? xas.xa_index : -1; 1113 } 1114 1115 /* 1116 * If swap found in inode, free it and move page from swapcache to filecache. 1117 */ 1118 static int shmem_unuse_inode(struct shmem_inode_info *info, 1119 swp_entry_t swap, struct page **pagep) 1120 { 1121 struct address_space *mapping = info->vfs_inode.i_mapping; 1122 void *radswap; 1123 pgoff_t index; 1124 gfp_t gfp; 1125 int error = 0; 1126 1127 radswap = swp_to_radix_entry(swap); 1128 index = find_swap_entry(&mapping->i_pages, radswap); 1129 if (index == -1) 1130 return -EAGAIN; /* tell shmem_unuse we found nothing */ 1131 1132 /* 1133 * Move _head_ to start search for next from here. 1134 * But be careful: shmem_evict_inode checks list_empty without taking 1135 * mutex, and there's an instant in list_move_tail when info->swaplist 1136 * would appear empty, if it were the only one on shmem_swaplist. 1137 */ 1138 if (shmem_swaplist.next != &info->swaplist) 1139 list_move_tail(&shmem_swaplist, &info->swaplist); 1140 1141 gfp = mapping_gfp_mask(mapping); 1142 if (shmem_should_replace_page(*pagep, gfp)) { 1143 mutex_unlock(&shmem_swaplist_mutex); 1144 error = shmem_replace_page(pagep, gfp, info, index); 1145 mutex_lock(&shmem_swaplist_mutex); 1146 /* 1147 * We needed to drop mutex to make that restrictive page 1148 * allocation, but the inode might have been freed while we 1149 * dropped it: although a racing shmem_evict_inode() cannot 1150 * complete without emptying the page cache, our page lock 1151 * on this swapcache page is not enough to prevent that - 1152 * free_swap_and_cache() of our swap entry will only 1153 * trylock_page(), removing swap from page cache whatever. 1154 * 1155 * We must not proceed to shmem_add_to_page_cache() if the 1156 * inode has been freed, but of course we cannot rely on 1157 * inode or mapping or info to check that. However, we can 1158 * safely check if our swap entry is still in use (and here 1159 * it can't have got reused for another page): if it's still 1160 * in use, then the inode cannot have been freed yet, and we 1161 * can safely proceed (if it's no longer in use, that tells 1162 * nothing about the inode, but we don't need to unuse swap). 1163 */ 1164 if (!page_swapcount(*pagep)) 1165 error = -ENOENT; 1166 } 1167 1168 /* 1169 * We rely on shmem_swaplist_mutex, not only to protect the swaplist, 1170 * but also to hold up shmem_evict_inode(): so inode cannot be freed 1171 * beneath us (pagelock doesn't help until the page is in pagecache). 1172 */ 1173 if (!error) 1174 error = shmem_add_to_page_cache(*pagep, mapping, index, 1175 radswap, gfp); 1176 if (error != -ENOMEM) { 1177 /* 1178 * Truncation and eviction use free_swap_and_cache(), which 1179 * only does trylock page: if we raced, best clean up here. 1180 */ 1181 delete_from_swap_cache(*pagep); 1182 set_page_dirty(*pagep); 1183 if (!error) { 1184 spin_lock_irq(&info->lock); 1185 info->swapped--; 1186 spin_unlock_irq(&info->lock); 1187 swap_free(swap); 1188 } 1189 } 1190 return error; 1191 } 1192 1193 /* 1194 * Search through swapped inodes to find and replace swap by page. 1195 */ 1196 int shmem_unuse(swp_entry_t swap, struct page *page) 1197 { 1198 struct list_head *this, *next; 1199 struct shmem_inode_info *info; 1200 struct mem_cgroup *memcg; 1201 int error = 0; 1202 1203 /* 1204 * There's a faint possibility that swap page was replaced before 1205 * caller locked it: caller will come back later with the right page. 1206 */ 1207 if (unlikely(!PageSwapCache(page) || page_private(page) != swap.val)) 1208 goto out; 1209 1210 /* 1211 * Charge page using GFP_KERNEL while we can wait, before taking 1212 * the shmem_swaplist_mutex which might hold up shmem_writepage(). 1213 * Charged back to the user (not to caller) when swap account is used. 1214 */ 1215 error = mem_cgroup_try_charge_delay(page, current->mm, GFP_KERNEL, 1216 &memcg, false); 1217 if (error) 1218 goto out; 1219 /* No memory allocation: swap entry occupies the slot for the page */ 1220 error = -EAGAIN; 1221 1222 mutex_lock(&shmem_swaplist_mutex); 1223 list_for_each_safe(this, next, &shmem_swaplist) { 1224 info = list_entry(this, struct shmem_inode_info, swaplist); 1225 if (info->swapped) 1226 error = shmem_unuse_inode(info, swap, &page); 1227 else 1228 list_del_init(&info->swaplist); 1229 cond_resched(); 1230 if (error != -EAGAIN) 1231 break; 1232 /* found nothing in this: move on to search the next */ 1233 } 1234 mutex_unlock(&shmem_swaplist_mutex); 1235 1236 if (error) { 1237 if (error != -ENOMEM) 1238 error = 0; 1239 mem_cgroup_cancel_charge(page, memcg, false); 1240 } else 1241 mem_cgroup_commit_charge(page, memcg, true, false); 1242 out: 1243 unlock_page(page); 1244 put_page(page); 1245 return error; 1246 } 1247 1248 /* 1249 * Move the page from the page cache to the swap cache. 1250 */ 1251 static int shmem_writepage(struct page *page, struct writeback_control *wbc) 1252 { 1253 struct shmem_inode_info *info; 1254 struct address_space *mapping; 1255 struct inode *inode; 1256 swp_entry_t swap; 1257 pgoff_t index; 1258 1259 VM_BUG_ON_PAGE(PageCompound(page), page); 1260 BUG_ON(!PageLocked(page)); 1261 mapping = page->mapping; 1262 index = page->index; 1263 inode = mapping->host; 1264 info = SHMEM_I(inode); 1265 if (info->flags & VM_LOCKED) 1266 goto redirty; 1267 if (!total_swap_pages) 1268 goto redirty; 1269 1270 /* 1271 * Our capabilities prevent regular writeback or sync from ever calling 1272 * shmem_writepage; but a stacking filesystem might use ->writepage of 1273 * its underlying filesystem, in which case tmpfs should write out to 1274 * swap only in response to memory pressure, and not for the writeback 1275 * threads or sync. 1276 */ 1277 if (!wbc->for_reclaim) { 1278 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */ 1279 goto redirty; 1280 } 1281 1282 /* 1283 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC 1284 * value into swapfile.c, the only way we can correctly account for a 1285 * fallocated page arriving here is now to initialize it and write it. 1286 * 1287 * That's okay for a page already fallocated earlier, but if we have 1288 * not yet completed the fallocation, then (a) we want to keep track 1289 * of this page in case we have to undo it, and (b) it may not be a 1290 * good idea to continue anyway, once we're pushing into swap. So 1291 * reactivate the page, and let shmem_fallocate() quit when too many. 1292 */ 1293 if (!PageUptodate(page)) { 1294 if (inode->i_private) { 1295 struct shmem_falloc *shmem_falloc; 1296 spin_lock(&inode->i_lock); 1297 shmem_falloc = inode->i_private; 1298 if (shmem_falloc && 1299 !shmem_falloc->waitq && 1300 index >= shmem_falloc->start && 1301 index < shmem_falloc->next) 1302 shmem_falloc->nr_unswapped++; 1303 else 1304 shmem_falloc = NULL; 1305 spin_unlock(&inode->i_lock); 1306 if (shmem_falloc) 1307 goto redirty; 1308 } 1309 clear_highpage(page); 1310 flush_dcache_page(page); 1311 SetPageUptodate(page); 1312 } 1313 1314 swap = get_swap_page(page); 1315 if (!swap.val) 1316 goto redirty; 1317 1318 /* 1319 * Add inode to shmem_unuse()'s list of swapped-out inodes, 1320 * if it's not already there. Do it now before the page is 1321 * moved to swap cache, when its pagelock no longer protects 1322 * the inode from eviction. But don't unlock the mutex until 1323 * we've incremented swapped, because shmem_unuse_inode() will 1324 * prune a !swapped inode from the swaplist under this mutex. 1325 */ 1326 mutex_lock(&shmem_swaplist_mutex); 1327 if (list_empty(&info->swaplist)) 1328 list_add_tail(&info->swaplist, &shmem_swaplist); 1329 1330 if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) { 1331 spin_lock_irq(&info->lock); 1332 shmem_recalc_inode(inode); 1333 info->swapped++; 1334 spin_unlock_irq(&info->lock); 1335 1336 swap_shmem_alloc(swap); 1337 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap)); 1338 1339 mutex_unlock(&shmem_swaplist_mutex); 1340 BUG_ON(page_mapped(page)); 1341 swap_writepage(page, wbc); 1342 return 0; 1343 } 1344 1345 mutex_unlock(&shmem_swaplist_mutex); 1346 put_swap_page(page, swap); 1347 redirty: 1348 set_page_dirty(page); 1349 if (wbc->for_reclaim) 1350 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */ 1351 unlock_page(page); 1352 return 0; 1353 } 1354 1355 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) 1356 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1357 { 1358 char buffer[64]; 1359 1360 if (!mpol || mpol->mode == MPOL_DEFAULT) 1361 return; /* show nothing */ 1362 1363 mpol_to_str(buffer, sizeof(buffer), mpol); 1364 1365 seq_printf(seq, ",mpol=%s", buffer); 1366 } 1367 1368 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1369 { 1370 struct mempolicy *mpol = NULL; 1371 if (sbinfo->mpol) { 1372 spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 1373 mpol = sbinfo->mpol; 1374 mpol_get(mpol); 1375 spin_unlock(&sbinfo->stat_lock); 1376 } 1377 return mpol; 1378 } 1379 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */ 1380 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1381 { 1382 } 1383 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1384 { 1385 return NULL; 1386 } 1387 #endif /* CONFIG_NUMA && CONFIG_TMPFS */ 1388 #ifndef CONFIG_NUMA 1389 #define vm_policy vm_private_data 1390 #endif 1391 1392 static void shmem_pseudo_vma_init(struct vm_area_struct *vma, 1393 struct shmem_inode_info *info, pgoff_t index) 1394 { 1395 /* Create a pseudo vma that just contains the policy */ 1396 vma_init(vma, NULL); 1397 /* Bias interleave by inode number to distribute better across nodes */ 1398 vma->vm_pgoff = index + info->vfs_inode.i_ino; 1399 vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index); 1400 } 1401 1402 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma) 1403 { 1404 /* Drop reference taken by mpol_shared_policy_lookup() */ 1405 mpol_cond_put(vma->vm_policy); 1406 } 1407 1408 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, 1409 struct shmem_inode_info *info, pgoff_t index) 1410 { 1411 struct vm_area_struct pvma; 1412 struct page *page; 1413 struct vm_fault vmf; 1414 1415 shmem_pseudo_vma_init(&pvma, info, index); 1416 vmf.vma = &pvma; 1417 vmf.address = 0; 1418 page = swap_cluster_readahead(swap, gfp, &vmf); 1419 shmem_pseudo_vma_destroy(&pvma); 1420 1421 return page; 1422 } 1423 1424 static struct page *shmem_alloc_hugepage(gfp_t gfp, 1425 struct shmem_inode_info *info, pgoff_t index) 1426 { 1427 struct vm_area_struct pvma; 1428 struct address_space *mapping = info->vfs_inode.i_mapping; 1429 pgoff_t hindex; 1430 struct page *page; 1431 1432 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1433 return NULL; 1434 1435 hindex = round_down(index, HPAGE_PMD_NR); 1436 if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1, 1437 XA_PRESENT)) 1438 return NULL; 1439 1440 shmem_pseudo_vma_init(&pvma, info, hindex); 1441 page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN, 1442 HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true); 1443 shmem_pseudo_vma_destroy(&pvma); 1444 if (page) 1445 prep_transhuge_page(page); 1446 return page; 1447 } 1448 1449 static struct page *shmem_alloc_page(gfp_t gfp, 1450 struct shmem_inode_info *info, pgoff_t index) 1451 { 1452 struct vm_area_struct pvma; 1453 struct page *page; 1454 1455 shmem_pseudo_vma_init(&pvma, info, index); 1456 page = alloc_page_vma(gfp, &pvma, 0); 1457 shmem_pseudo_vma_destroy(&pvma); 1458 1459 return page; 1460 } 1461 1462 static struct page *shmem_alloc_and_acct_page(gfp_t gfp, 1463 struct inode *inode, 1464 pgoff_t index, bool huge) 1465 { 1466 struct shmem_inode_info *info = SHMEM_I(inode); 1467 struct page *page; 1468 int nr; 1469 int err = -ENOSPC; 1470 1471 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 1472 huge = false; 1473 nr = huge ? HPAGE_PMD_NR : 1; 1474 1475 if (!shmem_inode_acct_block(inode, nr)) 1476 goto failed; 1477 1478 if (huge) 1479 page = shmem_alloc_hugepage(gfp, info, index); 1480 else 1481 page = shmem_alloc_page(gfp, info, index); 1482 if (page) { 1483 __SetPageLocked(page); 1484 __SetPageSwapBacked(page); 1485 return page; 1486 } 1487 1488 err = -ENOMEM; 1489 shmem_inode_unacct_blocks(inode, nr); 1490 failed: 1491 return ERR_PTR(err); 1492 } 1493 1494 /* 1495 * When a page is moved from swapcache to shmem filecache (either by the 1496 * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of 1497 * shmem_unuse_inode()), it may have been read in earlier from swap, in 1498 * ignorance of the mapping it belongs to. If that mapping has special 1499 * constraints (like the gma500 GEM driver, which requires RAM below 4GB), 1500 * we may need to copy to a suitable page before moving to filecache. 1501 * 1502 * In a future release, this may well be extended to respect cpuset and 1503 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page(); 1504 * but for now it is a simple matter of zone. 1505 */ 1506 static bool shmem_should_replace_page(struct page *page, gfp_t gfp) 1507 { 1508 return page_zonenum(page) > gfp_zone(gfp); 1509 } 1510 1511 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 1512 struct shmem_inode_info *info, pgoff_t index) 1513 { 1514 struct page *oldpage, *newpage; 1515 struct address_space *swap_mapping; 1516 swp_entry_t entry; 1517 pgoff_t swap_index; 1518 int error; 1519 1520 oldpage = *pagep; 1521 entry.val = page_private(oldpage); 1522 swap_index = swp_offset(entry); 1523 swap_mapping = page_mapping(oldpage); 1524 1525 /* 1526 * We have arrived here because our zones are constrained, so don't 1527 * limit chance of success by further cpuset and node constraints. 1528 */ 1529 gfp &= ~GFP_CONSTRAINT_MASK; 1530 newpage = shmem_alloc_page(gfp, info, index); 1531 if (!newpage) 1532 return -ENOMEM; 1533 1534 get_page(newpage); 1535 copy_highpage(newpage, oldpage); 1536 flush_dcache_page(newpage); 1537 1538 __SetPageLocked(newpage); 1539 __SetPageSwapBacked(newpage); 1540 SetPageUptodate(newpage); 1541 set_page_private(newpage, entry.val); 1542 SetPageSwapCache(newpage); 1543 1544 /* 1545 * Our caller will very soon move newpage out of swapcache, but it's 1546 * a nice clean interface for us to replace oldpage by newpage there. 1547 */ 1548 xa_lock_irq(&swap_mapping->i_pages); 1549 error = shmem_replace_entry(swap_mapping, swap_index, oldpage, newpage); 1550 if (!error) { 1551 __inc_node_page_state(newpage, NR_FILE_PAGES); 1552 __dec_node_page_state(oldpage, NR_FILE_PAGES); 1553 } 1554 xa_unlock_irq(&swap_mapping->i_pages); 1555 1556 if (unlikely(error)) { 1557 /* 1558 * Is this possible? I think not, now that our callers check 1559 * both PageSwapCache and page_private after getting page lock; 1560 * but be defensive. Reverse old to newpage for clear and free. 1561 */ 1562 oldpage = newpage; 1563 } else { 1564 mem_cgroup_migrate(oldpage, newpage); 1565 lru_cache_add_anon(newpage); 1566 *pagep = newpage; 1567 } 1568 1569 ClearPageSwapCache(oldpage); 1570 set_page_private(oldpage, 0); 1571 1572 unlock_page(oldpage); 1573 put_page(oldpage); 1574 put_page(oldpage); 1575 return error; 1576 } 1577 1578 /* 1579 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate 1580 * 1581 * If we allocate a new one we do not mark it dirty. That's up to the 1582 * vm. If we swap it in we mark it dirty since we also free the swap 1583 * entry since a page cannot live in both the swap and page cache. 1584 * 1585 * fault_mm and fault_type are only supplied by shmem_fault: 1586 * otherwise they are NULL. 1587 */ 1588 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 1589 struct page **pagep, enum sgp_type sgp, gfp_t gfp, 1590 struct vm_area_struct *vma, struct vm_fault *vmf, 1591 vm_fault_t *fault_type) 1592 { 1593 struct address_space *mapping = inode->i_mapping; 1594 struct shmem_inode_info *info = SHMEM_I(inode); 1595 struct shmem_sb_info *sbinfo; 1596 struct mm_struct *charge_mm; 1597 struct mem_cgroup *memcg; 1598 struct page *page; 1599 swp_entry_t swap; 1600 enum sgp_type sgp_huge = sgp; 1601 pgoff_t hindex = index; 1602 int error; 1603 int once = 0; 1604 int alloced = 0; 1605 1606 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT)) 1607 return -EFBIG; 1608 if (sgp == SGP_NOHUGE || sgp == SGP_HUGE) 1609 sgp = SGP_CACHE; 1610 repeat: 1611 swap.val = 0; 1612 page = find_lock_entry(mapping, index); 1613 if (xa_is_value(page)) { 1614 swap = radix_to_swp_entry(page); 1615 page = NULL; 1616 } 1617 1618 if (sgp <= SGP_CACHE && 1619 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1620 error = -EINVAL; 1621 goto unlock; 1622 } 1623 1624 if (page && sgp == SGP_WRITE) 1625 mark_page_accessed(page); 1626 1627 /* fallocated page? */ 1628 if (page && !PageUptodate(page)) { 1629 if (sgp != SGP_READ) 1630 goto clear; 1631 unlock_page(page); 1632 put_page(page); 1633 page = NULL; 1634 } 1635 if (page || (sgp == SGP_READ && !swap.val)) { 1636 *pagep = page; 1637 return 0; 1638 } 1639 1640 /* 1641 * Fast cache lookup did not find it: 1642 * bring it back from swap or allocate. 1643 */ 1644 sbinfo = SHMEM_SB(inode->i_sb); 1645 charge_mm = vma ? vma->vm_mm : current->mm; 1646 1647 if (swap.val) { 1648 /* Look it up and read it in.. */ 1649 page = lookup_swap_cache(swap, NULL, 0); 1650 if (!page) { 1651 /* Or update major stats only when swapin succeeds?? */ 1652 if (fault_type) { 1653 *fault_type |= VM_FAULT_MAJOR; 1654 count_vm_event(PGMAJFAULT); 1655 count_memcg_event_mm(charge_mm, PGMAJFAULT); 1656 } 1657 /* Here we actually start the io */ 1658 page = shmem_swapin(swap, gfp, info, index); 1659 if (!page) { 1660 error = -ENOMEM; 1661 goto failed; 1662 } 1663 } 1664 1665 /* We have to do this with page locked to prevent races */ 1666 lock_page(page); 1667 if (!PageSwapCache(page) || page_private(page) != swap.val || 1668 !shmem_confirm_swap(mapping, index, swap)) { 1669 error = -EEXIST; /* try again */ 1670 goto unlock; 1671 } 1672 if (!PageUptodate(page)) { 1673 error = -EIO; 1674 goto failed; 1675 } 1676 wait_on_page_writeback(page); 1677 1678 if (shmem_should_replace_page(page, gfp)) { 1679 error = shmem_replace_page(&page, gfp, info, index); 1680 if (error) 1681 goto failed; 1682 } 1683 1684 error = mem_cgroup_try_charge_delay(page, charge_mm, gfp, &memcg, 1685 false); 1686 if (!error) { 1687 error = shmem_add_to_page_cache(page, mapping, index, 1688 swp_to_radix_entry(swap), gfp); 1689 /* 1690 * We already confirmed swap under page lock, and make 1691 * no memory allocation here, so usually no possibility 1692 * of error; but free_swap_and_cache() only trylocks a 1693 * page, so it is just possible that the entry has been 1694 * truncated or holepunched since swap was confirmed. 1695 * shmem_undo_range() will have done some of the 1696 * unaccounting, now delete_from_swap_cache() will do 1697 * the rest. 1698 * Reset swap.val? No, leave it so "failed" goes back to 1699 * "repeat": reading a hole and writing should succeed. 1700 */ 1701 if (error) { 1702 mem_cgroup_cancel_charge(page, memcg, false); 1703 delete_from_swap_cache(page); 1704 } 1705 } 1706 if (error) 1707 goto failed; 1708 1709 mem_cgroup_commit_charge(page, memcg, true, false); 1710 1711 spin_lock_irq(&info->lock); 1712 info->swapped--; 1713 shmem_recalc_inode(inode); 1714 spin_unlock_irq(&info->lock); 1715 1716 if (sgp == SGP_WRITE) 1717 mark_page_accessed(page); 1718 1719 delete_from_swap_cache(page); 1720 set_page_dirty(page); 1721 swap_free(swap); 1722 1723 } else { 1724 if (vma && userfaultfd_missing(vma)) { 1725 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING); 1726 return 0; 1727 } 1728 1729 /* shmem_symlink() */ 1730 if (mapping->a_ops != &shmem_aops) 1731 goto alloc_nohuge; 1732 if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE) 1733 goto alloc_nohuge; 1734 if (shmem_huge == SHMEM_HUGE_FORCE) 1735 goto alloc_huge; 1736 switch (sbinfo->huge) { 1737 loff_t i_size; 1738 pgoff_t off; 1739 case SHMEM_HUGE_NEVER: 1740 goto alloc_nohuge; 1741 case SHMEM_HUGE_WITHIN_SIZE: 1742 off = round_up(index, HPAGE_PMD_NR); 1743 i_size = round_up(i_size_read(inode), PAGE_SIZE); 1744 if (i_size >= HPAGE_PMD_SIZE && 1745 i_size >> PAGE_SHIFT >= off) 1746 goto alloc_huge; 1747 /* fallthrough */ 1748 case SHMEM_HUGE_ADVISE: 1749 if (sgp_huge == SGP_HUGE) 1750 goto alloc_huge; 1751 /* TODO: implement fadvise() hints */ 1752 goto alloc_nohuge; 1753 } 1754 1755 alloc_huge: 1756 page = shmem_alloc_and_acct_page(gfp, inode, index, true); 1757 if (IS_ERR(page)) { 1758 alloc_nohuge: page = shmem_alloc_and_acct_page(gfp, inode, 1759 index, false); 1760 } 1761 if (IS_ERR(page)) { 1762 int retry = 5; 1763 error = PTR_ERR(page); 1764 page = NULL; 1765 if (error != -ENOSPC) 1766 goto failed; 1767 /* 1768 * Try to reclaim some spece by splitting a huge page 1769 * beyond i_size on the filesystem. 1770 */ 1771 while (retry--) { 1772 int ret; 1773 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1); 1774 if (ret == SHRINK_STOP) 1775 break; 1776 if (ret) 1777 goto alloc_nohuge; 1778 } 1779 goto failed; 1780 } 1781 1782 if (PageTransHuge(page)) 1783 hindex = round_down(index, HPAGE_PMD_NR); 1784 else 1785 hindex = index; 1786 1787 if (sgp == SGP_WRITE) 1788 __SetPageReferenced(page); 1789 1790 error = mem_cgroup_try_charge_delay(page, charge_mm, gfp, &memcg, 1791 PageTransHuge(page)); 1792 if (error) 1793 goto unacct; 1794 error = shmem_add_to_page_cache(page, mapping, hindex, 1795 NULL, gfp & GFP_RECLAIM_MASK); 1796 if (error) { 1797 mem_cgroup_cancel_charge(page, memcg, 1798 PageTransHuge(page)); 1799 goto unacct; 1800 } 1801 mem_cgroup_commit_charge(page, memcg, false, 1802 PageTransHuge(page)); 1803 lru_cache_add_anon(page); 1804 1805 spin_lock_irq(&info->lock); 1806 info->alloced += 1 << compound_order(page); 1807 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page); 1808 shmem_recalc_inode(inode); 1809 spin_unlock_irq(&info->lock); 1810 alloced = true; 1811 1812 if (PageTransHuge(page) && 1813 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < 1814 hindex + HPAGE_PMD_NR - 1) { 1815 /* 1816 * Part of the huge page is beyond i_size: subject 1817 * to shrink under memory pressure. 1818 */ 1819 spin_lock(&sbinfo->shrinklist_lock); 1820 /* 1821 * _careful to defend against unlocked access to 1822 * ->shrink_list in shmem_unused_huge_shrink() 1823 */ 1824 if (list_empty_careful(&info->shrinklist)) { 1825 list_add_tail(&info->shrinklist, 1826 &sbinfo->shrinklist); 1827 sbinfo->shrinklist_len++; 1828 } 1829 spin_unlock(&sbinfo->shrinklist_lock); 1830 } 1831 1832 /* 1833 * Let SGP_FALLOC use the SGP_WRITE optimization on a new page. 1834 */ 1835 if (sgp == SGP_FALLOC) 1836 sgp = SGP_WRITE; 1837 clear: 1838 /* 1839 * Let SGP_WRITE caller clear ends if write does not fill page; 1840 * but SGP_FALLOC on a page fallocated earlier must initialize 1841 * it now, lest undo on failure cancel our earlier guarantee. 1842 */ 1843 if (sgp != SGP_WRITE && !PageUptodate(page)) { 1844 struct page *head = compound_head(page); 1845 int i; 1846 1847 for (i = 0; i < (1 << compound_order(head)); i++) { 1848 clear_highpage(head + i); 1849 flush_dcache_page(head + i); 1850 } 1851 SetPageUptodate(head); 1852 } 1853 } 1854 1855 /* Perhaps the file has been truncated since we checked */ 1856 if (sgp <= SGP_CACHE && 1857 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1858 if (alloced) { 1859 ClearPageDirty(page); 1860 delete_from_page_cache(page); 1861 spin_lock_irq(&info->lock); 1862 shmem_recalc_inode(inode); 1863 spin_unlock_irq(&info->lock); 1864 } 1865 error = -EINVAL; 1866 goto unlock; 1867 } 1868 *pagep = page + index - hindex; 1869 return 0; 1870 1871 /* 1872 * Error recovery. 1873 */ 1874 unacct: 1875 shmem_inode_unacct_blocks(inode, 1 << compound_order(page)); 1876 1877 if (PageTransHuge(page)) { 1878 unlock_page(page); 1879 put_page(page); 1880 goto alloc_nohuge; 1881 } 1882 failed: 1883 if (swap.val && !shmem_confirm_swap(mapping, index, swap)) 1884 error = -EEXIST; 1885 unlock: 1886 if (page) { 1887 unlock_page(page); 1888 put_page(page); 1889 } 1890 if (error == -ENOSPC && !once++) { 1891 spin_lock_irq(&info->lock); 1892 shmem_recalc_inode(inode); 1893 spin_unlock_irq(&info->lock); 1894 goto repeat; 1895 } 1896 if (error == -EEXIST) 1897 goto repeat; 1898 return error; 1899 } 1900 1901 /* 1902 * This is like autoremove_wake_function, but it removes the wait queue 1903 * entry unconditionally - even if something else had already woken the 1904 * target. 1905 */ 1906 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) 1907 { 1908 int ret = default_wake_function(wait, mode, sync, key); 1909 list_del_init(&wait->entry); 1910 return ret; 1911 } 1912 1913 static vm_fault_t shmem_fault(struct vm_fault *vmf) 1914 { 1915 struct vm_area_struct *vma = vmf->vma; 1916 struct inode *inode = file_inode(vma->vm_file); 1917 gfp_t gfp = mapping_gfp_mask(inode->i_mapping); 1918 enum sgp_type sgp; 1919 int err; 1920 vm_fault_t ret = VM_FAULT_LOCKED; 1921 1922 /* 1923 * Trinity finds that probing a hole which tmpfs is punching can 1924 * prevent the hole-punch from ever completing: which in turn 1925 * locks writers out with its hold on i_mutex. So refrain from 1926 * faulting pages into the hole while it's being punched. Although 1927 * shmem_undo_range() does remove the additions, it may be unable to 1928 * keep up, as each new page needs its own unmap_mapping_range() call, 1929 * and the i_mmap tree grows ever slower to scan if new vmas are added. 1930 * 1931 * It does not matter if we sometimes reach this check just before the 1932 * hole-punch begins, so that one fault then races with the punch: 1933 * we just need to make racing faults a rare case. 1934 * 1935 * The implementation below would be much simpler if we just used a 1936 * standard mutex or completion: but we cannot take i_mutex in fault, 1937 * and bloating every shmem inode for this unlikely case would be sad. 1938 */ 1939 if (unlikely(inode->i_private)) { 1940 struct shmem_falloc *shmem_falloc; 1941 1942 spin_lock(&inode->i_lock); 1943 shmem_falloc = inode->i_private; 1944 if (shmem_falloc && 1945 shmem_falloc->waitq && 1946 vmf->pgoff >= shmem_falloc->start && 1947 vmf->pgoff < shmem_falloc->next) { 1948 wait_queue_head_t *shmem_falloc_waitq; 1949 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function); 1950 1951 ret = VM_FAULT_NOPAGE; 1952 if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) && 1953 !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { 1954 /* It's polite to up mmap_sem if we can */ 1955 up_read(&vma->vm_mm->mmap_sem); 1956 ret = VM_FAULT_RETRY; 1957 } 1958 1959 shmem_falloc_waitq = shmem_falloc->waitq; 1960 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait, 1961 TASK_UNINTERRUPTIBLE); 1962 spin_unlock(&inode->i_lock); 1963 schedule(); 1964 1965 /* 1966 * shmem_falloc_waitq points into the shmem_fallocate() 1967 * stack of the hole-punching task: shmem_falloc_waitq 1968 * is usually invalid by the time we reach here, but 1969 * finish_wait() does not dereference it in that case; 1970 * though i_lock needed lest racing with wake_up_all(). 1971 */ 1972 spin_lock(&inode->i_lock); 1973 finish_wait(shmem_falloc_waitq, &shmem_fault_wait); 1974 spin_unlock(&inode->i_lock); 1975 return ret; 1976 } 1977 spin_unlock(&inode->i_lock); 1978 } 1979 1980 sgp = SGP_CACHE; 1981 1982 if ((vma->vm_flags & VM_NOHUGEPAGE) || 1983 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)) 1984 sgp = SGP_NOHUGE; 1985 else if (vma->vm_flags & VM_HUGEPAGE) 1986 sgp = SGP_HUGE; 1987 1988 err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp, 1989 gfp, vma, vmf, &ret); 1990 if (err) 1991 return vmf_error(err); 1992 return ret; 1993 } 1994 1995 unsigned long shmem_get_unmapped_area(struct file *file, 1996 unsigned long uaddr, unsigned long len, 1997 unsigned long pgoff, unsigned long flags) 1998 { 1999 unsigned long (*get_area)(struct file *, 2000 unsigned long, unsigned long, unsigned long, unsigned long); 2001 unsigned long addr; 2002 unsigned long offset; 2003 unsigned long inflated_len; 2004 unsigned long inflated_addr; 2005 unsigned long inflated_offset; 2006 2007 if (len > TASK_SIZE) 2008 return -ENOMEM; 2009 2010 get_area = current->mm->get_unmapped_area; 2011 addr = get_area(file, uaddr, len, pgoff, flags); 2012 2013 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) 2014 return addr; 2015 if (IS_ERR_VALUE(addr)) 2016 return addr; 2017 if (addr & ~PAGE_MASK) 2018 return addr; 2019 if (addr > TASK_SIZE - len) 2020 return addr; 2021 2022 if (shmem_huge == SHMEM_HUGE_DENY) 2023 return addr; 2024 if (len < HPAGE_PMD_SIZE) 2025 return addr; 2026 if (flags & MAP_FIXED) 2027 return addr; 2028 /* 2029 * Our priority is to support MAP_SHARED mapped hugely; 2030 * and support MAP_PRIVATE mapped hugely too, until it is COWed. 2031 * But if caller specified an address hint, respect that as before. 2032 */ 2033 if (uaddr) 2034 return addr; 2035 2036 if (shmem_huge != SHMEM_HUGE_FORCE) { 2037 struct super_block *sb; 2038 2039 if (file) { 2040 VM_BUG_ON(file->f_op != &shmem_file_operations); 2041 sb = file_inode(file)->i_sb; 2042 } else { 2043 /* 2044 * Called directly from mm/mmap.c, or drivers/char/mem.c 2045 * for "/dev/zero", to create a shared anonymous object. 2046 */ 2047 if (IS_ERR(shm_mnt)) 2048 return addr; 2049 sb = shm_mnt->mnt_sb; 2050 } 2051 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER) 2052 return addr; 2053 } 2054 2055 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1); 2056 if (offset && offset + len < 2 * HPAGE_PMD_SIZE) 2057 return addr; 2058 if ((addr & (HPAGE_PMD_SIZE-1)) == offset) 2059 return addr; 2060 2061 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE; 2062 if (inflated_len > TASK_SIZE) 2063 return addr; 2064 if (inflated_len < len) 2065 return addr; 2066 2067 inflated_addr = get_area(NULL, 0, inflated_len, 0, flags); 2068 if (IS_ERR_VALUE(inflated_addr)) 2069 return addr; 2070 if (inflated_addr & ~PAGE_MASK) 2071 return addr; 2072 2073 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1); 2074 inflated_addr += offset - inflated_offset; 2075 if (inflated_offset > offset) 2076 inflated_addr += HPAGE_PMD_SIZE; 2077 2078 if (inflated_addr > TASK_SIZE - len) 2079 return addr; 2080 return inflated_addr; 2081 } 2082 2083 #ifdef CONFIG_NUMA 2084 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) 2085 { 2086 struct inode *inode = file_inode(vma->vm_file); 2087 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); 2088 } 2089 2090 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 2091 unsigned long addr) 2092 { 2093 struct inode *inode = file_inode(vma->vm_file); 2094 pgoff_t index; 2095 2096 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 2097 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); 2098 } 2099 #endif 2100 2101 int shmem_lock(struct file *file, int lock, struct user_struct *user) 2102 { 2103 struct inode *inode = file_inode(file); 2104 struct shmem_inode_info *info = SHMEM_I(inode); 2105 int retval = -ENOMEM; 2106 2107 spin_lock_irq(&info->lock); 2108 if (lock && !(info->flags & VM_LOCKED)) { 2109 if (!user_shm_lock(inode->i_size, user)) 2110 goto out_nomem; 2111 info->flags |= VM_LOCKED; 2112 mapping_set_unevictable(file->f_mapping); 2113 } 2114 if (!lock && (info->flags & VM_LOCKED) && user) { 2115 user_shm_unlock(inode->i_size, user); 2116 info->flags &= ~VM_LOCKED; 2117 mapping_clear_unevictable(file->f_mapping); 2118 } 2119 retval = 0; 2120 2121 out_nomem: 2122 spin_unlock_irq(&info->lock); 2123 return retval; 2124 } 2125 2126 static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 2127 { 2128 file_accessed(file); 2129 vma->vm_ops = &shmem_vm_ops; 2130 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && 2131 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 2132 (vma->vm_end & HPAGE_PMD_MASK)) { 2133 khugepaged_enter(vma, vma->vm_flags); 2134 } 2135 return 0; 2136 } 2137 2138 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir, 2139 umode_t mode, dev_t dev, unsigned long flags) 2140 { 2141 struct inode *inode; 2142 struct shmem_inode_info *info; 2143 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2144 2145 if (shmem_reserve_inode(sb)) 2146 return NULL; 2147 2148 inode = new_inode(sb); 2149 if (inode) { 2150 inode->i_ino = get_next_ino(); 2151 inode_init_owner(inode, dir, mode); 2152 inode->i_blocks = 0; 2153 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); 2154 inode->i_generation = prandom_u32(); 2155 info = SHMEM_I(inode); 2156 memset(info, 0, (char *)inode - (char *)info); 2157 spin_lock_init(&info->lock); 2158 info->seals = F_SEAL_SEAL; 2159 info->flags = flags & VM_NORESERVE; 2160 INIT_LIST_HEAD(&info->shrinklist); 2161 INIT_LIST_HEAD(&info->swaplist); 2162 simple_xattrs_init(&info->xattrs); 2163 cache_no_acl(inode); 2164 2165 switch (mode & S_IFMT) { 2166 default: 2167 inode->i_op = &shmem_special_inode_operations; 2168 init_special_inode(inode, mode, dev); 2169 break; 2170 case S_IFREG: 2171 inode->i_mapping->a_ops = &shmem_aops; 2172 inode->i_op = &shmem_inode_operations; 2173 inode->i_fop = &shmem_file_operations; 2174 mpol_shared_policy_init(&info->policy, 2175 shmem_get_sbmpol(sbinfo)); 2176 break; 2177 case S_IFDIR: 2178 inc_nlink(inode); 2179 /* Some things misbehave if size == 0 on a directory */ 2180 inode->i_size = 2 * BOGO_DIRENT_SIZE; 2181 inode->i_op = &shmem_dir_inode_operations; 2182 inode->i_fop = &simple_dir_operations; 2183 break; 2184 case S_IFLNK: 2185 /* 2186 * Must not load anything in the rbtree, 2187 * mpol_free_shared_policy will not be called. 2188 */ 2189 mpol_shared_policy_init(&info->policy, NULL); 2190 break; 2191 } 2192 2193 lockdep_annotate_inode_mutex_key(inode); 2194 } else 2195 shmem_free_inode(sb); 2196 return inode; 2197 } 2198 2199 bool shmem_mapping(struct address_space *mapping) 2200 { 2201 return mapping->a_ops == &shmem_aops; 2202 } 2203 2204 static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, 2205 pmd_t *dst_pmd, 2206 struct vm_area_struct *dst_vma, 2207 unsigned long dst_addr, 2208 unsigned long src_addr, 2209 bool zeropage, 2210 struct page **pagep) 2211 { 2212 struct inode *inode = file_inode(dst_vma->vm_file); 2213 struct shmem_inode_info *info = SHMEM_I(inode); 2214 struct address_space *mapping = inode->i_mapping; 2215 gfp_t gfp = mapping_gfp_mask(mapping); 2216 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); 2217 struct mem_cgroup *memcg; 2218 spinlock_t *ptl; 2219 void *page_kaddr; 2220 struct page *page; 2221 pte_t _dst_pte, *dst_pte; 2222 int ret; 2223 pgoff_t offset, max_off; 2224 2225 ret = -ENOMEM; 2226 if (!shmem_inode_acct_block(inode, 1)) 2227 goto out; 2228 2229 if (!*pagep) { 2230 page = shmem_alloc_page(gfp, info, pgoff); 2231 if (!page) 2232 goto out_unacct_blocks; 2233 2234 if (!zeropage) { /* mcopy_atomic */ 2235 page_kaddr = kmap_atomic(page); 2236 ret = copy_from_user(page_kaddr, 2237 (const void __user *)src_addr, 2238 PAGE_SIZE); 2239 kunmap_atomic(page_kaddr); 2240 2241 /* fallback to copy_from_user outside mmap_sem */ 2242 if (unlikely(ret)) { 2243 *pagep = page; 2244 shmem_inode_unacct_blocks(inode, 1); 2245 /* don't free the page */ 2246 return -ENOENT; 2247 } 2248 } else { /* mfill_zeropage_atomic */ 2249 clear_highpage(page); 2250 } 2251 } else { 2252 page = *pagep; 2253 *pagep = NULL; 2254 } 2255 2256 VM_BUG_ON(PageLocked(page) || PageSwapBacked(page)); 2257 __SetPageLocked(page); 2258 __SetPageSwapBacked(page); 2259 __SetPageUptodate(page); 2260 2261 ret = -EFAULT; 2262 offset = linear_page_index(dst_vma, dst_addr); 2263 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 2264 if (unlikely(offset >= max_off)) 2265 goto out_release; 2266 2267 ret = mem_cgroup_try_charge_delay(page, dst_mm, gfp, &memcg, false); 2268 if (ret) 2269 goto out_release; 2270 2271 ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL, 2272 gfp & GFP_RECLAIM_MASK); 2273 if (ret) 2274 goto out_release_uncharge; 2275 2276 mem_cgroup_commit_charge(page, memcg, false, false); 2277 2278 _dst_pte = mk_pte(page, dst_vma->vm_page_prot); 2279 if (dst_vma->vm_flags & VM_WRITE) 2280 _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte)); 2281 else { 2282 /* 2283 * We don't set the pte dirty if the vma has no 2284 * VM_WRITE permission, so mark the page dirty or it 2285 * could be freed from under us. We could do it 2286 * unconditionally before unlock_page(), but doing it 2287 * only if VM_WRITE is not set is faster. 2288 */ 2289 set_page_dirty(page); 2290 } 2291 2292 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); 2293 2294 ret = -EFAULT; 2295 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 2296 if (unlikely(offset >= max_off)) 2297 goto out_release_uncharge_unlock; 2298 2299 ret = -EEXIST; 2300 if (!pte_none(*dst_pte)) 2301 goto out_release_uncharge_unlock; 2302 2303 lru_cache_add_anon(page); 2304 2305 spin_lock(&info->lock); 2306 info->alloced++; 2307 inode->i_blocks += BLOCKS_PER_PAGE; 2308 shmem_recalc_inode(inode); 2309 spin_unlock(&info->lock); 2310 2311 inc_mm_counter(dst_mm, mm_counter_file(page)); 2312 page_add_file_rmap(page, false); 2313 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); 2314 2315 /* No need to invalidate - it was non-present before */ 2316 update_mmu_cache(dst_vma, dst_addr, dst_pte); 2317 pte_unmap_unlock(dst_pte, ptl); 2318 unlock_page(page); 2319 ret = 0; 2320 out: 2321 return ret; 2322 out_release_uncharge_unlock: 2323 pte_unmap_unlock(dst_pte, ptl); 2324 ClearPageDirty(page); 2325 delete_from_page_cache(page); 2326 out_release_uncharge: 2327 mem_cgroup_cancel_charge(page, memcg, false); 2328 out_release: 2329 unlock_page(page); 2330 put_page(page); 2331 out_unacct_blocks: 2332 shmem_inode_unacct_blocks(inode, 1); 2333 goto out; 2334 } 2335 2336 int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm, 2337 pmd_t *dst_pmd, 2338 struct vm_area_struct *dst_vma, 2339 unsigned long dst_addr, 2340 unsigned long src_addr, 2341 struct page **pagep) 2342 { 2343 return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, 2344 dst_addr, src_addr, false, pagep); 2345 } 2346 2347 int shmem_mfill_zeropage_pte(struct mm_struct *dst_mm, 2348 pmd_t *dst_pmd, 2349 struct vm_area_struct *dst_vma, 2350 unsigned long dst_addr) 2351 { 2352 struct page *page = NULL; 2353 2354 return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, 2355 dst_addr, 0, true, &page); 2356 } 2357 2358 #ifdef CONFIG_TMPFS 2359 static const struct inode_operations shmem_symlink_inode_operations; 2360 static const struct inode_operations shmem_short_symlink_operations; 2361 2362 #ifdef CONFIG_TMPFS_XATTR 2363 static int shmem_initxattrs(struct inode *, const struct xattr *, void *); 2364 #else 2365 #define shmem_initxattrs NULL 2366 #endif 2367 2368 static int 2369 shmem_write_begin(struct file *file, struct address_space *mapping, 2370 loff_t pos, unsigned len, unsigned flags, 2371 struct page **pagep, void **fsdata) 2372 { 2373 struct inode *inode = mapping->host; 2374 struct shmem_inode_info *info = SHMEM_I(inode); 2375 pgoff_t index = pos >> PAGE_SHIFT; 2376 2377 /* i_mutex is held by caller */ 2378 if (unlikely(info->seals & (F_SEAL_WRITE | F_SEAL_GROW))) { 2379 if (info->seals & F_SEAL_WRITE) 2380 return -EPERM; 2381 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size) 2382 return -EPERM; 2383 } 2384 2385 return shmem_getpage(inode, index, pagep, SGP_WRITE); 2386 } 2387 2388 static int 2389 shmem_write_end(struct file *file, struct address_space *mapping, 2390 loff_t pos, unsigned len, unsigned copied, 2391 struct page *page, void *fsdata) 2392 { 2393 struct inode *inode = mapping->host; 2394 2395 if (pos + copied > inode->i_size) 2396 i_size_write(inode, pos + copied); 2397 2398 if (!PageUptodate(page)) { 2399 struct page *head = compound_head(page); 2400 if (PageTransCompound(page)) { 2401 int i; 2402 2403 for (i = 0; i < HPAGE_PMD_NR; i++) { 2404 if (head + i == page) 2405 continue; 2406 clear_highpage(head + i); 2407 flush_dcache_page(head + i); 2408 } 2409 } 2410 if (copied < PAGE_SIZE) { 2411 unsigned from = pos & (PAGE_SIZE - 1); 2412 zero_user_segments(page, 0, from, 2413 from + copied, PAGE_SIZE); 2414 } 2415 SetPageUptodate(head); 2416 } 2417 set_page_dirty(page); 2418 unlock_page(page); 2419 put_page(page); 2420 2421 return copied; 2422 } 2423 2424 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 2425 { 2426 struct file *file = iocb->ki_filp; 2427 struct inode *inode = file_inode(file); 2428 struct address_space *mapping = inode->i_mapping; 2429 pgoff_t index; 2430 unsigned long offset; 2431 enum sgp_type sgp = SGP_READ; 2432 int error = 0; 2433 ssize_t retval = 0; 2434 loff_t *ppos = &iocb->ki_pos; 2435 2436 /* 2437 * Might this read be for a stacking filesystem? Then when reading 2438 * holes of a sparse file, we actually need to allocate those pages, 2439 * and even mark them dirty, so it cannot exceed the max_blocks limit. 2440 */ 2441 if (!iter_is_iovec(to)) 2442 sgp = SGP_CACHE; 2443 2444 index = *ppos >> PAGE_SHIFT; 2445 offset = *ppos & ~PAGE_MASK; 2446 2447 for (;;) { 2448 struct page *page = NULL; 2449 pgoff_t end_index; 2450 unsigned long nr, ret; 2451 loff_t i_size = i_size_read(inode); 2452 2453 end_index = i_size >> PAGE_SHIFT; 2454 if (index > end_index) 2455 break; 2456 if (index == end_index) { 2457 nr = i_size & ~PAGE_MASK; 2458 if (nr <= offset) 2459 break; 2460 } 2461 2462 error = shmem_getpage(inode, index, &page, sgp); 2463 if (error) { 2464 if (error == -EINVAL) 2465 error = 0; 2466 break; 2467 } 2468 if (page) { 2469 if (sgp == SGP_CACHE) 2470 set_page_dirty(page); 2471 unlock_page(page); 2472 } 2473 2474 /* 2475 * We must evaluate after, since reads (unlike writes) 2476 * are called without i_mutex protection against truncate 2477 */ 2478 nr = PAGE_SIZE; 2479 i_size = i_size_read(inode); 2480 end_index = i_size >> PAGE_SHIFT; 2481 if (index == end_index) { 2482 nr = i_size & ~PAGE_MASK; 2483 if (nr <= offset) { 2484 if (page) 2485 put_page(page); 2486 break; 2487 } 2488 } 2489 nr -= offset; 2490 2491 if (page) { 2492 /* 2493 * If users can be writing to this page using arbitrary 2494 * virtual addresses, take care about potential aliasing 2495 * before reading the page on the kernel side. 2496 */ 2497 if (mapping_writably_mapped(mapping)) 2498 flush_dcache_page(page); 2499 /* 2500 * Mark the page accessed if we read the beginning. 2501 */ 2502 if (!offset) 2503 mark_page_accessed(page); 2504 } else { 2505 page = ZERO_PAGE(0); 2506 get_page(page); 2507 } 2508 2509 /* 2510 * Ok, we have the page, and it's up-to-date, so 2511 * now we can copy it to user space... 2512 */ 2513 ret = copy_page_to_iter(page, offset, nr, to); 2514 retval += ret; 2515 offset += ret; 2516 index += offset >> PAGE_SHIFT; 2517 offset &= ~PAGE_MASK; 2518 2519 put_page(page); 2520 if (!iov_iter_count(to)) 2521 break; 2522 if (ret < nr) { 2523 error = -EFAULT; 2524 break; 2525 } 2526 cond_resched(); 2527 } 2528 2529 *ppos = ((loff_t) index << PAGE_SHIFT) + offset; 2530 file_accessed(file); 2531 return retval ? retval : error; 2532 } 2533 2534 /* 2535 * llseek SEEK_DATA or SEEK_HOLE through the page cache. 2536 */ 2537 static pgoff_t shmem_seek_hole_data(struct address_space *mapping, 2538 pgoff_t index, pgoff_t end, int whence) 2539 { 2540 struct page *page; 2541 struct pagevec pvec; 2542 pgoff_t indices[PAGEVEC_SIZE]; 2543 bool done = false; 2544 int i; 2545 2546 pagevec_init(&pvec); 2547 pvec.nr = 1; /* start small: we may be there already */ 2548 while (!done) { 2549 pvec.nr = find_get_entries(mapping, index, 2550 pvec.nr, pvec.pages, indices); 2551 if (!pvec.nr) { 2552 if (whence == SEEK_DATA) 2553 index = end; 2554 break; 2555 } 2556 for (i = 0; i < pvec.nr; i++, index++) { 2557 if (index < indices[i]) { 2558 if (whence == SEEK_HOLE) { 2559 done = true; 2560 break; 2561 } 2562 index = indices[i]; 2563 } 2564 page = pvec.pages[i]; 2565 if (page && !xa_is_value(page)) { 2566 if (!PageUptodate(page)) 2567 page = NULL; 2568 } 2569 if (index >= end || 2570 (page && whence == SEEK_DATA) || 2571 (!page && whence == SEEK_HOLE)) { 2572 done = true; 2573 break; 2574 } 2575 } 2576 pagevec_remove_exceptionals(&pvec); 2577 pagevec_release(&pvec); 2578 pvec.nr = PAGEVEC_SIZE; 2579 cond_resched(); 2580 } 2581 return index; 2582 } 2583 2584 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence) 2585 { 2586 struct address_space *mapping = file->f_mapping; 2587 struct inode *inode = mapping->host; 2588 pgoff_t start, end; 2589 loff_t new_offset; 2590 2591 if (whence != SEEK_DATA && whence != SEEK_HOLE) 2592 return generic_file_llseek_size(file, offset, whence, 2593 MAX_LFS_FILESIZE, i_size_read(inode)); 2594 inode_lock(inode); 2595 /* We're holding i_mutex so we can access i_size directly */ 2596 2597 if (offset < 0 || offset >= inode->i_size) 2598 offset = -ENXIO; 2599 else { 2600 start = offset >> PAGE_SHIFT; 2601 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT; 2602 new_offset = shmem_seek_hole_data(mapping, start, end, whence); 2603 new_offset <<= PAGE_SHIFT; 2604 if (new_offset > offset) { 2605 if (new_offset < inode->i_size) 2606 offset = new_offset; 2607 else if (whence == SEEK_DATA) 2608 offset = -ENXIO; 2609 else 2610 offset = inode->i_size; 2611 } 2612 } 2613 2614 if (offset >= 0) 2615 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE); 2616 inode_unlock(inode); 2617 return offset; 2618 } 2619 2620 static long shmem_fallocate(struct file *file, int mode, loff_t offset, 2621 loff_t len) 2622 { 2623 struct inode *inode = file_inode(file); 2624 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 2625 struct shmem_inode_info *info = SHMEM_I(inode); 2626 struct shmem_falloc shmem_falloc; 2627 pgoff_t start, index, end; 2628 int error; 2629 2630 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) 2631 return -EOPNOTSUPP; 2632 2633 inode_lock(inode); 2634 2635 if (mode & FALLOC_FL_PUNCH_HOLE) { 2636 struct address_space *mapping = file->f_mapping; 2637 loff_t unmap_start = round_up(offset, PAGE_SIZE); 2638 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1; 2639 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq); 2640 2641 /* protected by i_mutex */ 2642 if (info->seals & F_SEAL_WRITE) { 2643 error = -EPERM; 2644 goto out; 2645 } 2646 2647 shmem_falloc.waitq = &shmem_falloc_waitq; 2648 shmem_falloc.start = unmap_start >> PAGE_SHIFT; 2649 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; 2650 spin_lock(&inode->i_lock); 2651 inode->i_private = &shmem_falloc; 2652 spin_unlock(&inode->i_lock); 2653 2654 if ((u64)unmap_end > (u64)unmap_start) 2655 unmap_mapping_range(mapping, unmap_start, 2656 1 + unmap_end - unmap_start, 0); 2657 shmem_truncate_range(inode, offset, offset + len - 1); 2658 /* No need to unmap again: hole-punching leaves COWed pages */ 2659 2660 spin_lock(&inode->i_lock); 2661 inode->i_private = NULL; 2662 wake_up_all(&shmem_falloc_waitq); 2663 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head)); 2664 spin_unlock(&inode->i_lock); 2665 error = 0; 2666 goto out; 2667 } 2668 2669 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */ 2670 error = inode_newsize_ok(inode, offset + len); 2671 if (error) 2672 goto out; 2673 2674 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) { 2675 error = -EPERM; 2676 goto out; 2677 } 2678 2679 start = offset >> PAGE_SHIFT; 2680 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 2681 /* Try to avoid a swapstorm if len is impossible to satisfy */ 2682 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) { 2683 error = -ENOSPC; 2684 goto out; 2685 } 2686 2687 shmem_falloc.waitq = NULL; 2688 shmem_falloc.start = start; 2689 shmem_falloc.next = start; 2690 shmem_falloc.nr_falloced = 0; 2691 shmem_falloc.nr_unswapped = 0; 2692 spin_lock(&inode->i_lock); 2693 inode->i_private = &shmem_falloc; 2694 spin_unlock(&inode->i_lock); 2695 2696 for (index = start; index < end; index++) { 2697 struct page *page; 2698 2699 /* 2700 * Good, the fallocate(2) manpage permits EINTR: we may have 2701 * been interrupted because we are using up too much memory. 2702 */ 2703 if (signal_pending(current)) 2704 error = -EINTR; 2705 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) 2706 error = -ENOMEM; 2707 else 2708 error = shmem_getpage(inode, index, &page, SGP_FALLOC); 2709 if (error) { 2710 /* Remove the !PageUptodate pages we added */ 2711 if (index > start) { 2712 shmem_undo_range(inode, 2713 (loff_t)start << PAGE_SHIFT, 2714 ((loff_t)index << PAGE_SHIFT) - 1, true); 2715 } 2716 goto undone; 2717 } 2718 2719 /* 2720 * Inform shmem_writepage() how far we have reached. 2721 * No need for lock or barrier: we have the page lock. 2722 */ 2723 shmem_falloc.next++; 2724 if (!PageUptodate(page)) 2725 shmem_falloc.nr_falloced++; 2726 2727 /* 2728 * If !PageUptodate, leave it that way so that freeable pages 2729 * can be recognized if we need to rollback on error later. 2730 * But set_page_dirty so that memory pressure will swap rather 2731 * than free the pages we are allocating (and SGP_CACHE pages 2732 * might still be clean: we now need to mark those dirty too). 2733 */ 2734 set_page_dirty(page); 2735 unlock_page(page); 2736 put_page(page); 2737 cond_resched(); 2738 } 2739 2740 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) 2741 i_size_write(inode, offset + len); 2742 inode->i_ctime = current_time(inode); 2743 undone: 2744 spin_lock(&inode->i_lock); 2745 inode->i_private = NULL; 2746 spin_unlock(&inode->i_lock); 2747 out: 2748 inode_unlock(inode); 2749 return error; 2750 } 2751 2752 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 2753 { 2754 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 2755 2756 buf->f_type = TMPFS_MAGIC; 2757 buf->f_bsize = PAGE_SIZE; 2758 buf->f_namelen = NAME_MAX; 2759 if (sbinfo->max_blocks) { 2760 buf->f_blocks = sbinfo->max_blocks; 2761 buf->f_bavail = 2762 buf->f_bfree = sbinfo->max_blocks - 2763 percpu_counter_sum(&sbinfo->used_blocks); 2764 } 2765 if (sbinfo->max_inodes) { 2766 buf->f_files = sbinfo->max_inodes; 2767 buf->f_ffree = sbinfo->free_inodes; 2768 } 2769 /* else leave those fields 0 like simple_statfs */ 2770 return 0; 2771 } 2772 2773 /* 2774 * File creation. Allocate an inode, and we're done.. 2775 */ 2776 static int 2777 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) 2778 { 2779 struct inode *inode; 2780 int error = -ENOSPC; 2781 2782 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); 2783 if (inode) { 2784 error = simple_acl_create(dir, inode); 2785 if (error) 2786 goto out_iput; 2787 error = security_inode_init_security(inode, dir, 2788 &dentry->d_name, 2789 shmem_initxattrs, NULL); 2790 if (error && error != -EOPNOTSUPP) 2791 goto out_iput; 2792 2793 error = 0; 2794 dir->i_size += BOGO_DIRENT_SIZE; 2795 dir->i_ctime = dir->i_mtime = current_time(dir); 2796 d_instantiate(dentry, inode); 2797 dget(dentry); /* Extra count - pin the dentry in core */ 2798 } 2799 return error; 2800 out_iput: 2801 iput(inode); 2802 return error; 2803 } 2804 2805 static int 2806 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode) 2807 { 2808 struct inode *inode; 2809 int error = -ENOSPC; 2810 2811 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE); 2812 if (inode) { 2813 error = security_inode_init_security(inode, dir, 2814 NULL, 2815 shmem_initxattrs, NULL); 2816 if (error && error != -EOPNOTSUPP) 2817 goto out_iput; 2818 error = simple_acl_create(dir, inode); 2819 if (error) 2820 goto out_iput; 2821 d_tmpfile(dentry, inode); 2822 } 2823 return error; 2824 out_iput: 2825 iput(inode); 2826 return error; 2827 } 2828 2829 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) 2830 { 2831 int error; 2832 2833 if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0))) 2834 return error; 2835 inc_nlink(dir); 2836 return 0; 2837 } 2838 2839 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode, 2840 bool excl) 2841 { 2842 return shmem_mknod(dir, dentry, mode | S_IFREG, 0); 2843 } 2844 2845 /* 2846 * Link a file.. 2847 */ 2848 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) 2849 { 2850 struct inode *inode = d_inode(old_dentry); 2851 int ret; 2852 2853 /* 2854 * No ordinary (disk based) filesystem counts links as inodes; 2855 * but each new link needs a new dentry, pinning lowmem, and 2856 * tmpfs dentries cannot be pruned until they are unlinked. 2857 */ 2858 ret = shmem_reserve_inode(inode->i_sb); 2859 if (ret) 2860 goto out; 2861 2862 dir->i_size += BOGO_DIRENT_SIZE; 2863 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2864 inc_nlink(inode); 2865 ihold(inode); /* New dentry reference */ 2866 dget(dentry); /* Extra pinning count for the created dentry */ 2867 d_instantiate(dentry, inode); 2868 out: 2869 return ret; 2870 } 2871 2872 static int shmem_unlink(struct inode *dir, struct dentry *dentry) 2873 { 2874 struct inode *inode = d_inode(dentry); 2875 2876 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 2877 shmem_free_inode(inode->i_sb); 2878 2879 dir->i_size -= BOGO_DIRENT_SIZE; 2880 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2881 drop_nlink(inode); 2882 dput(dentry); /* Undo the count from "create" - this does all the work */ 2883 return 0; 2884 } 2885 2886 static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 2887 { 2888 if (!simple_empty(dentry)) 2889 return -ENOTEMPTY; 2890 2891 drop_nlink(d_inode(dentry)); 2892 drop_nlink(dir); 2893 return shmem_unlink(dir, dentry); 2894 } 2895 2896 static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) 2897 { 2898 bool old_is_dir = d_is_dir(old_dentry); 2899 bool new_is_dir = d_is_dir(new_dentry); 2900 2901 if (old_dir != new_dir && old_is_dir != new_is_dir) { 2902 if (old_is_dir) { 2903 drop_nlink(old_dir); 2904 inc_nlink(new_dir); 2905 } else { 2906 drop_nlink(new_dir); 2907 inc_nlink(old_dir); 2908 } 2909 } 2910 old_dir->i_ctime = old_dir->i_mtime = 2911 new_dir->i_ctime = new_dir->i_mtime = 2912 d_inode(old_dentry)->i_ctime = 2913 d_inode(new_dentry)->i_ctime = current_time(old_dir); 2914 2915 return 0; 2916 } 2917 2918 static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry) 2919 { 2920 struct dentry *whiteout; 2921 int error; 2922 2923 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name); 2924 if (!whiteout) 2925 return -ENOMEM; 2926 2927 error = shmem_mknod(old_dir, whiteout, 2928 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); 2929 dput(whiteout); 2930 if (error) 2931 return error; 2932 2933 /* 2934 * Cheat and hash the whiteout while the old dentry is still in 2935 * place, instead of playing games with FS_RENAME_DOES_D_MOVE. 2936 * 2937 * d_lookup() will consistently find one of them at this point, 2938 * not sure which one, but that isn't even important. 2939 */ 2940 d_rehash(whiteout); 2941 return 0; 2942 } 2943 2944 /* 2945 * The VFS layer already does all the dentry stuff for rename, 2946 * we just have to decrement the usage count for the target if 2947 * it exists so that the VFS layer correctly free's it when it 2948 * gets overwritten. 2949 */ 2950 static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags) 2951 { 2952 struct inode *inode = d_inode(old_dentry); 2953 int they_are_dirs = S_ISDIR(inode->i_mode); 2954 2955 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 2956 return -EINVAL; 2957 2958 if (flags & RENAME_EXCHANGE) 2959 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry); 2960 2961 if (!simple_empty(new_dentry)) 2962 return -ENOTEMPTY; 2963 2964 if (flags & RENAME_WHITEOUT) { 2965 int error; 2966 2967 error = shmem_whiteout(old_dir, old_dentry); 2968 if (error) 2969 return error; 2970 } 2971 2972 if (d_really_is_positive(new_dentry)) { 2973 (void) shmem_unlink(new_dir, new_dentry); 2974 if (they_are_dirs) { 2975 drop_nlink(d_inode(new_dentry)); 2976 drop_nlink(old_dir); 2977 } 2978 } else if (they_are_dirs) { 2979 drop_nlink(old_dir); 2980 inc_nlink(new_dir); 2981 } 2982 2983 old_dir->i_size -= BOGO_DIRENT_SIZE; 2984 new_dir->i_size += BOGO_DIRENT_SIZE; 2985 old_dir->i_ctime = old_dir->i_mtime = 2986 new_dir->i_ctime = new_dir->i_mtime = 2987 inode->i_ctime = current_time(old_dir); 2988 return 0; 2989 } 2990 2991 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname) 2992 { 2993 int error; 2994 int len; 2995 struct inode *inode; 2996 struct page *page; 2997 2998 len = strlen(symname) + 1; 2999 if (len > PAGE_SIZE) 3000 return -ENAMETOOLONG; 3001 3002 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0, 3003 VM_NORESERVE); 3004 if (!inode) 3005 return -ENOSPC; 3006 3007 error = security_inode_init_security(inode, dir, &dentry->d_name, 3008 shmem_initxattrs, NULL); 3009 if (error) { 3010 if (error != -EOPNOTSUPP) { 3011 iput(inode); 3012 return error; 3013 } 3014 error = 0; 3015 } 3016 3017 inode->i_size = len-1; 3018 if (len <= SHORT_SYMLINK_LEN) { 3019 inode->i_link = kmemdup(symname, len, GFP_KERNEL); 3020 if (!inode->i_link) { 3021 iput(inode); 3022 return -ENOMEM; 3023 } 3024 inode->i_op = &shmem_short_symlink_operations; 3025 } else { 3026 inode_nohighmem(inode); 3027 error = shmem_getpage(inode, 0, &page, SGP_WRITE); 3028 if (error) { 3029 iput(inode); 3030 return error; 3031 } 3032 inode->i_mapping->a_ops = &shmem_aops; 3033 inode->i_op = &shmem_symlink_inode_operations; 3034 memcpy(page_address(page), symname, len); 3035 SetPageUptodate(page); 3036 set_page_dirty(page); 3037 unlock_page(page); 3038 put_page(page); 3039 } 3040 dir->i_size += BOGO_DIRENT_SIZE; 3041 dir->i_ctime = dir->i_mtime = current_time(dir); 3042 d_instantiate(dentry, inode); 3043 dget(dentry); 3044 return 0; 3045 } 3046 3047 static void shmem_put_link(void *arg) 3048 { 3049 mark_page_accessed(arg); 3050 put_page(arg); 3051 } 3052 3053 static const char *shmem_get_link(struct dentry *dentry, 3054 struct inode *inode, 3055 struct delayed_call *done) 3056 { 3057 struct page *page = NULL; 3058 int error; 3059 if (!dentry) { 3060 page = find_get_page(inode->i_mapping, 0); 3061 if (!page) 3062 return ERR_PTR(-ECHILD); 3063 if (!PageUptodate(page)) { 3064 put_page(page); 3065 return ERR_PTR(-ECHILD); 3066 } 3067 } else { 3068 error = shmem_getpage(inode, 0, &page, SGP_READ); 3069 if (error) 3070 return ERR_PTR(error); 3071 unlock_page(page); 3072 } 3073 set_delayed_call(done, shmem_put_link, page); 3074 return page_address(page); 3075 } 3076 3077 #ifdef CONFIG_TMPFS_XATTR 3078 /* 3079 * Superblocks without xattr inode operations may get some security.* xattr 3080 * support from the LSM "for free". As soon as we have any other xattrs 3081 * like ACLs, we also need to implement the security.* handlers at 3082 * filesystem level, though. 3083 */ 3084 3085 /* 3086 * Callback for security_inode_init_security() for acquiring xattrs. 3087 */ 3088 static int shmem_initxattrs(struct inode *inode, 3089 const struct xattr *xattr_array, 3090 void *fs_info) 3091 { 3092 struct shmem_inode_info *info = SHMEM_I(inode); 3093 const struct xattr *xattr; 3094 struct simple_xattr *new_xattr; 3095 size_t len; 3096 3097 for (xattr = xattr_array; xattr->name != NULL; xattr++) { 3098 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len); 3099 if (!new_xattr) 3100 return -ENOMEM; 3101 3102 len = strlen(xattr->name) + 1; 3103 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, 3104 GFP_KERNEL); 3105 if (!new_xattr->name) { 3106 kfree(new_xattr); 3107 return -ENOMEM; 3108 } 3109 3110 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, 3111 XATTR_SECURITY_PREFIX_LEN); 3112 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, 3113 xattr->name, len); 3114 3115 simple_xattr_list_add(&info->xattrs, new_xattr); 3116 } 3117 3118 return 0; 3119 } 3120 3121 static int shmem_xattr_handler_get(const struct xattr_handler *handler, 3122 struct dentry *unused, struct inode *inode, 3123 const char *name, void *buffer, size_t size) 3124 { 3125 struct shmem_inode_info *info = SHMEM_I(inode); 3126 3127 name = xattr_full_name(handler, name); 3128 return simple_xattr_get(&info->xattrs, name, buffer, size); 3129 } 3130 3131 static int shmem_xattr_handler_set(const struct xattr_handler *handler, 3132 struct dentry *unused, struct inode *inode, 3133 const char *name, const void *value, 3134 size_t size, int flags) 3135 { 3136 struct shmem_inode_info *info = SHMEM_I(inode); 3137 3138 name = xattr_full_name(handler, name); 3139 return simple_xattr_set(&info->xattrs, name, value, size, flags); 3140 } 3141 3142 static const struct xattr_handler shmem_security_xattr_handler = { 3143 .prefix = XATTR_SECURITY_PREFIX, 3144 .get = shmem_xattr_handler_get, 3145 .set = shmem_xattr_handler_set, 3146 }; 3147 3148 static const struct xattr_handler shmem_trusted_xattr_handler = { 3149 .prefix = XATTR_TRUSTED_PREFIX, 3150 .get = shmem_xattr_handler_get, 3151 .set = shmem_xattr_handler_set, 3152 }; 3153 3154 static const struct xattr_handler *shmem_xattr_handlers[] = { 3155 #ifdef CONFIG_TMPFS_POSIX_ACL 3156 &posix_acl_access_xattr_handler, 3157 &posix_acl_default_xattr_handler, 3158 #endif 3159 &shmem_security_xattr_handler, 3160 &shmem_trusted_xattr_handler, 3161 NULL 3162 }; 3163 3164 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 3165 { 3166 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3167 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size); 3168 } 3169 #endif /* CONFIG_TMPFS_XATTR */ 3170 3171 static const struct inode_operations shmem_short_symlink_operations = { 3172 .get_link = simple_get_link, 3173 #ifdef CONFIG_TMPFS_XATTR 3174 .listxattr = shmem_listxattr, 3175 #endif 3176 }; 3177 3178 static const struct inode_operations shmem_symlink_inode_operations = { 3179 .get_link = shmem_get_link, 3180 #ifdef CONFIG_TMPFS_XATTR 3181 .listxattr = shmem_listxattr, 3182 #endif 3183 }; 3184 3185 static struct dentry *shmem_get_parent(struct dentry *child) 3186 { 3187 return ERR_PTR(-ESTALE); 3188 } 3189 3190 static int shmem_match(struct inode *ino, void *vfh) 3191 { 3192 __u32 *fh = vfh; 3193 __u64 inum = fh[2]; 3194 inum = (inum << 32) | fh[1]; 3195 return ino->i_ino == inum && fh[0] == ino->i_generation; 3196 } 3197 3198 /* Find any alias of inode, but prefer a hashed alias */ 3199 static struct dentry *shmem_find_alias(struct inode *inode) 3200 { 3201 struct dentry *alias = d_find_alias(inode); 3202 3203 return alias ?: d_find_any_alias(inode); 3204 } 3205 3206 3207 static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 3208 struct fid *fid, int fh_len, int fh_type) 3209 { 3210 struct inode *inode; 3211 struct dentry *dentry = NULL; 3212 u64 inum; 3213 3214 if (fh_len < 3) 3215 return NULL; 3216 3217 inum = fid->raw[2]; 3218 inum = (inum << 32) | fid->raw[1]; 3219 3220 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 3221 shmem_match, fid->raw); 3222 if (inode) { 3223 dentry = shmem_find_alias(inode); 3224 iput(inode); 3225 } 3226 3227 return dentry; 3228 } 3229 3230 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len, 3231 struct inode *parent) 3232 { 3233 if (*len < 3) { 3234 *len = 3; 3235 return FILEID_INVALID; 3236 } 3237 3238 if (inode_unhashed(inode)) { 3239 /* Unfortunately insert_inode_hash is not idempotent, 3240 * so as we hash inodes here rather than at creation 3241 * time, we need a lock to ensure we only try 3242 * to do it once 3243 */ 3244 static DEFINE_SPINLOCK(lock); 3245 spin_lock(&lock); 3246 if (inode_unhashed(inode)) 3247 __insert_inode_hash(inode, 3248 inode->i_ino + inode->i_generation); 3249 spin_unlock(&lock); 3250 } 3251 3252 fh[0] = inode->i_generation; 3253 fh[1] = inode->i_ino; 3254 fh[2] = ((__u64)inode->i_ino) >> 32; 3255 3256 *len = 3; 3257 return 1; 3258 } 3259 3260 static const struct export_operations shmem_export_ops = { 3261 .get_parent = shmem_get_parent, 3262 .encode_fh = shmem_encode_fh, 3263 .fh_to_dentry = shmem_fh_to_dentry, 3264 }; 3265 3266 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo, 3267 bool remount) 3268 { 3269 char *this_char, *value, *rest; 3270 struct mempolicy *mpol = NULL; 3271 uid_t uid; 3272 gid_t gid; 3273 3274 while (options != NULL) { 3275 this_char = options; 3276 for (;;) { 3277 /* 3278 * NUL-terminate this option: unfortunately, 3279 * mount options form a comma-separated list, 3280 * but mpol's nodelist may also contain commas. 3281 */ 3282 options = strchr(options, ','); 3283 if (options == NULL) 3284 break; 3285 options++; 3286 if (!isdigit(*options)) { 3287 options[-1] = '\0'; 3288 break; 3289 } 3290 } 3291 if (!*this_char) 3292 continue; 3293 if ((value = strchr(this_char,'=')) != NULL) { 3294 *value++ = 0; 3295 } else { 3296 pr_err("tmpfs: No value for mount option '%s'\n", 3297 this_char); 3298 goto error; 3299 } 3300 3301 if (!strcmp(this_char,"size")) { 3302 unsigned long long size; 3303 size = memparse(value,&rest); 3304 if (*rest == '%') { 3305 size <<= PAGE_SHIFT; 3306 size *= totalram_pages(); 3307 do_div(size, 100); 3308 rest++; 3309 } 3310 if (*rest) 3311 goto bad_val; 3312 sbinfo->max_blocks = 3313 DIV_ROUND_UP(size, PAGE_SIZE); 3314 } else if (!strcmp(this_char,"nr_blocks")) { 3315 sbinfo->max_blocks = memparse(value, &rest); 3316 if (*rest) 3317 goto bad_val; 3318 } else if (!strcmp(this_char,"nr_inodes")) { 3319 sbinfo->max_inodes = memparse(value, &rest); 3320 if (*rest) 3321 goto bad_val; 3322 } else if (!strcmp(this_char,"mode")) { 3323 if (remount) 3324 continue; 3325 sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777; 3326 if (*rest) 3327 goto bad_val; 3328 } else if (!strcmp(this_char,"uid")) { 3329 if (remount) 3330 continue; 3331 uid = simple_strtoul(value, &rest, 0); 3332 if (*rest) 3333 goto bad_val; 3334 sbinfo->uid = make_kuid(current_user_ns(), uid); 3335 if (!uid_valid(sbinfo->uid)) 3336 goto bad_val; 3337 } else if (!strcmp(this_char,"gid")) { 3338 if (remount) 3339 continue; 3340 gid = simple_strtoul(value, &rest, 0); 3341 if (*rest) 3342 goto bad_val; 3343 sbinfo->gid = make_kgid(current_user_ns(), gid); 3344 if (!gid_valid(sbinfo->gid)) 3345 goto bad_val; 3346 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3347 } else if (!strcmp(this_char, "huge")) { 3348 int huge; 3349 huge = shmem_parse_huge(value); 3350 if (huge < 0) 3351 goto bad_val; 3352 if (!has_transparent_hugepage() && 3353 huge != SHMEM_HUGE_NEVER) 3354 goto bad_val; 3355 sbinfo->huge = huge; 3356 #endif 3357 #ifdef CONFIG_NUMA 3358 } else if (!strcmp(this_char,"mpol")) { 3359 mpol_put(mpol); 3360 mpol = NULL; 3361 if (mpol_parse_str(value, &mpol)) 3362 goto bad_val; 3363 #endif 3364 } else { 3365 pr_err("tmpfs: Bad mount option %s\n", this_char); 3366 goto error; 3367 } 3368 } 3369 sbinfo->mpol = mpol; 3370 return 0; 3371 3372 bad_val: 3373 pr_err("tmpfs: Bad value '%s' for mount option '%s'\n", 3374 value, this_char); 3375 error: 3376 mpol_put(mpol); 3377 return 1; 3378 3379 } 3380 3381 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data) 3382 { 3383 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3384 struct shmem_sb_info config = *sbinfo; 3385 unsigned long inodes; 3386 int error = -EINVAL; 3387 3388 config.mpol = NULL; 3389 if (shmem_parse_options(data, &config, true)) 3390 return error; 3391 3392 spin_lock(&sbinfo->stat_lock); 3393 inodes = sbinfo->max_inodes - sbinfo->free_inodes; 3394 if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0) 3395 goto out; 3396 if (config.max_inodes < inodes) 3397 goto out; 3398 /* 3399 * Those tests disallow limited->unlimited while any are in use; 3400 * but we must separately disallow unlimited->limited, because 3401 * in that case we have no record of how much is already in use. 3402 */ 3403 if (config.max_blocks && !sbinfo->max_blocks) 3404 goto out; 3405 if (config.max_inodes && !sbinfo->max_inodes) 3406 goto out; 3407 3408 error = 0; 3409 sbinfo->huge = config.huge; 3410 sbinfo->max_blocks = config.max_blocks; 3411 sbinfo->max_inodes = config.max_inodes; 3412 sbinfo->free_inodes = config.max_inodes - inodes; 3413 3414 /* 3415 * Preserve previous mempolicy unless mpol remount option was specified. 3416 */ 3417 if (config.mpol) { 3418 mpol_put(sbinfo->mpol); 3419 sbinfo->mpol = config.mpol; /* transfers initial ref */ 3420 } 3421 out: 3422 spin_unlock(&sbinfo->stat_lock); 3423 return error; 3424 } 3425 3426 static int shmem_show_options(struct seq_file *seq, struct dentry *root) 3427 { 3428 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb); 3429 3430 if (sbinfo->max_blocks != shmem_default_max_blocks()) 3431 seq_printf(seq, ",size=%luk", 3432 sbinfo->max_blocks << (PAGE_SHIFT - 10)); 3433 if (sbinfo->max_inodes != shmem_default_max_inodes()) 3434 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 3435 if (sbinfo->mode != (0777 | S_ISVTX)) 3436 seq_printf(seq, ",mode=%03ho", sbinfo->mode); 3437 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID)) 3438 seq_printf(seq, ",uid=%u", 3439 from_kuid_munged(&init_user_ns, sbinfo->uid)); 3440 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) 3441 seq_printf(seq, ",gid=%u", 3442 from_kgid_munged(&init_user_ns, sbinfo->gid)); 3443 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3444 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ 3445 if (sbinfo->huge) 3446 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); 3447 #endif 3448 shmem_show_mpol(seq, sbinfo->mpol); 3449 return 0; 3450 } 3451 3452 #endif /* CONFIG_TMPFS */ 3453 3454 static void shmem_put_super(struct super_block *sb) 3455 { 3456 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3457 3458 percpu_counter_destroy(&sbinfo->used_blocks); 3459 mpol_put(sbinfo->mpol); 3460 kfree(sbinfo); 3461 sb->s_fs_info = NULL; 3462 } 3463 3464 int shmem_fill_super(struct super_block *sb, void *data, int silent) 3465 { 3466 struct inode *inode; 3467 struct shmem_sb_info *sbinfo; 3468 int err = -ENOMEM; 3469 3470 /* Round up to L1_CACHE_BYTES to resist false sharing */ 3471 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 3472 L1_CACHE_BYTES), GFP_KERNEL); 3473 if (!sbinfo) 3474 return -ENOMEM; 3475 3476 sbinfo->mode = 0777 | S_ISVTX; 3477 sbinfo->uid = current_fsuid(); 3478 sbinfo->gid = current_fsgid(); 3479 sb->s_fs_info = sbinfo; 3480 3481 #ifdef CONFIG_TMPFS 3482 /* 3483 * Per default we only allow half of the physical ram per 3484 * tmpfs instance, limiting inodes to one per page of lowmem; 3485 * but the internal instance is left unlimited. 3486 */ 3487 if (!(sb->s_flags & SB_KERNMOUNT)) { 3488 sbinfo->max_blocks = shmem_default_max_blocks(); 3489 sbinfo->max_inodes = shmem_default_max_inodes(); 3490 if (shmem_parse_options(data, sbinfo, false)) { 3491 err = -EINVAL; 3492 goto failed; 3493 } 3494 } else { 3495 sb->s_flags |= SB_NOUSER; 3496 } 3497 sb->s_export_op = &shmem_export_ops; 3498 sb->s_flags |= SB_NOSEC; 3499 #else 3500 sb->s_flags |= SB_NOUSER; 3501 #endif 3502 3503 spin_lock_init(&sbinfo->stat_lock); 3504 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) 3505 goto failed; 3506 sbinfo->free_inodes = sbinfo->max_inodes; 3507 spin_lock_init(&sbinfo->shrinklist_lock); 3508 INIT_LIST_HEAD(&sbinfo->shrinklist); 3509 3510 sb->s_maxbytes = MAX_LFS_FILESIZE; 3511 sb->s_blocksize = PAGE_SIZE; 3512 sb->s_blocksize_bits = PAGE_SHIFT; 3513 sb->s_magic = TMPFS_MAGIC; 3514 sb->s_op = &shmem_ops; 3515 sb->s_time_gran = 1; 3516 #ifdef CONFIG_TMPFS_XATTR 3517 sb->s_xattr = shmem_xattr_handlers; 3518 #endif 3519 #ifdef CONFIG_TMPFS_POSIX_ACL 3520 sb->s_flags |= SB_POSIXACL; 3521 #endif 3522 uuid_gen(&sb->s_uuid); 3523 3524 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 3525 if (!inode) 3526 goto failed; 3527 inode->i_uid = sbinfo->uid; 3528 inode->i_gid = sbinfo->gid; 3529 sb->s_root = d_make_root(inode); 3530 if (!sb->s_root) 3531 goto failed; 3532 return 0; 3533 3534 failed: 3535 shmem_put_super(sb); 3536 return err; 3537 } 3538 3539 static struct kmem_cache *shmem_inode_cachep; 3540 3541 static struct inode *shmem_alloc_inode(struct super_block *sb) 3542 { 3543 struct shmem_inode_info *info; 3544 info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); 3545 if (!info) 3546 return NULL; 3547 return &info->vfs_inode; 3548 } 3549 3550 static void shmem_destroy_callback(struct rcu_head *head) 3551 { 3552 struct inode *inode = container_of(head, struct inode, i_rcu); 3553 if (S_ISLNK(inode->i_mode)) 3554 kfree(inode->i_link); 3555 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 3556 } 3557 3558 static void shmem_destroy_inode(struct inode *inode) 3559 { 3560 if (S_ISREG(inode->i_mode)) 3561 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 3562 call_rcu(&inode->i_rcu, shmem_destroy_callback); 3563 } 3564 3565 static void shmem_init_inode(void *foo) 3566 { 3567 struct shmem_inode_info *info = foo; 3568 inode_init_once(&info->vfs_inode); 3569 } 3570 3571 static void shmem_init_inodecache(void) 3572 { 3573 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 3574 sizeof(struct shmem_inode_info), 3575 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode); 3576 } 3577 3578 static void shmem_destroy_inodecache(void) 3579 { 3580 kmem_cache_destroy(shmem_inode_cachep); 3581 } 3582 3583 static const struct address_space_operations shmem_aops = { 3584 .writepage = shmem_writepage, 3585 .set_page_dirty = __set_page_dirty_no_writeback, 3586 #ifdef CONFIG_TMPFS 3587 .write_begin = shmem_write_begin, 3588 .write_end = shmem_write_end, 3589 #endif 3590 #ifdef CONFIG_MIGRATION 3591 .migratepage = migrate_page, 3592 #endif 3593 .error_remove_page = generic_error_remove_page, 3594 }; 3595 3596 static const struct file_operations shmem_file_operations = { 3597 .mmap = shmem_mmap, 3598 .get_unmapped_area = shmem_get_unmapped_area, 3599 #ifdef CONFIG_TMPFS 3600 .llseek = shmem_file_llseek, 3601 .read_iter = shmem_file_read_iter, 3602 .write_iter = generic_file_write_iter, 3603 .fsync = noop_fsync, 3604 .splice_read = generic_file_splice_read, 3605 .splice_write = iter_file_splice_write, 3606 .fallocate = shmem_fallocate, 3607 #endif 3608 }; 3609 3610 static const struct inode_operations shmem_inode_operations = { 3611 .getattr = shmem_getattr, 3612 .setattr = shmem_setattr, 3613 #ifdef CONFIG_TMPFS_XATTR 3614 .listxattr = shmem_listxattr, 3615 .set_acl = simple_set_acl, 3616 #endif 3617 }; 3618 3619 static const struct inode_operations shmem_dir_inode_operations = { 3620 #ifdef CONFIG_TMPFS 3621 .create = shmem_create, 3622 .lookup = simple_lookup, 3623 .link = shmem_link, 3624 .unlink = shmem_unlink, 3625 .symlink = shmem_symlink, 3626 .mkdir = shmem_mkdir, 3627 .rmdir = shmem_rmdir, 3628 .mknod = shmem_mknod, 3629 .rename = shmem_rename2, 3630 .tmpfile = shmem_tmpfile, 3631 #endif 3632 #ifdef CONFIG_TMPFS_XATTR 3633 .listxattr = shmem_listxattr, 3634 #endif 3635 #ifdef CONFIG_TMPFS_POSIX_ACL 3636 .setattr = shmem_setattr, 3637 .set_acl = simple_set_acl, 3638 #endif 3639 }; 3640 3641 static const struct inode_operations shmem_special_inode_operations = { 3642 #ifdef CONFIG_TMPFS_XATTR 3643 .listxattr = shmem_listxattr, 3644 #endif 3645 #ifdef CONFIG_TMPFS_POSIX_ACL 3646 .setattr = shmem_setattr, 3647 .set_acl = simple_set_acl, 3648 #endif 3649 }; 3650 3651 static const struct super_operations shmem_ops = { 3652 .alloc_inode = shmem_alloc_inode, 3653 .destroy_inode = shmem_destroy_inode, 3654 #ifdef CONFIG_TMPFS 3655 .statfs = shmem_statfs, 3656 .remount_fs = shmem_remount_fs, 3657 .show_options = shmem_show_options, 3658 #endif 3659 .evict_inode = shmem_evict_inode, 3660 .drop_inode = generic_delete_inode, 3661 .put_super = shmem_put_super, 3662 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3663 .nr_cached_objects = shmem_unused_huge_count, 3664 .free_cached_objects = shmem_unused_huge_scan, 3665 #endif 3666 }; 3667 3668 static const struct vm_operations_struct shmem_vm_ops = { 3669 .fault = shmem_fault, 3670 .map_pages = filemap_map_pages, 3671 #ifdef CONFIG_NUMA 3672 .set_policy = shmem_set_policy, 3673 .get_policy = shmem_get_policy, 3674 #endif 3675 }; 3676 3677 static struct dentry *shmem_mount(struct file_system_type *fs_type, 3678 int flags, const char *dev_name, void *data) 3679 { 3680 return mount_nodev(fs_type, flags, data, shmem_fill_super); 3681 } 3682 3683 static struct file_system_type shmem_fs_type = { 3684 .owner = THIS_MODULE, 3685 .name = "tmpfs", 3686 .mount = shmem_mount, 3687 .kill_sb = kill_litter_super, 3688 .fs_flags = FS_USERNS_MOUNT, 3689 }; 3690 3691 int __init shmem_init(void) 3692 { 3693 int error; 3694 3695 /* If rootfs called this, don't re-init */ 3696 if (shmem_inode_cachep) 3697 return 0; 3698 3699 shmem_init_inodecache(); 3700 3701 error = register_filesystem(&shmem_fs_type); 3702 if (error) { 3703 pr_err("Could not register tmpfs\n"); 3704 goto out2; 3705 } 3706 3707 shm_mnt = kern_mount(&shmem_fs_type); 3708 if (IS_ERR(shm_mnt)) { 3709 error = PTR_ERR(shm_mnt); 3710 pr_err("Could not kern_mount tmpfs\n"); 3711 goto out1; 3712 } 3713 3714 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3715 if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY) 3716 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3717 else 3718 shmem_huge = 0; /* just in case it was patched */ 3719 #endif 3720 return 0; 3721 3722 out1: 3723 unregister_filesystem(&shmem_fs_type); 3724 out2: 3725 shmem_destroy_inodecache(); 3726 shm_mnt = ERR_PTR(error); 3727 return error; 3728 } 3729 3730 #if defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && defined(CONFIG_SYSFS) 3731 static ssize_t shmem_enabled_show(struct kobject *kobj, 3732 struct kobj_attribute *attr, char *buf) 3733 { 3734 int values[] = { 3735 SHMEM_HUGE_ALWAYS, 3736 SHMEM_HUGE_WITHIN_SIZE, 3737 SHMEM_HUGE_ADVISE, 3738 SHMEM_HUGE_NEVER, 3739 SHMEM_HUGE_DENY, 3740 SHMEM_HUGE_FORCE, 3741 }; 3742 int i, count; 3743 3744 for (i = 0, count = 0; i < ARRAY_SIZE(values); i++) { 3745 const char *fmt = shmem_huge == values[i] ? "[%s] " : "%s "; 3746 3747 count += sprintf(buf + count, fmt, 3748 shmem_format_huge(values[i])); 3749 } 3750 buf[count - 1] = '\n'; 3751 return count; 3752 } 3753 3754 static ssize_t shmem_enabled_store(struct kobject *kobj, 3755 struct kobj_attribute *attr, const char *buf, size_t count) 3756 { 3757 char tmp[16]; 3758 int huge; 3759 3760 if (count + 1 > sizeof(tmp)) 3761 return -EINVAL; 3762 memcpy(tmp, buf, count); 3763 tmp[count] = '\0'; 3764 if (count && tmp[count - 1] == '\n') 3765 tmp[count - 1] = '\0'; 3766 3767 huge = shmem_parse_huge(tmp); 3768 if (huge == -EINVAL) 3769 return -EINVAL; 3770 if (!has_transparent_hugepage() && 3771 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY) 3772 return -EINVAL; 3773 3774 shmem_huge = huge; 3775 if (shmem_huge > SHMEM_HUGE_DENY) 3776 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3777 return count; 3778 } 3779 3780 struct kobj_attribute shmem_enabled_attr = 3781 __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store); 3782 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE && CONFIG_SYSFS */ 3783 3784 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE 3785 bool shmem_huge_enabled(struct vm_area_struct *vma) 3786 { 3787 struct inode *inode = file_inode(vma->vm_file); 3788 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 3789 loff_t i_size; 3790 pgoff_t off; 3791 3792 if (shmem_huge == SHMEM_HUGE_FORCE) 3793 return true; 3794 if (shmem_huge == SHMEM_HUGE_DENY) 3795 return false; 3796 switch (sbinfo->huge) { 3797 case SHMEM_HUGE_NEVER: 3798 return false; 3799 case SHMEM_HUGE_ALWAYS: 3800 return true; 3801 case SHMEM_HUGE_WITHIN_SIZE: 3802 off = round_up(vma->vm_pgoff, HPAGE_PMD_NR); 3803 i_size = round_up(i_size_read(inode), PAGE_SIZE); 3804 if (i_size >= HPAGE_PMD_SIZE && 3805 i_size >> PAGE_SHIFT >= off) 3806 return true; 3807 /* fall through */ 3808 case SHMEM_HUGE_ADVISE: 3809 /* TODO: implement fadvise() hints */ 3810 return (vma->vm_flags & VM_HUGEPAGE); 3811 default: 3812 VM_BUG_ON(1); 3813 return false; 3814 } 3815 } 3816 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */ 3817 3818 #else /* !CONFIG_SHMEM */ 3819 3820 /* 3821 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 3822 * 3823 * This is intended for small system where the benefits of the full 3824 * shmem code (swap-backed and resource-limited) are outweighed by 3825 * their complexity. On systems without swap this code should be 3826 * effectively equivalent, but much lighter weight. 3827 */ 3828 3829 static struct file_system_type shmem_fs_type = { 3830 .name = "tmpfs", 3831 .mount = ramfs_mount, 3832 .kill_sb = kill_litter_super, 3833 .fs_flags = FS_USERNS_MOUNT, 3834 }; 3835 3836 int __init shmem_init(void) 3837 { 3838 BUG_ON(register_filesystem(&shmem_fs_type) != 0); 3839 3840 shm_mnt = kern_mount(&shmem_fs_type); 3841 BUG_ON(IS_ERR(shm_mnt)); 3842 3843 return 0; 3844 } 3845 3846 int shmem_unuse(swp_entry_t swap, struct page *page) 3847 { 3848 return 0; 3849 } 3850 3851 int shmem_lock(struct file *file, int lock, struct user_struct *user) 3852 { 3853 return 0; 3854 } 3855 3856 void shmem_unlock_mapping(struct address_space *mapping) 3857 { 3858 } 3859 3860 #ifdef CONFIG_MMU 3861 unsigned long shmem_get_unmapped_area(struct file *file, 3862 unsigned long addr, unsigned long len, 3863 unsigned long pgoff, unsigned long flags) 3864 { 3865 return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); 3866 } 3867 #endif 3868 3869 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 3870 { 3871 truncate_inode_pages_range(inode->i_mapping, lstart, lend); 3872 } 3873 EXPORT_SYMBOL_GPL(shmem_truncate_range); 3874 3875 #define shmem_vm_ops generic_file_vm_ops 3876 #define shmem_file_operations ramfs_file_operations 3877 #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) 3878 #define shmem_acct_size(flags, size) 0 3879 #define shmem_unacct_size(flags, size) do {} while (0) 3880 3881 #endif /* CONFIG_SHMEM */ 3882 3883 /* common code */ 3884 3885 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size, 3886 unsigned long flags, unsigned int i_flags) 3887 { 3888 struct inode *inode; 3889 struct file *res; 3890 3891 if (IS_ERR(mnt)) 3892 return ERR_CAST(mnt); 3893 3894 if (size < 0 || size > MAX_LFS_FILESIZE) 3895 return ERR_PTR(-EINVAL); 3896 3897 if (shmem_acct_size(flags, size)) 3898 return ERR_PTR(-ENOMEM); 3899 3900 inode = shmem_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0, 3901 flags); 3902 if (unlikely(!inode)) { 3903 shmem_unacct_size(flags, size); 3904 return ERR_PTR(-ENOSPC); 3905 } 3906 inode->i_flags |= i_flags; 3907 inode->i_size = size; 3908 clear_nlink(inode); /* It is unlinked */ 3909 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); 3910 if (!IS_ERR(res)) 3911 res = alloc_file_pseudo(inode, mnt, name, O_RDWR, 3912 &shmem_file_operations); 3913 if (IS_ERR(res)) 3914 iput(inode); 3915 return res; 3916 } 3917 3918 /** 3919 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be 3920 * kernel internal. There will be NO LSM permission checks against the 3921 * underlying inode. So users of this interface must do LSM checks at a 3922 * higher layer. The users are the big_key and shm implementations. LSM 3923 * checks are provided at the key or shm level rather than the inode. 3924 * @name: name for dentry (to be seen in /proc/<pid>/maps 3925 * @size: size to be set for the file 3926 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 3927 */ 3928 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags) 3929 { 3930 return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE); 3931 } 3932 3933 /** 3934 * shmem_file_setup - get an unlinked file living in tmpfs 3935 * @name: name for dentry (to be seen in /proc/<pid>/maps 3936 * @size: size to be set for the file 3937 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 3938 */ 3939 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 3940 { 3941 return __shmem_file_setup(shm_mnt, name, size, flags, 0); 3942 } 3943 EXPORT_SYMBOL_GPL(shmem_file_setup); 3944 3945 /** 3946 * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs 3947 * @mnt: the tmpfs mount where the file will be created 3948 * @name: name for dentry (to be seen in /proc/<pid>/maps 3949 * @size: size to be set for the file 3950 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 3951 */ 3952 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name, 3953 loff_t size, unsigned long flags) 3954 { 3955 return __shmem_file_setup(mnt, name, size, flags, 0); 3956 } 3957 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt); 3958 3959 /** 3960 * shmem_zero_setup - setup a shared anonymous mapping 3961 * @vma: the vma to be mmapped is prepared by do_mmap_pgoff 3962 */ 3963 int shmem_zero_setup(struct vm_area_struct *vma) 3964 { 3965 struct file *file; 3966 loff_t size = vma->vm_end - vma->vm_start; 3967 3968 /* 3969 * Cloning a new file under mmap_sem leads to a lock ordering conflict 3970 * between XFS directory reading and selinux: since this file is only 3971 * accessible to the user through its mapping, use S_PRIVATE flag to 3972 * bypass file security, in the same way as shmem_kernel_file_setup(). 3973 */ 3974 file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags); 3975 if (IS_ERR(file)) 3976 return PTR_ERR(file); 3977 3978 if (vma->vm_file) 3979 fput(vma->vm_file); 3980 vma->vm_file = file; 3981 vma->vm_ops = &shmem_vm_ops; 3982 3983 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && 3984 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 3985 (vma->vm_end & HPAGE_PMD_MASK)) { 3986 khugepaged_enter(vma, vma->vm_flags); 3987 } 3988 3989 return 0; 3990 } 3991 3992 /** 3993 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags. 3994 * @mapping: the page's address_space 3995 * @index: the page index 3996 * @gfp: the page allocator flags to use if allocating 3997 * 3998 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", 3999 * with any new page allocations done using the specified allocation flags. 4000 * But read_cache_page_gfp() uses the ->readpage() method: which does not 4001 * suit tmpfs, since it may have pages in swapcache, and needs to find those 4002 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. 4003 * 4004 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in 4005 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. 4006 */ 4007 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, 4008 pgoff_t index, gfp_t gfp) 4009 { 4010 #ifdef CONFIG_SHMEM 4011 struct inode *inode = mapping->host; 4012 struct page *page; 4013 int error; 4014 4015 BUG_ON(mapping->a_ops != &shmem_aops); 4016 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, 4017 gfp, NULL, NULL, NULL); 4018 if (error) 4019 page = ERR_PTR(error); 4020 else 4021 unlock_page(page); 4022 return page; 4023 #else 4024 /* 4025 * The tiny !SHMEM case uses ramfs without swap 4026 */ 4027 return read_cache_page_gfp(mapping, index, gfp); 4028 #endif 4029 } 4030 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); 4031