1 /* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2011 Hugh Dickins. 10 * Copyright (C) 2011 Google Inc. 11 * Copyright (C) 2002-2005 VERITAS Software Corporation. 12 * Copyright (C) 2004 Andi Kleen, SuSE Labs 13 * 14 * Extended attribute support for tmpfs: 15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 17 * 18 * tiny-shmem: 19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 20 * 21 * This file is released under the GPL. 22 */ 23 24 #include <linux/fs.h> 25 #include <linux/init.h> 26 #include <linux/vfs.h> 27 #include <linux/mount.h> 28 #include <linux/ramfs.h> 29 #include <linux/pagemap.h> 30 #include <linux/file.h> 31 #include <linux/fileattr.h> 32 #include <linux/mm.h> 33 #include <linux/random.h> 34 #include <linux/sched/signal.h> 35 #include <linux/export.h> 36 #include <linux/shmem_fs.h> 37 #include <linux/swap.h> 38 #include <linux/uio.h> 39 #include <linux/hugetlb.h> 40 #include <linux/fs_parser.h> 41 #include <linux/swapfile.h> 42 #include <linux/iversion.h> 43 #include "swap.h" 44 45 static struct vfsmount *shm_mnt __ro_after_init; 46 47 #ifdef CONFIG_SHMEM 48 /* 49 * This virtual memory filesystem is heavily based on the ramfs. It 50 * extends ramfs by the ability to use swap and honor resource limits 51 * which makes it a completely usable filesystem. 52 */ 53 54 #include <linux/xattr.h> 55 #include <linux/exportfs.h> 56 #include <linux/posix_acl.h> 57 #include <linux/posix_acl_xattr.h> 58 #include <linux/mman.h> 59 #include <linux/string.h> 60 #include <linux/slab.h> 61 #include <linux/backing-dev.h> 62 #include <linux/writeback.h> 63 #include <linux/pagevec.h> 64 #include <linux/percpu_counter.h> 65 #include <linux/falloc.h> 66 #include <linux/splice.h> 67 #include <linux/security.h> 68 #include <linux/swapops.h> 69 #include <linux/mempolicy.h> 70 #include <linux/namei.h> 71 #include <linux/ctype.h> 72 #include <linux/migrate.h> 73 #include <linux/highmem.h> 74 #include <linux/seq_file.h> 75 #include <linux/magic.h> 76 #include <linux/syscalls.h> 77 #include <linux/fcntl.h> 78 #include <uapi/linux/memfd.h> 79 #include <linux/rmap.h> 80 #include <linux/uuid.h> 81 #include <linux/quotaops.h> 82 #include <linux/rcupdate_wait.h> 83 84 #include <linux/uaccess.h> 85 86 #include "internal.h" 87 88 #define BLOCKS_PER_PAGE (PAGE_SIZE/512) 89 #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT) 90 91 /* Pretend that each entry is of this size in directory's i_size */ 92 #define BOGO_DIRENT_SIZE 20 93 94 /* Pretend that one inode + its dentry occupy this much memory */ 95 #define BOGO_INODE_SIZE 1024 96 97 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */ 98 #define SHORT_SYMLINK_LEN 128 99 100 /* 101 * shmem_fallocate communicates with shmem_fault or shmem_writepage via 102 * inode->i_private (with i_rwsem making sure that it has only one user at 103 * a time): we would prefer not to enlarge the shmem inode just for that. 104 */ 105 struct shmem_falloc { 106 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */ 107 pgoff_t start; /* start of range currently being fallocated */ 108 pgoff_t next; /* the next page offset to be fallocated */ 109 pgoff_t nr_falloced; /* how many new pages have been fallocated */ 110 pgoff_t nr_unswapped; /* how often writepage refused to swap out */ 111 }; 112 113 struct shmem_options { 114 unsigned long long blocks; 115 unsigned long long inodes; 116 struct mempolicy *mpol; 117 kuid_t uid; 118 kgid_t gid; 119 umode_t mode; 120 bool full_inums; 121 int huge; 122 int seen; 123 bool noswap; 124 unsigned short quota_types; 125 struct shmem_quota_limits qlimits; 126 #define SHMEM_SEEN_BLOCKS 1 127 #define SHMEM_SEEN_INODES 2 128 #define SHMEM_SEEN_HUGE 4 129 #define SHMEM_SEEN_INUMS 8 130 #define SHMEM_SEEN_NOSWAP 16 131 #define SHMEM_SEEN_QUOTA 32 132 }; 133 134 #ifdef CONFIG_TMPFS 135 static unsigned long shmem_default_max_blocks(void) 136 { 137 return totalram_pages() / 2; 138 } 139 140 static unsigned long shmem_default_max_inodes(void) 141 { 142 unsigned long nr_pages = totalram_pages(); 143 144 return min3(nr_pages - totalhigh_pages(), nr_pages / 2, 145 ULONG_MAX / BOGO_INODE_SIZE); 146 } 147 #endif 148 149 static int shmem_swapin_folio(struct inode *inode, pgoff_t index, 150 struct folio **foliop, enum sgp_type sgp, gfp_t gfp, 151 struct mm_struct *fault_mm, vm_fault_t *fault_type); 152 153 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 154 { 155 return sb->s_fs_info; 156 } 157 158 /* 159 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 160 * for shared memory and for shared anonymous (/dev/zero) mappings 161 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 162 * consistent with the pre-accounting of private mappings ... 163 */ 164 static inline int shmem_acct_size(unsigned long flags, loff_t size) 165 { 166 return (flags & VM_NORESERVE) ? 167 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); 168 } 169 170 static inline void shmem_unacct_size(unsigned long flags, loff_t size) 171 { 172 if (!(flags & VM_NORESERVE)) 173 vm_unacct_memory(VM_ACCT(size)); 174 } 175 176 static inline int shmem_reacct_size(unsigned long flags, 177 loff_t oldsize, loff_t newsize) 178 { 179 if (!(flags & VM_NORESERVE)) { 180 if (VM_ACCT(newsize) > VM_ACCT(oldsize)) 181 return security_vm_enough_memory_mm(current->mm, 182 VM_ACCT(newsize) - VM_ACCT(oldsize)); 183 else if (VM_ACCT(newsize) < VM_ACCT(oldsize)) 184 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize)); 185 } 186 return 0; 187 } 188 189 /* 190 * ... whereas tmpfs objects are accounted incrementally as 191 * pages are allocated, in order to allow large sparse files. 192 * shmem_get_folio reports shmem_acct_blocks failure as -ENOSPC not -ENOMEM, 193 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 194 */ 195 static inline int shmem_acct_blocks(unsigned long flags, long pages) 196 { 197 if (!(flags & VM_NORESERVE)) 198 return 0; 199 200 return security_vm_enough_memory_mm(current->mm, 201 pages * VM_ACCT(PAGE_SIZE)); 202 } 203 204 static inline void shmem_unacct_blocks(unsigned long flags, long pages) 205 { 206 if (flags & VM_NORESERVE) 207 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE)); 208 } 209 210 static int shmem_inode_acct_blocks(struct inode *inode, long pages) 211 { 212 struct shmem_inode_info *info = SHMEM_I(inode); 213 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 214 int err = -ENOSPC; 215 216 if (shmem_acct_blocks(info->flags, pages)) 217 return err; 218 219 might_sleep(); /* when quotas */ 220 if (sbinfo->max_blocks) { 221 if (!percpu_counter_limited_add(&sbinfo->used_blocks, 222 sbinfo->max_blocks, pages)) 223 goto unacct; 224 225 err = dquot_alloc_block_nodirty(inode, pages); 226 if (err) { 227 percpu_counter_sub(&sbinfo->used_blocks, pages); 228 goto unacct; 229 } 230 } else { 231 err = dquot_alloc_block_nodirty(inode, pages); 232 if (err) 233 goto unacct; 234 } 235 236 return 0; 237 238 unacct: 239 shmem_unacct_blocks(info->flags, pages); 240 return err; 241 } 242 243 static void shmem_inode_unacct_blocks(struct inode *inode, long pages) 244 { 245 struct shmem_inode_info *info = SHMEM_I(inode); 246 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 247 248 might_sleep(); /* when quotas */ 249 dquot_free_block_nodirty(inode, pages); 250 251 if (sbinfo->max_blocks) 252 percpu_counter_sub(&sbinfo->used_blocks, pages); 253 shmem_unacct_blocks(info->flags, pages); 254 } 255 256 static const struct super_operations shmem_ops; 257 static const struct address_space_operations shmem_aops; 258 static const struct file_operations shmem_file_operations; 259 static const struct inode_operations shmem_inode_operations; 260 static const struct inode_operations shmem_dir_inode_operations; 261 static const struct inode_operations shmem_special_inode_operations; 262 static const struct vm_operations_struct shmem_vm_ops; 263 static const struct vm_operations_struct shmem_anon_vm_ops; 264 static struct file_system_type shmem_fs_type; 265 266 bool shmem_mapping(struct address_space *mapping) 267 { 268 return mapping->a_ops == &shmem_aops; 269 } 270 EXPORT_SYMBOL_GPL(shmem_mapping); 271 272 bool vma_is_anon_shmem(struct vm_area_struct *vma) 273 { 274 return vma->vm_ops == &shmem_anon_vm_ops; 275 } 276 277 bool vma_is_shmem(struct vm_area_struct *vma) 278 { 279 return vma_is_anon_shmem(vma) || vma->vm_ops == &shmem_vm_ops; 280 } 281 282 static LIST_HEAD(shmem_swaplist); 283 static DEFINE_MUTEX(shmem_swaplist_mutex); 284 285 #ifdef CONFIG_TMPFS_QUOTA 286 287 static int shmem_enable_quotas(struct super_block *sb, 288 unsigned short quota_types) 289 { 290 int type, err = 0; 291 292 sb_dqopt(sb)->flags |= DQUOT_QUOTA_SYS_FILE | DQUOT_NOLIST_DIRTY; 293 for (type = 0; type < SHMEM_MAXQUOTAS; type++) { 294 if (!(quota_types & (1 << type))) 295 continue; 296 err = dquot_load_quota_sb(sb, type, QFMT_SHMEM, 297 DQUOT_USAGE_ENABLED | 298 DQUOT_LIMITS_ENABLED); 299 if (err) 300 goto out_err; 301 } 302 return 0; 303 304 out_err: 305 pr_warn("tmpfs: failed to enable quota tracking (type=%d, err=%d)\n", 306 type, err); 307 for (type--; type >= 0; type--) 308 dquot_quota_off(sb, type); 309 return err; 310 } 311 312 static void shmem_disable_quotas(struct super_block *sb) 313 { 314 int type; 315 316 for (type = 0; type < SHMEM_MAXQUOTAS; type++) 317 dquot_quota_off(sb, type); 318 } 319 320 static struct dquot __rcu **shmem_get_dquots(struct inode *inode) 321 { 322 return SHMEM_I(inode)->i_dquot; 323 } 324 #endif /* CONFIG_TMPFS_QUOTA */ 325 326 /* 327 * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and 328 * produces a novel ino for the newly allocated inode. 329 * 330 * It may also be called when making a hard link to permit the space needed by 331 * each dentry. However, in that case, no new inode number is needed since that 332 * internally draws from another pool of inode numbers (currently global 333 * get_next_ino()). This case is indicated by passing NULL as inop. 334 */ 335 #define SHMEM_INO_BATCH 1024 336 static int shmem_reserve_inode(struct super_block *sb, ino_t *inop) 337 { 338 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 339 ino_t ino; 340 341 if (!(sb->s_flags & SB_KERNMOUNT)) { 342 raw_spin_lock(&sbinfo->stat_lock); 343 if (sbinfo->max_inodes) { 344 if (sbinfo->free_ispace < BOGO_INODE_SIZE) { 345 raw_spin_unlock(&sbinfo->stat_lock); 346 return -ENOSPC; 347 } 348 sbinfo->free_ispace -= BOGO_INODE_SIZE; 349 } 350 if (inop) { 351 ino = sbinfo->next_ino++; 352 if (unlikely(is_zero_ino(ino))) 353 ino = sbinfo->next_ino++; 354 if (unlikely(!sbinfo->full_inums && 355 ino > UINT_MAX)) { 356 /* 357 * Emulate get_next_ino uint wraparound for 358 * compatibility 359 */ 360 if (IS_ENABLED(CONFIG_64BIT)) 361 pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n", 362 __func__, MINOR(sb->s_dev)); 363 sbinfo->next_ino = 1; 364 ino = sbinfo->next_ino++; 365 } 366 *inop = ino; 367 } 368 raw_spin_unlock(&sbinfo->stat_lock); 369 } else if (inop) { 370 /* 371 * __shmem_file_setup, one of our callers, is lock-free: it 372 * doesn't hold stat_lock in shmem_reserve_inode since 373 * max_inodes is always 0, and is called from potentially 374 * unknown contexts. As such, use a per-cpu batched allocator 375 * which doesn't require the per-sb stat_lock unless we are at 376 * the batch boundary. 377 * 378 * We don't need to worry about inode{32,64} since SB_KERNMOUNT 379 * shmem mounts are not exposed to userspace, so we don't need 380 * to worry about things like glibc compatibility. 381 */ 382 ino_t *next_ino; 383 384 next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu()); 385 ino = *next_ino; 386 if (unlikely(ino % SHMEM_INO_BATCH == 0)) { 387 raw_spin_lock(&sbinfo->stat_lock); 388 ino = sbinfo->next_ino; 389 sbinfo->next_ino += SHMEM_INO_BATCH; 390 raw_spin_unlock(&sbinfo->stat_lock); 391 if (unlikely(is_zero_ino(ino))) 392 ino++; 393 } 394 *inop = ino; 395 *next_ino = ++ino; 396 put_cpu(); 397 } 398 399 return 0; 400 } 401 402 static void shmem_free_inode(struct super_block *sb, size_t freed_ispace) 403 { 404 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 405 if (sbinfo->max_inodes) { 406 raw_spin_lock(&sbinfo->stat_lock); 407 sbinfo->free_ispace += BOGO_INODE_SIZE + freed_ispace; 408 raw_spin_unlock(&sbinfo->stat_lock); 409 } 410 } 411 412 /** 413 * shmem_recalc_inode - recalculate the block usage of an inode 414 * @inode: inode to recalc 415 * @alloced: the change in number of pages allocated to inode 416 * @swapped: the change in number of pages swapped from inode 417 * 418 * We have to calculate the free blocks since the mm can drop 419 * undirtied hole pages behind our back. 420 * 421 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 422 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 423 */ 424 static void shmem_recalc_inode(struct inode *inode, long alloced, long swapped) 425 { 426 struct shmem_inode_info *info = SHMEM_I(inode); 427 long freed; 428 429 spin_lock(&info->lock); 430 info->alloced += alloced; 431 info->swapped += swapped; 432 freed = info->alloced - info->swapped - 433 READ_ONCE(inode->i_mapping->nrpages); 434 /* 435 * Special case: whereas normally shmem_recalc_inode() is called 436 * after i_mapping->nrpages has already been adjusted (up or down), 437 * shmem_writepage() has to raise swapped before nrpages is lowered - 438 * to stop a racing shmem_recalc_inode() from thinking that a page has 439 * been freed. Compensate here, to avoid the need for a followup call. 440 */ 441 if (swapped > 0) 442 freed += swapped; 443 if (freed > 0) 444 info->alloced -= freed; 445 spin_unlock(&info->lock); 446 447 /* The quota case may block */ 448 if (freed > 0) 449 shmem_inode_unacct_blocks(inode, freed); 450 } 451 452 bool shmem_charge(struct inode *inode, long pages) 453 { 454 struct address_space *mapping = inode->i_mapping; 455 456 if (shmem_inode_acct_blocks(inode, pages)) 457 return false; 458 459 /* nrpages adjustment first, then shmem_recalc_inode() when balanced */ 460 xa_lock_irq(&mapping->i_pages); 461 mapping->nrpages += pages; 462 xa_unlock_irq(&mapping->i_pages); 463 464 shmem_recalc_inode(inode, pages, 0); 465 return true; 466 } 467 468 void shmem_uncharge(struct inode *inode, long pages) 469 { 470 /* pages argument is currently unused: keep it to help debugging */ 471 /* nrpages adjustment done by __filemap_remove_folio() or caller */ 472 473 shmem_recalc_inode(inode, 0, 0); 474 } 475 476 /* 477 * Replace item expected in xarray by a new item, while holding xa_lock. 478 */ 479 static int shmem_replace_entry(struct address_space *mapping, 480 pgoff_t index, void *expected, void *replacement) 481 { 482 XA_STATE(xas, &mapping->i_pages, index); 483 void *item; 484 485 VM_BUG_ON(!expected); 486 VM_BUG_ON(!replacement); 487 item = xas_load(&xas); 488 if (item != expected) 489 return -ENOENT; 490 xas_store(&xas, replacement); 491 return 0; 492 } 493 494 /* 495 * Sometimes, before we decide whether to proceed or to fail, we must check 496 * that an entry was not already brought back from swap by a racing thread. 497 * 498 * Checking page is not enough: by the time a SwapCache page is locked, it 499 * might be reused, and again be SwapCache, using the same swap as before. 500 */ 501 static bool shmem_confirm_swap(struct address_space *mapping, 502 pgoff_t index, swp_entry_t swap) 503 { 504 return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap); 505 } 506 507 /* 508 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option 509 * 510 * SHMEM_HUGE_NEVER: 511 * disables huge pages for the mount; 512 * SHMEM_HUGE_ALWAYS: 513 * enables huge pages for the mount; 514 * SHMEM_HUGE_WITHIN_SIZE: 515 * only allocate huge pages if the page will be fully within i_size, 516 * also respect fadvise()/madvise() hints; 517 * SHMEM_HUGE_ADVISE: 518 * only allocate huge pages if requested with fadvise()/madvise(); 519 */ 520 521 #define SHMEM_HUGE_NEVER 0 522 #define SHMEM_HUGE_ALWAYS 1 523 #define SHMEM_HUGE_WITHIN_SIZE 2 524 #define SHMEM_HUGE_ADVISE 3 525 526 /* 527 * Special values. 528 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled: 529 * 530 * SHMEM_HUGE_DENY: 531 * disables huge on shm_mnt and all mounts, for emergency use; 532 * SHMEM_HUGE_FORCE: 533 * enables huge on shm_mnt and all mounts, w/o needing option, for testing; 534 * 535 */ 536 #define SHMEM_HUGE_DENY (-1) 537 #define SHMEM_HUGE_FORCE (-2) 538 539 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 540 /* ifdef here to avoid bloating shmem.o when not necessary */ 541 542 static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER; 543 544 static bool __shmem_is_huge(struct inode *inode, pgoff_t index, 545 bool shmem_huge_force, struct mm_struct *mm, 546 unsigned long vm_flags) 547 { 548 loff_t i_size; 549 550 if (!S_ISREG(inode->i_mode)) 551 return false; 552 if (mm && ((vm_flags & VM_NOHUGEPAGE) || test_bit(MMF_DISABLE_THP, &mm->flags))) 553 return false; 554 if (shmem_huge == SHMEM_HUGE_DENY) 555 return false; 556 if (shmem_huge_force || shmem_huge == SHMEM_HUGE_FORCE) 557 return true; 558 559 switch (SHMEM_SB(inode->i_sb)->huge) { 560 case SHMEM_HUGE_ALWAYS: 561 return true; 562 case SHMEM_HUGE_WITHIN_SIZE: 563 index = round_up(index + 1, HPAGE_PMD_NR); 564 i_size = round_up(i_size_read(inode), PAGE_SIZE); 565 if (i_size >> PAGE_SHIFT >= index) 566 return true; 567 fallthrough; 568 case SHMEM_HUGE_ADVISE: 569 if (mm && (vm_flags & VM_HUGEPAGE)) 570 return true; 571 fallthrough; 572 default: 573 return false; 574 } 575 } 576 577 bool shmem_is_huge(struct inode *inode, pgoff_t index, 578 bool shmem_huge_force, struct mm_struct *mm, 579 unsigned long vm_flags) 580 { 581 if (HPAGE_PMD_ORDER > MAX_PAGECACHE_ORDER) 582 return false; 583 584 return __shmem_is_huge(inode, index, shmem_huge_force, mm, vm_flags); 585 } 586 587 #if defined(CONFIG_SYSFS) 588 static int shmem_parse_huge(const char *str) 589 { 590 if (!strcmp(str, "never")) 591 return SHMEM_HUGE_NEVER; 592 if (!strcmp(str, "always")) 593 return SHMEM_HUGE_ALWAYS; 594 if (!strcmp(str, "within_size")) 595 return SHMEM_HUGE_WITHIN_SIZE; 596 if (!strcmp(str, "advise")) 597 return SHMEM_HUGE_ADVISE; 598 if (!strcmp(str, "deny")) 599 return SHMEM_HUGE_DENY; 600 if (!strcmp(str, "force")) 601 return SHMEM_HUGE_FORCE; 602 return -EINVAL; 603 } 604 #endif 605 606 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS) 607 static const char *shmem_format_huge(int huge) 608 { 609 switch (huge) { 610 case SHMEM_HUGE_NEVER: 611 return "never"; 612 case SHMEM_HUGE_ALWAYS: 613 return "always"; 614 case SHMEM_HUGE_WITHIN_SIZE: 615 return "within_size"; 616 case SHMEM_HUGE_ADVISE: 617 return "advise"; 618 case SHMEM_HUGE_DENY: 619 return "deny"; 620 case SHMEM_HUGE_FORCE: 621 return "force"; 622 default: 623 VM_BUG_ON(1); 624 return "bad_val"; 625 } 626 } 627 #endif 628 629 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 630 struct shrink_control *sc, unsigned long nr_to_split) 631 { 632 LIST_HEAD(list), *pos, *next; 633 LIST_HEAD(to_remove); 634 struct inode *inode; 635 struct shmem_inode_info *info; 636 struct folio *folio; 637 unsigned long batch = sc ? sc->nr_to_scan : 128; 638 int split = 0; 639 640 if (list_empty(&sbinfo->shrinklist)) 641 return SHRINK_STOP; 642 643 spin_lock(&sbinfo->shrinklist_lock); 644 list_for_each_safe(pos, next, &sbinfo->shrinklist) { 645 info = list_entry(pos, struct shmem_inode_info, shrinklist); 646 647 /* pin the inode */ 648 inode = igrab(&info->vfs_inode); 649 650 /* inode is about to be evicted */ 651 if (!inode) { 652 list_del_init(&info->shrinklist); 653 goto next; 654 } 655 656 /* Check if there's anything to gain */ 657 if (round_up(inode->i_size, PAGE_SIZE) == 658 round_up(inode->i_size, HPAGE_PMD_SIZE)) { 659 list_move(&info->shrinklist, &to_remove); 660 goto next; 661 } 662 663 list_move(&info->shrinklist, &list); 664 next: 665 sbinfo->shrinklist_len--; 666 if (!--batch) 667 break; 668 } 669 spin_unlock(&sbinfo->shrinklist_lock); 670 671 list_for_each_safe(pos, next, &to_remove) { 672 info = list_entry(pos, struct shmem_inode_info, shrinklist); 673 inode = &info->vfs_inode; 674 list_del_init(&info->shrinklist); 675 iput(inode); 676 } 677 678 list_for_each_safe(pos, next, &list) { 679 int ret; 680 pgoff_t index; 681 682 info = list_entry(pos, struct shmem_inode_info, shrinklist); 683 inode = &info->vfs_inode; 684 685 if (nr_to_split && split >= nr_to_split) 686 goto move_back; 687 688 index = (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT; 689 folio = filemap_get_folio(inode->i_mapping, index); 690 if (IS_ERR(folio)) 691 goto drop; 692 693 /* No huge page at the end of the file: nothing to split */ 694 if (!folio_test_large(folio)) { 695 folio_put(folio); 696 goto drop; 697 } 698 699 /* 700 * Move the inode on the list back to shrinklist if we failed 701 * to lock the page at this time. 702 * 703 * Waiting for the lock may lead to deadlock in the 704 * reclaim path. 705 */ 706 if (!folio_trylock(folio)) { 707 folio_put(folio); 708 goto move_back; 709 } 710 711 ret = split_folio(folio); 712 folio_unlock(folio); 713 folio_put(folio); 714 715 /* If split failed move the inode on the list back to shrinklist */ 716 if (ret) 717 goto move_back; 718 719 split++; 720 drop: 721 list_del_init(&info->shrinklist); 722 goto put; 723 move_back: 724 /* 725 * Make sure the inode is either on the global list or deleted 726 * from any local list before iput() since it could be deleted 727 * in another thread once we put the inode (then the local list 728 * is corrupted). 729 */ 730 spin_lock(&sbinfo->shrinklist_lock); 731 list_move(&info->shrinklist, &sbinfo->shrinklist); 732 sbinfo->shrinklist_len++; 733 spin_unlock(&sbinfo->shrinklist_lock); 734 put: 735 iput(inode); 736 } 737 738 return split; 739 } 740 741 static long shmem_unused_huge_scan(struct super_block *sb, 742 struct shrink_control *sc) 743 { 744 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 745 746 if (!READ_ONCE(sbinfo->shrinklist_len)) 747 return SHRINK_STOP; 748 749 return shmem_unused_huge_shrink(sbinfo, sc, 0); 750 } 751 752 static long shmem_unused_huge_count(struct super_block *sb, 753 struct shrink_control *sc) 754 { 755 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 756 return READ_ONCE(sbinfo->shrinklist_len); 757 } 758 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 759 760 #define shmem_huge SHMEM_HUGE_DENY 761 762 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 763 struct shrink_control *sc, unsigned long nr_to_split) 764 { 765 return 0; 766 } 767 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 768 769 /* 770 * Somewhat like filemap_add_folio, but error if expected item has gone. 771 */ 772 static int shmem_add_to_page_cache(struct folio *folio, 773 struct address_space *mapping, 774 pgoff_t index, void *expected, gfp_t gfp) 775 { 776 XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio)); 777 long nr = folio_nr_pages(folio); 778 779 VM_BUG_ON_FOLIO(index != round_down(index, nr), folio); 780 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio); 781 VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio); 782 VM_BUG_ON(expected && folio_test_large(folio)); 783 784 folio_ref_add(folio, nr); 785 folio->mapping = mapping; 786 folio->index = index; 787 788 gfp &= GFP_RECLAIM_MASK; 789 folio_throttle_swaprate(folio, gfp); 790 791 do { 792 xas_lock_irq(&xas); 793 if (expected != xas_find_conflict(&xas)) { 794 xas_set_err(&xas, -EEXIST); 795 goto unlock; 796 } 797 if (expected && xas_find_conflict(&xas)) { 798 xas_set_err(&xas, -EEXIST); 799 goto unlock; 800 } 801 xas_store(&xas, folio); 802 if (xas_error(&xas)) 803 goto unlock; 804 if (folio_test_pmd_mappable(folio)) 805 __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, nr); 806 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr); 807 __lruvec_stat_mod_folio(folio, NR_SHMEM, nr); 808 mapping->nrpages += nr; 809 unlock: 810 xas_unlock_irq(&xas); 811 } while (xas_nomem(&xas, gfp)); 812 813 if (xas_error(&xas)) { 814 folio->mapping = NULL; 815 folio_ref_sub(folio, nr); 816 return xas_error(&xas); 817 } 818 819 return 0; 820 } 821 822 /* 823 * Somewhat like filemap_remove_folio, but substitutes swap for @folio. 824 */ 825 static void shmem_delete_from_page_cache(struct folio *folio, void *radswap) 826 { 827 struct address_space *mapping = folio->mapping; 828 long nr = folio_nr_pages(folio); 829 int error; 830 831 xa_lock_irq(&mapping->i_pages); 832 error = shmem_replace_entry(mapping, folio->index, folio, radswap); 833 folio->mapping = NULL; 834 mapping->nrpages -= nr; 835 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr); 836 __lruvec_stat_mod_folio(folio, NR_SHMEM, -nr); 837 xa_unlock_irq(&mapping->i_pages); 838 folio_put(folio); 839 BUG_ON(error); 840 } 841 842 /* 843 * Remove swap entry from page cache, free the swap and its page cache. 844 */ 845 static int shmem_free_swap(struct address_space *mapping, 846 pgoff_t index, void *radswap) 847 { 848 void *old; 849 850 old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0); 851 if (old != radswap) 852 return -ENOENT; 853 free_swap_and_cache(radix_to_swp_entry(radswap)); 854 return 0; 855 } 856 857 /* 858 * Determine (in bytes) how many of the shmem object's pages mapped by the 859 * given offsets are swapped out. 860 * 861 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU, 862 * as long as the inode doesn't go away and racy results are not a problem. 863 */ 864 unsigned long shmem_partial_swap_usage(struct address_space *mapping, 865 pgoff_t start, pgoff_t end) 866 { 867 XA_STATE(xas, &mapping->i_pages, start); 868 struct page *page; 869 unsigned long swapped = 0; 870 unsigned long max = end - 1; 871 872 rcu_read_lock(); 873 xas_for_each(&xas, page, max) { 874 if (xas_retry(&xas, page)) 875 continue; 876 if (xa_is_value(page)) 877 swapped++; 878 if (xas.xa_index == max) 879 break; 880 if (need_resched()) { 881 xas_pause(&xas); 882 cond_resched_rcu(); 883 } 884 } 885 rcu_read_unlock(); 886 887 return swapped << PAGE_SHIFT; 888 } 889 890 /* 891 * Determine (in bytes) how many of the shmem object's pages mapped by the 892 * given vma is swapped out. 893 * 894 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU, 895 * as long as the inode doesn't go away and racy results are not a problem. 896 */ 897 unsigned long shmem_swap_usage(struct vm_area_struct *vma) 898 { 899 struct inode *inode = file_inode(vma->vm_file); 900 struct shmem_inode_info *info = SHMEM_I(inode); 901 struct address_space *mapping = inode->i_mapping; 902 unsigned long swapped; 903 904 /* Be careful as we don't hold info->lock */ 905 swapped = READ_ONCE(info->swapped); 906 907 /* 908 * The easier cases are when the shmem object has nothing in swap, or 909 * the vma maps it whole. Then we can simply use the stats that we 910 * already track. 911 */ 912 if (!swapped) 913 return 0; 914 915 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) 916 return swapped << PAGE_SHIFT; 917 918 /* Here comes the more involved part */ 919 return shmem_partial_swap_usage(mapping, vma->vm_pgoff, 920 vma->vm_pgoff + vma_pages(vma)); 921 } 922 923 /* 924 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists. 925 */ 926 void shmem_unlock_mapping(struct address_space *mapping) 927 { 928 struct folio_batch fbatch; 929 pgoff_t index = 0; 930 931 folio_batch_init(&fbatch); 932 /* 933 * Minor point, but we might as well stop if someone else SHM_LOCKs it. 934 */ 935 while (!mapping_unevictable(mapping) && 936 filemap_get_folios(mapping, &index, ~0UL, &fbatch)) { 937 check_move_unevictable_folios(&fbatch); 938 folio_batch_release(&fbatch); 939 cond_resched(); 940 } 941 } 942 943 static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index) 944 { 945 struct folio *folio; 946 947 /* 948 * At first avoid shmem_get_folio(,,,SGP_READ): that fails 949 * beyond i_size, and reports fallocated folios as holes. 950 */ 951 folio = filemap_get_entry(inode->i_mapping, index); 952 if (!folio) 953 return folio; 954 if (!xa_is_value(folio)) { 955 folio_lock(folio); 956 if (folio->mapping == inode->i_mapping) 957 return folio; 958 /* The folio has been swapped out */ 959 folio_unlock(folio); 960 folio_put(folio); 961 } 962 /* 963 * But read a folio back from swap if any of it is within i_size 964 * (although in some cases this is just a waste of time). 965 */ 966 folio = NULL; 967 shmem_get_folio(inode, index, &folio, SGP_READ); 968 return folio; 969 } 970 971 /* 972 * Remove range of pages and swap entries from page cache, and free them. 973 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. 974 */ 975 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, 976 bool unfalloc) 977 { 978 struct address_space *mapping = inode->i_mapping; 979 struct shmem_inode_info *info = SHMEM_I(inode); 980 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT; 981 pgoff_t end = (lend + 1) >> PAGE_SHIFT; 982 struct folio_batch fbatch; 983 pgoff_t indices[PAGEVEC_SIZE]; 984 struct folio *folio; 985 bool same_folio; 986 long nr_swaps_freed = 0; 987 pgoff_t index; 988 int i; 989 990 if (lend == -1) 991 end = -1; /* unsigned, so actually very big */ 992 993 if (info->fallocend > start && info->fallocend <= end && !unfalloc) 994 info->fallocend = start; 995 996 folio_batch_init(&fbatch); 997 index = start; 998 while (index < end && find_lock_entries(mapping, &index, end - 1, 999 &fbatch, indices)) { 1000 for (i = 0; i < folio_batch_count(&fbatch); i++) { 1001 folio = fbatch.folios[i]; 1002 1003 if (xa_is_value(folio)) { 1004 if (unfalloc) 1005 continue; 1006 nr_swaps_freed += !shmem_free_swap(mapping, 1007 indices[i], folio); 1008 continue; 1009 } 1010 1011 if (!unfalloc || !folio_test_uptodate(folio)) 1012 truncate_inode_folio(mapping, folio); 1013 folio_unlock(folio); 1014 } 1015 folio_batch_remove_exceptionals(&fbatch); 1016 folio_batch_release(&fbatch); 1017 cond_resched(); 1018 } 1019 1020 /* 1021 * When undoing a failed fallocate, we want none of the partial folio 1022 * zeroing and splitting below, but shall want to truncate the whole 1023 * folio when !uptodate indicates that it was added by this fallocate, 1024 * even when [lstart, lend] covers only a part of the folio. 1025 */ 1026 if (unfalloc) 1027 goto whole_folios; 1028 1029 same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT); 1030 folio = shmem_get_partial_folio(inode, lstart >> PAGE_SHIFT); 1031 if (folio) { 1032 same_folio = lend < folio_pos(folio) + folio_size(folio); 1033 folio_mark_dirty(folio); 1034 if (!truncate_inode_partial_folio(folio, lstart, lend)) { 1035 start = folio_next_index(folio); 1036 if (same_folio) 1037 end = folio->index; 1038 } 1039 folio_unlock(folio); 1040 folio_put(folio); 1041 folio = NULL; 1042 } 1043 1044 if (!same_folio) 1045 folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT); 1046 if (folio) { 1047 folio_mark_dirty(folio); 1048 if (!truncate_inode_partial_folio(folio, lstart, lend)) 1049 end = folio->index; 1050 folio_unlock(folio); 1051 folio_put(folio); 1052 } 1053 1054 whole_folios: 1055 1056 index = start; 1057 while (index < end) { 1058 cond_resched(); 1059 1060 if (!find_get_entries(mapping, &index, end - 1, &fbatch, 1061 indices)) { 1062 /* If all gone or hole-punch or unfalloc, we're done */ 1063 if (index == start || end != -1) 1064 break; 1065 /* But if truncating, restart to make sure all gone */ 1066 index = start; 1067 continue; 1068 } 1069 for (i = 0; i < folio_batch_count(&fbatch); i++) { 1070 folio = fbatch.folios[i]; 1071 1072 if (xa_is_value(folio)) { 1073 if (unfalloc) 1074 continue; 1075 if (shmem_free_swap(mapping, indices[i], folio)) { 1076 /* Swap was replaced by page: retry */ 1077 index = indices[i]; 1078 break; 1079 } 1080 nr_swaps_freed++; 1081 continue; 1082 } 1083 1084 folio_lock(folio); 1085 1086 if (!unfalloc || !folio_test_uptodate(folio)) { 1087 if (folio_mapping(folio) != mapping) { 1088 /* Page was replaced by swap: retry */ 1089 folio_unlock(folio); 1090 index = indices[i]; 1091 break; 1092 } 1093 VM_BUG_ON_FOLIO(folio_test_writeback(folio), 1094 folio); 1095 1096 if (!folio_test_large(folio)) { 1097 truncate_inode_folio(mapping, folio); 1098 } else if (truncate_inode_partial_folio(folio, lstart, lend)) { 1099 /* 1100 * If we split a page, reset the loop so 1101 * that we pick up the new sub pages. 1102 * Otherwise the THP was entirely 1103 * dropped or the target range was 1104 * zeroed, so just continue the loop as 1105 * is. 1106 */ 1107 if (!folio_test_large(folio)) { 1108 folio_unlock(folio); 1109 index = start; 1110 break; 1111 } 1112 } 1113 } 1114 folio_unlock(folio); 1115 } 1116 folio_batch_remove_exceptionals(&fbatch); 1117 folio_batch_release(&fbatch); 1118 } 1119 1120 shmem_recalc_inode(inode, 0, -nr_swaps_freed); 1121 } 1122 1123 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 1124 { 1125 shmem_undo_range(inode, lstart, lend, false); 1126 inode_set_mtime_to_ts(inode, inode_set_ctime_current(inode)); 1127 inode_inc_iversion(inode); 1128 } 1129 EXPORT_SYMBOL_GPL(shmem_truncate_range); 1130 1131 static int shmem_getattr(struct mnt_idmap *idmap, 1132 const struct path *path, struct kstat *stat, 1133 u32 request_mask, unsigned int query_flags) 1134 { 1135 struct inode *inode = path->dentry->d_inode; 1136 struct shmem_inode_info *info = SHMEM_I(inode); 1137 1138 if (info->alloced - info->swapped != inode->i_mapping->nrpages) 1139 shmem_recalc_inode(inode, 0, 0); 1140 1141 if (info->fsflags & FS_APPEND_FL) 1142 stat->attributes |= STATX_ATTR_APPEND; 1143 if (info->fsflags & FS_IMMUTABLE_FL) 1144 stat->attributes |= STATX_ATTR_IMMUTABLE; 1145 if (info->fsflags & FS_NODUMP_FL) 1146 stat->attributes |= STATX_ATTR_NODUMP; 1147 stat->attributes_mask |= (STATX_ATTR_APPEND | 1148 STATX_ATTR_IMMUTABLE | 1149 STATX_ATTR_NODUMP); 1150 generic_fillattr(idmap, request_mask, inode, stat); 1151 1152 if (shmem_is_huge(inode, 0, false, NULL, 0)) 1153 stat->blksize = HPAGE_PMD_SIZE; 1154 1155 if (request_mask & STATX_BTIME) { 1156 stat->result_mask |= STATX_BTIME; 1157 stat->btime.tv_sec = info->i_crtime.tv_sec; 1158 stat->btime.tv_nsec = info->i_crtime.tv_nsec; 1159 } 1160 1161 return 0; 1162 } 1163 1164 static int shmem_setattr(struct mnt_idmap *idmap, 1165 struct dentry *dentry, struct iattr *attr) 1166 { 1167 struct inode *inode = d_inode(dentry); 1168 struct shmem_inode_info *info = SHMEM_I(inode); 1169 int error; 1170 bool update_mtime = false; 1171 bool update_ctime = true; 1172 1173 error = setattr_prepare(idmap, dentry, attr); 1174 if (error) 1175 return error; 1176 1177 if ((info->seals & F_SEAL_EXEC) && (attr->ia_valid & ATTR_MODE)) { 1178 if ((inode->i_mode ^ attr->ia_mode) & 0111) { 1179 return -EPERM; 1180 } 1181 } 1182 1183 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 1184 loff_t oldsize = inode->i_size; 1185 loff_t newsize = attr->ia_size; 1186 1187 /* protected by i_rwsem */ 1188 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || 1189 (newsize > oldsize && (info->seals & F_SEAL_GROW))) 1190 return -EPERM; 1191 1192 if (newsize != oldsize) { 1193 error = shmem_reacct_size(SHMEM_I(inode)->flags, 1194 oldsize, newsize); 1195 if (error) 1196 return error; 1197 i_size_write(inode, newsize); 1198 update_mtime = true; 1199 } else { 1200 update_ctime = false; 1201 } 1202 if (newsize <= oldsize) { 1203 loff_t holebegin = round_up(newsize, PAGE_SIZE); 1204 if (oldsize > holebegin) 1205 unmap_mapping_range(inode->i_mapping, 1206 holebegin, 0, 1); 1207 if (info->alloced) 1208 shmem_truncate_range(inode, 1209 newsize, (loff_t)-1); 1210 /* unmap again to remove racily COWed private pages */ 1211 if (oldsize > holebegin) 1212 unmap_mapping_range(inode->i_mapping, 1213 holebegin, 0, 1); 1214 } 1215 } 1216 1217 if (is_quota_modification(idmap, inode, attr)) { 1218 error = dquot_initialize(inode); 1219 if (error) 1220 return error; 1221 } 1222 1223 /* Transfer quota accounting */ 1224 if (i_uid_needs_update(idmap, attr, inode) || 1225 i_gid_needs_update(idmap, attr, inode)) { 1226 error = dquot_transfer(idmap, inode, attr); 1227 if (error) 1228 return error; 1229 } 1230 1231 setattr_copy(idmap, inode, attr); 1232 if (attr->ia_valid & ATTR_MODE) 1233 error = posix_acl_chmod(idmap, dentry, inode->i_mode); 1234 if (!error && update_ctime) { 1235 inode_set_ctime_current(inode); 1236 if (update_mtime) 1237 inode_set_mtime_to_ts(inode, inode_get_ctime(inode)); 1238 inode_inc_iversion(inode); 1239 } 1240 return error; 1241 } 1242 1243 static void shmem_evict_inode(struct inode *inode) 1244 { 1245 struct shmem_inode_info *info = SHMEM_I(inode); 1246 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1247 size_t freed = 0; 1248 1249 if (shmem_mapping(inode->i_mapping)) { 1250 shmem_unacct_size(info->flags, inode->i_size); 1251 inode->i_size = 0; 1252 mapping_set_exiting(inode->i_mapping); 1253 shmem_truncate_range(inode, 0, (loff_t)-1); 1254 if (!list_empty(&info->shrinklist)) { 1255 spin_lock(&sbinfo->shrinklist_lock); 1256 if (!list_empty(&info->shrinklist)) { 1257 list_del_init(&info->shrinklist); 1258 sbinfo->shrinklist_len--; 1259 } 1260 spin_unlock(&sbinfo->shrinklist_lock); 1261 } 1262 while (!list_empty(&info->swaplist)) { 1263 /* Wait while shmem_unuse() is scanning this inode... */ 1264 wait_var_event(&info->stop_eviction, 1265 !atomic_read(&info->stop_eviction)); 1266 mutex_lock(&shmem_swaplist_mutex); 1267 /* ...but beware of the race if we peeked too early */ 1268 if (!atomic_read(&info->stop_eviction)) 1269 list_del_init(&info->swaplist); 1270 mutex_unlock(&shmem_swaplist_mutex); 1271 } 1272 } 1273 1274 simple_xattrs_free(&info->xattrs, sbinfo->max_inodes ? &freed : NULL); 1275 shmem_free_inode(inode->i_sb, freed); 1276 WARN_ON(inode->i_blocks); 1277 clear_inode(inode); 1278 #ifdef CONFIG_TMPFS_QUOTA 1279 dquot_free_inode(inode); 1280 dquot_drop(inode); 1281 #endif 1282 } 1283 1284 static int shmem_find_swap_entries(struct address_space *mapping, 1285 pgoff_t start, struct folio_batch *fbatch, 1286 pgoff_t *indices, unsigned int type) 1287 { 1288 XA_STATE(xas, &mapping->i_pages, start); 1289 struct folio *folio; 1290 swp_entry_t entry; 1291 1292 rcu_read_lock(); 1293 xas_for_each(&xas, folio, ULONG_MAX) { 1294 if (xas_retry(&xas, folio)) 1295 continue; 1296 1297 if (!xa_is_value(folio)) 1298 continue; 1299 1300 entry = radix_to_swp_entry(folio); 1301 /* 1302 * swapin error entries can be found in the mapping. But they're 1303 * deliberately ignored here as we've done everything we can do. 1304 */ 1305 if (swp_type(entry) != type) 1306 continue; 1307 1308 indices[folio_batch_count(fbatch)] = xas.xa_index; 1309 if (!folio_batch_add(fbatch, folio)) 1310 break; 1311 1312 if (need_resched()) { 1313 xas_pause(&xas); 1314 cond_resched_rcu(); 1315 } 1316 } 1317 rcu_read_unlock(); 1318 1319 return xas.xa_index; 1320 } 1321 1322 /* 1323 * Move the swapped pages for an inode to page cache. Returns the count 1324 * of pages swapped in, or the error in case of failure. 1325 */ 1326 static int shmem_unuse_swap_entries(struct inode *inode, 1327 struct folio_batch *fbatch, pgoff_t *indices) 1328 { 1329 int i = 0; 1330 int ret = 0; 1331 int error = 0; 1332 struct address_space *mapping = inode->i_mapping; 1333 1334 for (i = 0; i < folio_batch_count(fbatch); i++) { 1335 struct folio *folio = fbatch->folios[i]; 1336 1337 if (!xa_is_value(folio)) 1338 continue; 1339 error = shmem_swapin_folio(inode, indices[i], &folio, SGP_CACHE, 1340 mapping_gfp_mask(mapping), NULL, NULL); 1341 if (error == 0) { 1342 folio_unlock(folio); 1343 folio_put(folio); 1344 ret++; 1345 } 1346 if (error == -ENOMEM) 1347 break; 1348 error = 0; 1349 } 1350 return error ? error : ret; 1351 } 1352 1353 /* 1354 * If swap found in inode, free it and move page from swapcache to filecache. 1355 */ 1356 static int shmem_unuse_inode(struct inode *inode, unsigned int type) 1357 { 1358 struct address_space *mapping = inode->i_mapping; 1359 pgoff_t start = 0; 1360 struct folio_batch fbatch; 1361 pgoff_t indices[PAGEVEC_SIZE]; 1362 int ret = 0; 1363 1364 do { 1365 folio_batch_init(&fbatch); 1366 shmem_find_swap_entries(mapping, start, &fbatch, indices, type); 1367 if (folio_batch_count(&fbatch) == 0) { 1368 ret = 0; 1369 break; 1370 } 1371 1372 ret = shmem_unuse_swap_entries(inode, &fbatch, indices); 1373 if (ret < 0) 1374 break; 1375 1376 start = indices[folio_batch_count(&fbatch) - 1]; 1377 } while (true); 1378 1379 return ret; 1380 } 1381 1382 /* 1383 * Read all the shared memory data that resides in the swap 1384 * device 'type' back into memory, so the swap device can be 1385 * unused. 1386 */ 1387 int shmem_unuse(unsigned int type) 1388 { 1389 struct shmem_inode_info *info, *next; 1390 int error = 0; 1391 1392 if (list_empty(&shmem_swaplist)) 1393 return 0; 1394 1395 mutex_lock(&shmem_swaplist_mutex); 1396 list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) { 1397 if (!info->swapped) { 1398 list_del_init(&info->swaplist); 1399 continue; 1400 } 1401 /* 1402 * Drop the swaplist mutex while searching the inode for swap; 1403 * but before doing so, make sure shmem_evict_inode() will not 1404 * remove placeholder inode from swaplist, nor let it be freed 1405 * (igrab() would protect from unlink, but not from unmount). 1406 */ 1407 atomic_inc(&info->stop_eviction); 1408 mutex_unlock(&shmem_swaplist_mutex); 1409 1410 error = shmem_unuse_inode(&info->vfs_inode, type); 1411 cond_resched(); 1412 1413 mutex_lock(&shmem_swaplist_mutex); 1414 next = list_next_entry(info, swaplist); 1415 if (!info->swapped) 1416 list_del_init(&info->swaplist); 1417 if (atomic_dec_and_test(&info->stop_eviction)) 1418 wake_up_var(&info->stop_eviction); 1419 if (error) 1420 break; 1421 } 1422 mutex_unlock(&shmem_swaplist_mutex); 1423 1424 return error; 1425 } 1426 1427 /* 1428 * Move the page from the page cache to the swap cache. 1429 */ 1430 static int shmem_writepage(struct page *page, struct writeback_control *wbc) 1431 { 1432 struct folio *folio = page_folio(page); 1433 struct address_space *mapping = folio->mapping; 1434 struct inode *inode = mapping->host; 1435 struct shmem_inode_info *info = SHMEM_I(inode); 1436 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1437 swp_entry_t swap; 1438 pgoff_t index; 1439 1440 /* 1441 * Our capabilities prevent regular writeback or sync from ever calling 1442 * shmem_writepage; but a stacking filesystem might use ->writepage of 1443 * its underlying filesystem, in which case tmpfs should write out to 1444 * swap only in response to memory pressure, and not for the writeback 1445 * threads or sync. 1446 */ 1447 if (WARN_ON_ONCE(!wbc->for_reclaim)) 1448 goto redirty; 1449 1450 if (WARN_ON_ONCE((info->flags & VM_LOCKED) || sbinfo->noswap)) 1451 goto redirty; 1452 1453 if (!total_swap_pages) 1454 goto redirty; 1455 1456 /* 1457 * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or 1458 * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages, 1459 * and its shmem_writeback() needs them to be split when swapping. 1460 */ 1461 if (folio_test_large(folio)) { 1462 /* Ensure the subpages are still dirty */ 1463 folio_test_set_dirty(folio); 1464 if (split_huge_page(page) < 0) 1465 goto redirty; 1466 folio = page_folio(page); 1467 folio_clear_dirty(folio); 1468 } 1469 1470 index = folio->index; 1471 1472 /* 1473 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC 1474 * value into swapfile.c, the only way we can correctly account for a 1475 * fallocated folio arriving here is now to initialize it and write it. 1476 * 1477 * That's okay for a folio already fallocated earlier, but if we have 1478 * not yet completed the fallocation, then (a) we want to keep track 1479 * of this folio in case we have to undo it, and (b) it may not be a 1480 * good idea to continue anyway, once we're pushing into swap. So 1481 * reactivate the folio, and let shmem_fallocate() quit when too many. 1482 */ 1483 if (!folio_test_uptodate(folio)) { 1484 if (inode->i_private) { 1485 struct shmem_falloc *shmem_falloc; 1486 spin_lock(&inode->i_lock); 1487 shmem_falloc = inode->i_private; 1488 if (shmem_falloc && 1489 !shmem_falloc->waitq && 1490 index >= shmem_falloc->start && 1491 index < shmem_falloc->next) 1492 shmem_falloc->nr_unswapped++; 1493 else 1494 shmem_falloc = NULL; 1495 spin_unlock(&inode->i_lock); 1496 if (shmem_falloc) 1497 goto redirty; 1498 } 1499 folio_zero_range(folio, 0, folio_size(folio)); 1500 flush_dcache_folio(folio); 1501 folio_mark_uptodate(folio); 1502 } 1503 1504 swap = folio_alloc_swap(folio); 1505 if (!swap.val) 1506 goto redirty; 1507 1508 /* 1509 * Add inode to shmem_unuse()'s list of swapped-out inodes, 1510 * if it's not already there. Do it now before the folio is 1511 * moved to swap cache, when its pagelock no longer protects 1512 * the inode from eviction. But don't unlock the mutex until 1513 * we've incremented swapped, because shmem_unuse_inode() will 1514 * prune a !swapped inode from the swaplist under this mutex. 1515 */ 1516 mutex_lock(&shmem_swaplist_mutex); 1517 if (list_empty(&info->swaplist)) 1518 list_add(&info->swaplist, &shmem_swaplist); 1519 1520 if (add_to_swap_cache(folio, swap, 1521 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN, 1522 NULL) == 0) { 1523 shmem_recalc_inode(inode, 0, 1); 1524 swap_shmem_alloc(swap); 1525 shmem_delete_from_page_cache(folio, swp_to_radix_entry(swap)); 1526 1527 mutex_unlock(&shmem_swaplist_mutex); 1528 BUG_ON(folio_mapped(folio)); 1529 return swap_writepage(&folio->page, wbc); 1530 } 1531 1532 mutex_unlock(&shmem_swaplist_mutex); 1533 put_swap_folio(folio, swap); 1534 redirty: 1535 folio_mark_dirty(folio); 1536 if (wbc->for_reclaim) 1537 return AOP_WRITEPAGE_ACTIVATE; /* Return with folio locked */ 1538 folio_unlock(folio); 1539 return 0; 1540 } 1541 1542 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) 1543 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1544 { 1545 char buffer[64]; 1546 1547 if (!mpol || mpol->mode == MPOL_DEFAULT) 1548 return; /* show nothing */ 1549 1550 mpol_to_str(buffer, sizeof(buffer), mpol); 1551 1552 seq_printf(seq, ",mpol=%s", buffer); 1553 } 1554 1555 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1556 { 1557 struct mempolicy *mpol = NULL; 1558 if (sbinfo->mpol) { 1559 raw_spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 1560 mpol = sbinfo->mpol; 1561 mpol_get(mpol); 1562 raw_spin_unlock(&sbinfo->stat_lock); 1563 } 1564 return mpol; 1565 } 1566 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */ 1567 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1568 { 1569 } 1570 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1571 { 1572 return NULL; 1573 } 1574 #endif /* CONFIG_NUMA && CONFIG_TMPFS */ 1575 1576 static struct mempolicy *shmem_get_pgoff_policy(struct shmem_inode_info *info, 1577 pgoff_t index, unsigned int order, pgoff_t *ilx); 1578 1579 static struct folio *shmem_swapin_cluster(swp_entry_t swap, gfp_t gfp, 1580 struct shmem_inode_info *info, pgoff_t index) 1581 { 1582 struct mempolicy *mpol; 1583 pgoff_t ilx; 1584 struct folio *folio; 1585 1586 mpol = shmem_get_pgoff_policy(info, index, 0, &ilx); 1587 folio = swap_cluster_readahead(swap, gfp, mpol, ilx); 1588 mpol_cond_put(mpol); 1589 1590 return folio; 1591 } 1592 1593 /* 1594 * Make sure huge_gfp is always more limited than limit_gfp. 1595 * Some of the flags set permissions, while others set limitations. 1596 */ 1597 static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp) 1598 { 1599 gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM; 1600 gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY; 1601 gfp_t zoneflags = limit_gfp & GFP_ZONEMASK; 1602 gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK); 1603 1604 /* Allow allocations only from the originally specified zones. */ 1605 result |= zoneflags; 1606 1607 /* 1608 * Minimize the result gfp by taking the union with the deny flags, 1609 * and the intersection of the allow flags. 1610 */ 1611 result |= (limit_gfp & denyflags); 1612 result |= (huge_gfp & limit_gfp) & allowflags; 1613 1614 return result; 1615 } 1616 1617 static struct folio *shmem_alloc_hugefolio(gfp_t gfp, 1618 struct shmem_inode_info *info, pgoff_t index) 1619 { 1620 struct mempolicy *mpol; 1621 pgoff_t ilx; 1622 struct page *page; 1623 1624 mpol = shmem_get_pgoff_policy(info, index, HPAGE_PMD_ORDER, &ilx); 1625 page = alloc_pages_mpol(gfp, HPAGE_PMD_ORDER, mpol, ilx, numa_node_id()); 1626 mpol_cond_put(mpol); 1627 1628 return page_rmappable_folio(page); 1629 } 1630 1631 static struct folio *shmem_alloc_folio(gfp_t gfp, 1632 struct shmem_inode_info *info, pgoff_t index) 1633 { 1634 struct mempolicy *mpol; 1635 pgoff_t ilx; 1636 struct page *page; 1637 1638 mpol = shmem_get_pgoff_policy(info, index, 0, &ilx); 1639 page = alloc_pages_mpol(gfp, 0, mpol, ilx, numa_node_id()); 1640 mpol_cond_put(mpol); 1641 1642 return (struct folio *)page; 1643 } 1644 1645 static struct folio *shmem_alloc_and_add_folio(gfp_t gfp, 1646 struct inode *inode, pgoff_t index, 1647 struct mm_struct *fault_mm, bool huge) 1648 { 1649 struct address_space *mapping = inode->i_mapping; 1650 struct shmem_inode_info *info = SHMEM_I(inode); 1651 struct folio *folio; 1652 long pages; 1653 int error; 1654 1655 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 1656 huge = false; 1657 1658 if (huge) { 1659 pages = HPAGE_PMD_NR; 1660 index = round_down(index, HPAGE_PMD_NR); 1661 1662 /* 1663 * Check for conflict before waiting on a huge allocation. 1664 * Conflict might be that a huge page has just been allocated 1665 * and added to page cache by a racing thread, or that there 1666 * is already at least one small page in the huge extent. 1667 * Be careful to retry when appropriate, but not forever! 1668 * Elsewhere -EEXIST would be the right code, but not here. 1669 */ 1670 if (xa_find(&mapping->i_pages, &index, 1671 index + HPAGE_PMD_NR - 1, XA_PRESENT)) 1672 return ERR_PTR(-E2BIG); 1673 1674 folio = shmem_alloc_hugefolio(gfp, info, index); 1675 if (!folio) 1676 count_vm_event(THP_FILE_FALLBACK); 1677 } else { 1678 pages = 1; 1679 folio = shmem_alloc_folio(gfp, info, index); 1680 } 1681 if (!folio) 1682 return ERR_PTR(-ENOMEM); 1683 1684 __folio_set_locked(folio); 1685 __folio_set_swapbacked(folio); 1686 1687 gfp &= GFP_RECLAIM_MASK; 1688 error = mem_cgroup_charge(folio, fault_mm, gfp); 1689 if (error) { 1690 if (xa_find(&mapping->i_pages, &index, 1691 index + pages - 1, XA_PRESENT)) { 1692 error = -EEXIST; 1693 } else if (huge) { 1694 count_vm_event(THP_FILE_FALLBACK); 1695 count_vm_event(THP_FILE_FALLBACK_CHARGE); 1696 } 1697 goto unlock; 1698 } 1699 1700 error = shmem_add_to_page_cache(folio, mapping, index, NULL, gfp); 1701 if (error) 1702 goto unlock; 1703 1704 error = shmem_inode_acct_blocks(inode, pages); 1705 if (error) { 1706 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1707 long freed; 1708 /* 1709 * Try to reclaim some space by splitting a few 1710 * large folios beyond i_size on the filesystem. 1711 */ 1712 shmem_unused_huge_shrink(sbinfo, NULL, 2); 1713 /* 1714 * And do a shmem_recalc_inode() to account for freed pages: 1715 * except our folio is there in cache, so not quite balanced. 1716 */ 1717 spin_lock(&info->lock); 1718 freed = pages + info->alloced - info->swapped - 1719 READ_ONCE(mapping->nrpages); 1720 if (freed > 0) 1721 info->alloced -= freed; 1722 spin_unlock(&info->lock); 1723 if (freed > 0) 1724 shmem_inode_unacct_blocks(inode, freed); 1725 error = shmem_inode_acct_blocks(inode, pages); 1726 if (error) { 1727 filemap_remove_folio(folio); 1728 goto unlock; 1729 } 1730 } 1731 1732 shmem_recalc_inode(inode, pages, 0); 1733 folio_add_lru(folio); 1734 return folio; 1735 1736 unlock: 1737 folio_unlock(folio); 1738 folio_put(folio); 1739 return ERR_PTR(error); 1740 } 1741 1742 /* 1743 * When a page is moved from swapcache to shmem filecache (either by the 1744 * usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of 1745 * shmem_unuse_inode()), it may have been read in earlier from swap, in 1746 * ignorance of the mapping it belongs to. If that mapping has special 1747 * constraints (like the gma500 GEM driver, which requires RAM below 4GB), 1748 * we may need to copy to a suitable page before moving to filecache. 1749 * 1750 * In a future release, this may well be extended to respect cpuset and 1751 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page(); 1752 * but for now it is a simple matter of zone. 1753 */ 1754 static bool shmem_should_replace_folio(struct folio *folio, gfp_t gfp) 1755 { 1756 return folio_zonenum(folio) > gfp_zone(gfp); 1757 } 1758 1759 static int shmem_replace_folio(struct folio **foliop, gfp_t gfp, 1760 struct shmem_inode_info *info, pgoff_t index) 1761 { 1762 struct folio *old, *new; 1763 struct address_space *swap_mapping; 1764 swp_entry_t entry; 1765 pgoff_t swap_index; 1766 int error; 1767 1768 old = *foliop; 1769 entry = old->swap; 1770 swap_index = swp_offset(entry); 1771 swap_mapping = swap_address_space(entry); 1772 1773 /* 1774 * We have arrived here because our zones are constrained, so don't 1775 * limit chance of success by further cpuset and node constraints. 1776 */ 1777 gfp &= ~GFP_CONSTRAINT_MASK; 1778 VM_BUG_ON_FOLIO(folio_test_large(old), old); 1779 new = shmem_alloc_folio(gfp, info, index); 1780 if (!new) 1781 return -ENOMEM; 1782 1783 folio_get(new); 1784 folio_copy(new, old); 1785 flush_dcache_folio(new); 1786 1787 __folio_set_locked(new); 1788 __folio_set_swapbacked(new); 1789 folio_mark_uptodate(new); 1790 new->swap = entry; 1791 folio_set_swapcache(new); 1792 1793 /* 1794 * Our caller will very soon move newpage out of swapcache, but it's 1795 * a nice clean interface for us to replace oldpage by newpage there. 1796 */ 1797 xa_lock_irq(&swap_mapping->i_pages); 1798 error = shmem_replace_entry(swap_mapping, swap_index, old, new); 1799 if (!error) { 1800 mem_cgroup_replace_folio(old, new); 1801 __lruvec_stat_mod_folio(new, NR_FILE_PAGES, 1); 1802 __lruvec_stat_mod_folio(new, NR_SHMEM, 1); 1803 __lruvec_stat_mod_folio(old, NR_FILE_PAGES, -1); 1804 __lruvec_stat_mod_folio(old, NR_SHMEM, -1); 1805 } 1806 xa_unlock_irq(&swap_mapping->i_pages); 1807 1808 if (unlikely(error)) { 1809 /* 1810 * Is this possible? I think not, now that our callers check 1811 * both PageSwapCache and page_private after getting page lock; 1812 * but be defensive. Reverse old to newpage for clear and free. 1813 */ 1814 old = new; 1815 } else { 1816 folio_add_lru(new); 1817 *foliop = new; 1818 } 1819 1820 folio_clear_swapcache(old); 1821 old->private = NULL; 1822 1823 folio_unlock(old); 1824 folio_put_refs(old, 2); 1825 return error; 1826 } 1827 1828 static void shmem_set_folio_swapin_error(struct inode *inode, pgoff_t index, 1829 struct folio *folio, swp_entry_t swap) 1830 { 1831 struct address_space *mapping = inode->i_mapping; 1832 swp_entry_t swapin_error; 1833 void *old; 1834 1835 swapin_error = make_poisoned_swp_entry(); 1836 old = xa_cmpxchg_irq(&mapping->i_pages, index, 1837 swp_to_radix_entry(swap), 1838 swp_to_radix_entry(swapin_error), 0); 1839 if (old != swp_to_radix_entry(swap)) 1840 return; 1841 1842 folio_wait_writeback(folio); 1843 delete_from_swap_cache(folio); 1844 /* 1845 * Don't treat swapin error folio as alloced. Otherwise inode->i_blocks 1846 * won't be 0 when inode is released and thus trigger WARN_ON(i_blocks) 1847 * in shmem_evict_inode(). 1848 */ 1849 shmem_recalc_inode(inode, -1, -1); 1850 swap_free(swap); 1851 } 1852 1853 /* 1854 * Swap in the folio pointed to by *foliop. 1855 * Caller has to make sure that *foliop contains a valid swapped folio. 1856 * Returns 0 and the folio in foliop if success. On failure, returns the 1857 * error code and NULL in *foliop. 1858 */ 1859 static int shmem_swapin_folio(struct inode *inode, pgoff_t index, 1860 struct folio **foliop, enum sgp_type sgp, 1861 gfp_t gfp, struct mm_struct *fault_mm, 1862 vm_fault_t *fault_type) 1863 { 1864 struct address_space *mapping = inode->i_mapping; 1865 struct shmem_inode_info *info = SHMEM_I(inode); 1866 struct swap_info_struct *si; 1867 struct folio *folio = NULL; 1868 swp_entry_t swap; 1869 int error; 1870 1871 VM_BUG_ON(!*foliop || !xa_is_value(*foliop)); 1872 swap = radix_to_swp_entry(*foliop); 1873 *foliop = NULL; 1874 1875 if (is_poisoned_swp_entry(swap)) 1876 return -EIO; 1877 1878 si = get_swap_device(swap); 1879 if (!si) { 1880 if (!shmem_confirm_swap(mapping, index, swap)) 1881 return -EEXIST; 1882 else 1883 return -EINVAL; 1884 } 1885 1886 /* Look it up and read it in.. */ 1887 folio = swap_cache_get_folio(swap, NULL, 0); 1888 if (!folio) { 1889 /* Or update major stats only when swapin succeeds?? */ 1890 if (fault_type) { 1891 *fault_type |= VM_FAULT_MAJOR; 1892 count_vm_event(PGMAJFAULT); 1893 count_memcg_event_mm(fault_mm, PGMAJFAULT); 1894 } 1895 /* Here we actually start the io */ 1896 folio = shmem_swapin_cluster(swap, gfp, info, index); 1897 if (!folio) { 1898 error = -ENOMEM; 1899 goto failed; 1900 } 1901 } 1902 1903 /* We have to do this with folio locked to prevent races */ 1904 folio_lock(folio); 1905 if (!folio_test_swapcache(folio) || 1906 folio->swap.val != swap.val || 1907 !shmem_confirm_swap(mapping, index, swap)) { 1908 error = -EEXIST; 1909 goto unlock; 1910 } 1911 if (!folio_test_uptodate(folio)) { 1912 error = -EIO; 1913 goto failed; 1914 } 1915 folio_wait_writeback(folio); 1916 1917 /* 1918 * Some architectures may have to restore extra metadata to the 1919 * folio after reading from swap. 1920 */ 1921 arch_swap_restore(folio_swap(swap, folio), folio); 1922 1923 if (shmem_should_replace_folio(folio, gfp)) { 1924 error = shmem_replace_folio(&folio, gfp, info, index); 1925 if (error) 1926 goto failed; 1927 } 1928 1929 error = shmem_add_to_page_cache(folio, mapping, index, 1930 swp_to_radix_entry(swap), gfp); 1931 if (error) 1932 goto failed; 1933 1934 shmem_recalc_inode(inode, 0, -1); 1935 1936 if (sgp == SGP_WRITE) 1937 folio_mark_accessed(folio); 1938 1939 delete_from_swap_cache(folio); 1940 folio_mark_dirty(folio); 1941 swap_free(swap); 1942 put_swap_device(si); 1943 1944 *foliop = folio; 1945 return 0; 1946 failed: 1947 if (!shmem_confirm_swap(mapping, index, swap)) 1948 error = -EEXIST; 1949 if (error == -EIO) 1950 shmem_set_folio_swapin_error(inode, index, folio, swap); 1951 unlock: 1952 if (folio) { 1953 folio_unlock(folio); 1954 folio_put(folio); 1955 } 1956 put_swap_device(si); 1957 1958 return error; 1959 } 1960 1961 /* 1962 * shmem_get_folio_gfp - find page in cache, or get from swap, or allocate 1963 * 1964 * If we allocate a new one we do not mark it dirty. That's up to the 1965 * vm. If we swap it in we mark it dirty since we also free the swap 1966 * entry since a page cannot live in both the swap and page cache. 1967 * 1968 * vmf and fault_type are only supplied by shmem_fault: otherwise they are NULL. 1969 */ 1970 static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index, 1971 struct folio **foliop, enum sgp_type sgp, gfp_t gfp, 1972 struct vm_fault *vmf, vm_fault_t *fault_type) 1973 { 1974 struct vm_area_struct *vma = vmf ? vmf->vma : NULL; 1975 struct mm_struct *fault_mm; 1976 struct folio *folio; 1977 int error; 1978 bool alloced; 1979 1980 if (WARN_ON_ONCE(!shmem_mapping(inode->i_mapping))) 1981 return -EINVAL; 1982 1983 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT)) 1984 return -EFBIG; 1985 repeat: 1986 if (sgp <= SGP_CACHE && 1987 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) 1988 return -EINVAL; 1989 1990 alloced = false; 1991 fault_mm = vma ? vma->vm_mm : NULL; 1992 1993 folio = filemap_get_entry(inode->i_mapping, index); 1994 if (folio && vma && userfaultfd_minor(vma)) { 1995 if (!xa_is_value(folio)) 1996 folio_put(folio); 1997 *fault_type = handle_userfault(vmf, VM_UFFD_MINOR); 1998 return 0; 1999 } 2000 2001 if (xa_is_value(folio)) { 2002 error = shmem_swapin_folio(inode, index, &folio, 2003 sgp, gfp, fault_mm, fault_type); 2004 if (error == -EEXIST) 2005 goto repeat; 2006 2007 *foliop = folio; 2008 return error; 2009 } 2010 2011 if (folio) { 2012 folio_lock(folio); 2013 2014 /* Has the folio been truncated or swapped out? */ 2015 if (unlikely(folio->mapping != inode->i_mapping)) { 2016 folio_unlock(folio); 2017 folio_put(folio); 2018 goto repeat; 2019 } 2020 if (sgp == SGP_WRITE) 2021 folio_mark_accessed(folio); 2022 if (folio_test_uptodate(folio)) 2023 goto out; 2024 /* fallocated folio */ 2025 if (sgp != SGP_READ) 2026 goto clear; 2027 folio_unlock(folio); 2028 folio_put(folio); 2029 } 2030 2031 /* 2032 * SGP_READ: succeed on hole, with NULL folio, letting caller zero. 2033 * SGP_NOALLOC: fail on hole, with NULL folio, letting caller fail. 2034 */ 2035 *foliop = NULL; 2036 if (sgp == SGP_READ) 2037 return 0; 2038 if (sgp == SGP_NOALLOC) 2039 return -ENOENT; 2040 2041 /* 2042 * Fast cache lookup and swap lookup did not find it: allocate. 2043 */ 2044 2045 if (vma && userfaultfd_missing(vma)) { 2046 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING); 2047 return 0; 2048 } 2049 2050 if (shmem_is_huge(inode, index, false, fault_mm, 2051 vma ? vma->vm_flags : 0)) { 2052 gfp_t huge_gfp; 2053 2054 huge_gfp = vma_thp_gfp_mask(vma); 2055 huge_gfp = limit_gfp_mask(huge_gfp, gfp); 2056 folio = shmem_alloc_and_add_folio(huge_gfp, 2057 inode, index, fault_mm, true); 2058 if (!IS_ERR(folio)) { 2059 count_vm_event(THP_FILE_ALLOC); 2060 goto alloced; 2061 } 2062 if (PTR_ERR(folio) == -EEXIST) 2063 goto repeat; 2064 } 2065 2066 folio = shmem_alloc_and_add_folio(gfp, inode, index, fault_mm, false); 2067 if (IS_ERR(folio)) { 2068 error = PTR_ERR(folio); 2069 if (error == -EEXIST) 2070 goto repeat; 2071 folio = NULL; 2072 goto unlock; 2073 } 2074 2075 alloced: 2076 alloced = true; 2077 if (folio_test_pmd_mappable(folio) && 2078 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < 2079 folio_next_index(folio) - 1) { 2080 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 2081 struct shmem_inode_info *info = SHMEM_I(inode); 2082 /* 2083 * Part of the large folio is beyond i_size: subject 2084 * to shrink under memory pressure. 2085 */ 2086 spin_lock(&sbinfo->shrinklist_lock); 2087 /* 2088 * _careful to defend against unlocked access to 2089 * ->shrink_list in shmem_unused_huge_shrink() 2090 */ 2091 if (list_empty_careful(&info->shrinklist)) { 2092 list_add_tail(&info->shrinklist, 2093 &sbinfo->shrinklist); 2094 sbinfo->shrinklist_len++; 2095 } 2096 spin_unlock(&sbinfo->shrinklist_lock); 2097 } 2098 2099 if (sgp == SGP_WRITE) 2100 folio_set_referenced(folio); 2101 /* 2102 * Let SGP_FALLOC use the SGP_WRITE optimization on a new folio. 2103 */ 2104 if (sgp == SGP_FALLOC) 2105 sgp = SGP_WRITE; 2106 clear: 2107 /* 2108 * Let SGP_WRITE caller clear ends if write does not fill folio; 2109 * but SGP_FALLOC on a folio fallocated earlier must initialize 2110 * it now, lest undo on failure cancel our earlier guarantee. 2111 */ 2112 if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) { 2113 long i, n = folio_nr_pages(folio); 2114 2115 for (i = 0; i < n; i++) 2116 clear_highpage(folio_page(folio, i)); 2117 flush_dcache_folio(folio); 2118 folio_mark_uptodate(folio); 2119 } 2120 2121 /* Perhaps the file has been truncated since we checked */ 2122 if (sgp <= SGP_CACHE && 2123 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 2124 error = -EINVAL; 2125 goto unlock; 2126 } 2127 out: 2128 *foliop = folio; 2129 return 0; 2130 2131 /* 2132 * Error recovery. 2133 */ 2134 unlock: 2135 if (alloced) 2136 filemap_remove_folio(folio); 2137 shmem_recalc_inode(inode, 0, 0); 2138 if (folio) { 2139 folio_unlock(folio); 2140 folio_put(folio); 2141 } 2142 return error; 2143 } 2144 2145 /** 2146 * shmem_get_folio - find, and lock a shmem folio. 2147 * @inode: inode to search 2148 * @index: the page index. 2149 * @foliop: pointer to the folio if found 2150 * @sgp: SGP_* flags to control behavior 2151 * 2152 * Looks up the page cache entry at @inode & @index. If a folio is 2153 * present, it is returned locked with an increased refcount. 2154 * 2155 * If the caller modifies data in the folio, it must call folio_mark_dirty() 2156 * before unlocking the folio to ensure that the folio is not reclaimed. 2157 * There is no need to reserve space before calling folio_mark_dirty(). 2158 * 2159 * When no folio is found, the behavior depends on @sgp: 2160 * - for SGP_READ, *@foliop is %NULL and 0 is returned 2161 * - for SGP_NOALLOC, *@foliop is %NULL and -ENOENT is returned 2162 * - for all other flags a new folio is allocated, inserted into the 2163 * page cache and returned locked in @foliop. 2164 * 2165 * Context: May sleep. 2166 * Return: 0 if successful, else a negative error code. 2167 */ 2168 int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop, 2169 enum sgp_type sgp) 2170 { 2171 return shmem_get_folio_gfp(inode, index, foliop, sgp, 2172 mapping_gfp_mask(inode->i_mapping), NULL, NULL); 2173 } 2174 EXPORT_SYMBOL_GPL(shmem_get_folio); 2175 2176 /* 2177 * This is like autoremove_wake_function, but it removes the wait queue 2178 * entry unconditionally - even if something else had already woken the 2179 * target. 2180 */ 2181 static int synchronous_wake_function(wait_queue_entry_t *wait, 2182 unsigned int mode, int sync, void *key) 2183 { 2184 int ret = default_wake_function(wait, mode, sync, key); 2185 list_del_init(&wait->entry); 2186 return ret; 2187 } 2188 2189 /* 2190 * Trinity finds that probing a hole which tmpfs is punching can 2191 * prevent the hole-punch from ever completing: which in turn 2192 * locks writers out with its hold on i_rwsem. So refrain from 2193 * faulting pages into the hole while it's being punched. Although 2194 * shmem_undo_range() does remove the additions, it may be unable to 2195 * keep up, as each new page needs its own unmap_mapping_range() call, 2196 * and the i_mmap tree grows ever slower to scan if new vmas are added. 2197 * 2198 * It does not matter if we sometimes reach this check just before the 2199 * hole-punch begins, so that one fault then races with the punch: 2200 * we just need to make racing faults a rare case. 2201 * 2202 * The implementation below would be much simpler if we just used a 2203 * standard mutex or completion: but we cannot take i_rwsem in fault, 2204 * and bloating every shmem inode for this unlikely case would be sad. 2205 */ 2206 static vm_fault_t shmem_falloc_wait(struct vm_fault *vmf, struct inode *inode) 2207 { 2208 struct shmem_falloc *shmem_falloc; 2209 struct file *fpin = NULL; 2210 vm_fault_t ret = 0; 2211 2212 spin_lock(&inode->i_lock); 2213 shmem_falloc = inode->i_private; 2214 if (shmem_falloc && 2215 shmem_falloc->waitq && 2216 vmf->pgoff >= shmem_falloc->start && 2217 vmf->pgoff < shmem_falloc->next) { 2218 wait_queue_head_t *shmem_falloc_waitq; 2219 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function); 2220 2221 ret = VM_FAULT_NOPAGE; 2222 fpin = maybe_unlock_mmap_for_io(vmf, NULL); 2223 shmem_falloc_waitq = shmem_falloc->waitq; 2224 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait, 2225 TASK_UNINTERRUPTIBLE); 2226 spin_unlock(&inode->i_lock); 2227 schedule(); 2228 2229 /* 2230 * shmem_falloc_waitq points into the shmem_fallocate() 2231 * stack of the hole-punching task: shmem_falloc_waitq 2232 * is usually invalid by the time we reach here, but 2233 * finish_wait() does not dereference it in that case; 2234 * though i_lock needed lest racing with wake_up_all(). 2235 */ 2236 spin_lock(&inode->i_lock); 2237 finish_wait(shmem_falloc_waitq, &shmem_fault_wait); 2238 } 2239 spin_unlock(&inode->i_lock); 2240 if (fpin) { 2241 fput(fpin); 2242 ret = VM_FAULT_RETRY; 2243 } 2244 return ret; 2245 } 2246 2247 static vm_fault_t shmem_fault(struct vm_fault *vmf) 2248 { 2249 struct inode *inode = file_inode(vmf->vma->vm_file); 2250 gfp_t gfp = mapping_gfp_mask(inode->i_mapping); 2251 struct folio *folio = NULL; 2252 vm_fault_t ret = 0; 2253 int err; 2254 2255 /* 2256 * Trinity finds that probing a hole which tmpfs is punching can 2257 * prevent the hole-punch from ever completing: noted in i_private. 2258 */ 2259 if (unlikely(inode->i_private)) { 2260 ret = shmem_falloc_wait(vmf, inode); 2261 if (ret) 2262 return ret; 2263 } 2264 2265 WARN_ON_ONCE(vmf->page != NULL); 2266 err = shmem_get_folio_gfp(inode, vmf->pgoff, &folio, SGP_CACHE, 2267 gfp, vmf, &ret); 2268 if (err) 2269 return vmf_error(err); 2270 if (folio) { 2271 vmf->page = folio_file_page(folio, vmf->pgoff); 2272 ret |= VM_FAULT_LOCKED; 2273 } 2274 return ret; 2275 } 2276 2277 unsigned long shmem_get_unmapped_area(struct file *file, 2278 unsigned long uaddr, unsigned long len, 2279 unsigned long pgoff, unsigned long flags) 2280 { 2281 unsigned long addr; 2282 unsigned long offset; 2283 unsigned long inflated_len; 2284 unsigned long inflated_addr; 2285 unsigned long inflated_offset; 2286 2287 if (len > TASK_SIZE) 2288 return -ENOMEM; 2289 2290 addr = mm_get_unmapped_area(current->mm, file, uaddr, len, pgoff, 2291 flags); 2292 2293 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 2294 return addr; 2295 if (IS_ERR_VALUE(addr)) 2296 return addr; 2297 if (addr & ~PAGE_MASK) 2298 return addr; 2299 if (addr > TASK_SIZE - len) 2300 return addr; 2301 2302 if (shmem_huge == SHMEM_HUGE_DENY) 2303 return addr; 2304 if (len < HPAGE_PMD_SIZE) 2305 return addr; 2306 if (flags & MAP_FIXED) 2307 return addr; 2308 /* 2309 * Our priority is to support MAP_SHARED mapped hugely; 2310 * and support MAP_PRIVATE mapped hugely too, until it is COWed. 2311 * But if caller specified an address hint and we allocated area there 2312 * successfully, respect that as before. 2313 */ 2314 if (uaddr == addr) 2315 return addr; 2316 2317 if (shmem_huge != SHMEM_HUGE_FORCE) { 2318 struct super_block *sb; 2319 2320 if (file) { 2321 VM_BUG_ON(file->f_op != &shmem_file_operations); 2322 sb = file_inode(file)->i_sb; 2323 } else { 2324 /* 2325 * Called directly from mm/mmap.c, or drivers/char/mem.c 2326 * for "/dev/zero", to create a shared anonymous object. 2327 */ 2328 if (IS_ERR(shm_mnt)) 2329 return addr; 2330 sb = shm_mnt->mnt_sb; 2331 } 2332 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER) 2333 return addr; 2334 } 2335 2336 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1); 2337 if (offset && offset + len < 2 * HPAGE_PMD_SIZE) 2338 return addr; 2339 if ((addr & (HPAGE_PMD_SIZE-1)) == offset) 2340 return addr; 2341 2342 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE; 2343 if (inflated_len > TASK_SIZE) 2344 return addr; 2345 if (inflated_len < len) 2346 return addr; 2347 2348 inflated_addr = mm_get_unmapped_area(current->mm, NULL, uaddr, 2349 inflated_len, 0, flags); 2350 if (IS_ERR_VALUE(inflated_addr)) 2351 return addr; 2352 if (inflated_addr & ~PAGE_MASK) 2353 return addr; 2354 2355 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1); 2356 inflated_addr += offset - inflated_offset; 2357 if (inflated_offset > offset) 2358 inflated_addr += HPAGE_PMD_SIZE; 2359 2360 if (inflated_addr > TASK_SIZE - len) 2361 return addr; 2362 return inflated_addr; 2363 } 2364 2365 #ifdef CONFIG_NUMA 2366 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) 2367 { 2368 struct inode *inode = file_inode(vma->vm_file); 2369 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); 2370 } 2371 2372 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 2373 unsigned long addr, pgoff_t *ilx) 2374 { 2375 struct inode *inode = file_inode(vma->vm_file); 2376 pgoff_t index; 2377 2378 /* 2379 * Bias interleave by inode number to distribute better across nodes; 2380 * but this interface is independent of which page order is used, so 2381 * supplies only that bias, letting caller apply the offset (adjusted 2382 * by page order, as in shmem_get_pgoff_policy() and get_vma_policy()). 2383 */ 2384 *ilx = inode->i_ino; 2385 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 2386 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); 2387 } 2388 2389 static struct mempolicy *shmem_get_pgoff_policy(struct shmem_inode_info *info, 2390 pgoff_t index, unsigned int order, pgoff_t *ilx) 2391 { 2392 struct mempolicy *mpol; 2393 2394 /* Bias interleave by inode number to distribute better across nodes */ 2395 *ilx = info->vfs_inode.i_ino + (index >> order); 2396 2397 mpol = mpol_shared_policy_lookup(&info->policy, index); 2398 return mpol ? mpol : get_task_policy(current); 2399 } 2400 #else 2401 static struct mempolicy *shmem_get_pgoff_policy(struct shmem_inode_info *info, 2402 pgoff_t index, unsigned int order, pgoff_t *ilx) 2403 { 2404 *ilx = 0; 2405 return NULL; 2406 } 2407 #endif /* CONFIG_NUMA */ 2408 2409 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts) 2410 { 2411 struct inode *inode = file_inode(file); 2412 struct shmem_inode_info *info = SHMEM_I(inode); 2413 int retval = -ENOMEM; 2414 2415 /* 2416 * What serializes the accesses to info->flags? 2417 * ipc_lock_object() when called from shmctl_do_lock(), 2418 * no serialization needed when called from shm_destroy(). 2419 */ 2420 if (lock && !(info->flags & VM_LOCKED)) { 2421 if (!user_shm_lock(inode->i_size, ucounts)) 2422 goto out_nomem; 2423 info->flags |= VM_LOCKED; 2424 mapping_set_unevictable(file->f_mapping); 2425 } 2426 if (!lock && (info->flags & VM_LOCKED) && ucounts) { 2427 user_shm_unlock(inode->i_size, ucounts); 2428 info->flags &= ~VM_LOCKED; 2429 mapping_clear_unevictable(file->f_mapping); 2430 } 2431 retval = 0; 2432 2433 out_nomem: 2434 return retval; 2435 } 2436 2437 static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 2438 { 2439 struct inode *inode = file_inode(file); 2440 struct shmem_inode_info *info = SHMEM_I(inode); 2441 int ret; 2442 2443 ret = seal_check_write(info->seals, vma); 2444 if (ret) 2445 return ret; 2446 2447 /* arm64 - allow memory tagging on RAM-based files */ 2448 vm_flags_set(vma, VM_MTE_ALLOWED); 2449 2450 file_accessed(file); 2451 /* This is anonymous shared memory if it is unlinked at the time of mmap */ 2452 if (inode->i_nlink) 2453 vma->vm_ops = &shmem_vm_ops; 2454 else 2455 vma->vm_ops = &shmem_anon_vm_ops; 2456 return 0; 2457 } 2458 2459 static int shmem_file_open(struct inode *inode, struct file *file) 2460 { 2461 file->f_mode |= FMODE_CAN_ODIRECT; 2462 return generic_file_open(inode, file); 2463 } 2464 2465 #ifdef CONFIG_TMPFS_XATTR 2466 static int shmem_initxattrs(struct inode *, const struct xattr *, void *); 2467 2468 /* 2469 * chattr's fsflags are unrelated to extended attributes, 2470 * but tmpfs has chosen to enable them under the same config option. 2471 */ 2472 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags) 2473 { 2474 unsigned int i_flags = 0; 2475 2476 if (fsflags & FS_NOATIME_FL) 2477 i_flags |= S_NOATIME; 2478 if (fsflags & FS_APPEND_FL) 2479 i_flags |= S_APPEND; 2480 if (fsflags & FS_IMMUTABLE_FL) 2481 i_flags |= S_IMMUTABLE; 2482 /* 2483 * But FS_NODUMP_FL does not require any action in i_flags. 2484 */ 2485 inode_set_flags(inode, i_flags, S_NOATIME | S_APPEND | S_IMMUTABLE); 2486 } 2487 #else 2488 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags) 2489 { 2490 } 2491 #define shmem_initxattrs NULL 2492 #endif 2493 2494 static struct offset_ctx *shmem_get_offset_ctx(struct inode *inode) 2495 { 2496 return &SHMEM_I(inode)->dir_offsets; 2497 } 2498 2499 static struct inode *__shmem_get_inode(struct mnt_idmap *idmap, 2500 struct super_block *sb, 2501 struct inode *dir, umode_t mode, 2502 dev_t dev, unsigned long flags) 2503 { 2504 struct inode *inode; 2505 struct shmem_inode_info *info; 2506 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2507 ino_t ino; 2508 int err; 2509 2510 err = shmem_reserve_inode(sb, &ino); 2511 if (err) 2512 return ERR_PTR(err); 2513 2514 inode = new_inode(sb); 2515 if (!inode) { 2516 shmem_free_inode(sb, 0); 2517 return ERR_PTR(-ENOSPC); 2518 } 2519 2520 inode->i_ino = ino; 2521 inode_init_owner(idmap, inode, dir, mode); 2522 inode->i_blocks = 0; 2523 simple_inode_init_ts(inode); 2524 inode->i_generation = get_random_u32(); 2525 info = SHMEM_I(inode); 2526 memset(info, 0, (char *)inode - (char *)info); 2527 spin_lock_init(&info->lock); 2528 atomic_set(&info->stop_eviction, 0); 2529 info->seals = F_SEAL_SEAL; 2530 info->flags = flags & VM_NORESERVE; 2531 info->i_crtime = inode_get_mtime(inode); 2532 info->fsflags = (dir == NULL) ? 0 : 2533 SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED; 2534 if (info->fsflags) 2535 shmem_set_inode_flags(inode, info->fsflags); 2536 INIT_LIST_HEAD(&info->shrinklist); 2537 INIT_LIST_HEAD(&info->swaplist); 2538 simple_xattrs_init(&info->xattrs); 2539 cache_no_acl(inode); 2540 if (sbinfo->noswap) 2541 mapping_set_unevictable(inode->i_mapping); 2542 mapping_set_large_folios(inode->i_mapping); 2543 2544 switch (mode & S_IFMT) { 2545 default: 2546 inode->i_op = &shmem_special_inode_operations; 2547 init_special_inode(inode, mode, dev); 2548 break; 2549 case S_IFREG: 2550 inode->i_mapping->a_ops = &shmem_aops; 2551 inode->i_op = &shmem_inode_operations; 2552 inode->i_fop = &shmem_file_operations; 2553 mpol_shared_policy_init(&info->policy, 2554 shmem_get_sbmpol(sbinfo)); 2555 break; 2556 case S_IFDIR: 2557 inc_nlink(inode); 2558 /* Some things misbehave if size == 0 on a directory */ 2559 inode->i_size = 2 * BOGO_DIRENT_SIZE; 2560 inode->i_op = &shmem_dir_inode_operations; 2561 inode->i_fop = &simple_offset_dir_operations; 2562 simple_offset_init(shmem_get_offset_ctx(inode)); 2563 break; 2564 case S_IFLNK: 2565 /* 2566 * Must not load anything in the rbtree, 2567 * mpol_free_shared_policy will not be called. 2568 */ 2569 mpol_shared_policy_init(&info->policy, NULL); 2570 break; 2571 } 2572 2573 lockdep_annotate_inode_mutex_key(inode); 2574 return inode; 2575 } 2576 2577 #ifdef CONFIG_TMPFS_QUOTA 2578 static struct inode *shmem_get_inode(struct mnt_idmap *idmap, 2579 struct super_block *sb, struct inode *dir, 2580 umode_t mode, dev_t dev, unsigned long flags) 2581 { 2582 int err; 2583 struct inode *inode; 2584 2585 inode = __shmem_get_inode(idmap, sb, dir, mode, dev, flags); 2586 if (IS_ERR(inode)) 2587 return inode; 2588 2589 err = dquot_initialize(inode); 2590 if (err) 2591 goto errout; 2592 2593 err = dquot_alloc_inode(inode); 2594 if (err) { 2595 dquot_drop(inode); 2596 goto errout; 2597 } 2598 return inode; 2599 2600 errout: 2601 inode->i_flags |= S_NOQUOTA; 2602 iput(inode); 2603 return ERR_PTR(err); 2604 } 2605 #else 2606 static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap, 2607 struct super_block *sb, struct inode *dir, 2608 umode_t mode, dev_t dev, unsigned long flags) 2609 { 2610 return __shmem_get_inode(idmap, sb, dir, mode, dev, flags); 2611 } 2612 #endif /* CONFIG_TMPFS_QUOTA */ 2613 2614 #ifdef CONFIG_USERFAULTFD 2615 int shmem_mfill_atomic_pte(pmd_t *dst_pmd, 2616 struct vm_area_struct *dst_vma, 2617 unsigned long dst_addr, 2618 unsigned long src_addr, 2619 uffd_flags_t flags, 2620 struct folio **foliop) 2621 { 2622 struct inode *inode = file_inode(dst_vma->vm_file); 2623 struct shmem_inode_info *info = SHMEM_I(inode); 2624 struct address_space *mapping = inode->i_mapping; 2625 gfp_t gfp = mapping_gfp_mask(mapping); 2626 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); 2627 void *page_kaddr; 2628 struct folio *folio; 2629 int ret; 2630 pgoff_t max_off; 2631 2632 if (shmem_inode_acct_blocks(inode, 1)) { 2633 /* 2634 * We may have got a page, returned -ENOENT triggering a retry, 2635 * and now we find ourselves with -ENOMEM. Release the page, to 2636 * avoid a BUG_ON in our caller. 2637 */ 2638 if (unlikely(*foliop)) { 2639 folio_put(*foliop); 2640 *foliop = NULL; 2641 } 2642 return -ENOMEM; 2643 } 2644 2645 if (!*foliop) { 2646 ret = -ENOMEM; 2647 folio = shmem_alloc_folio(gfp, info, pgoff); 2648 if (!folio) 2649 goto out_unacct_blocks; 2650 2651 if (uffd_flags_mode_is(flags, MFILL_ATOMIC_COPY)) { 2652 page_kaddr = kmap_local_folio(folio, 0); 2653 /* 2654 * The read mmap_lock is held here. Despite the 2655 * mmap_lock being read recursive a deadlock is still 2656 * possible if a writer has taken a lock. For example: 2657 * 2658 * process A thread 1 takes read lock on own mmap_lock 2659 * process A thread 2 calls mmap, blocks taking write lock 2660 * process B thread 1 takes page fault, read lock on own mmap lock 2661 * process B thread 2 calls mmap, blocks taking write lock 2662 * process A thread 1 blocks taking read lock on process B 2663 * process B thread 1 blocks taking read lock on process A 2664 * 2665 * Disable page faults to prevent potential deadlock 2666 * and retry the copy outside the mmap_lock. 2667 */ 2668 pagefault_disable(); 2669 ret = copy_from_user(page_kaddr, 2670 (const void __user *)src_addr, 2671 PAGE_SIZE); 2672 pagefault_enable(); 2673 kunmap_local(page_kaddr); 2674 2675 /* fallback to copy_from_user outside mmap_lock */ 2676 if (unlikely(ret)) { 2677 *foliop = folio; 2678 ret = -ENOENT; 2679 /* don't free the page */ 2680 goto out_unacct_blocks; 2681 } 2682 2683 flush_dcache_folio(folio); 2684 } else { /* ZEROPAGE */ 2685 clear_user_highpage(&folio->page, dst_addr); 2686 } 2687 } else { 2688 folio = *foliop; 2689 VM_BUG_ON_FOLIO(folio_test_large(folio), folio); 2690 *foliop = NULL; 2691 } 2692 2693 VM_BUG_ON(folio_test_locked(folio)); 2694 VM_BUG_ON(folio_test_swapbacked(folio)); 2695 __folio_set_locked(folio); 2696 __folio_set_swapbacked(folio); 2697 __folio_mark_uptodate(folio); 2698 2699 ret = -EFAULT; 2700 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 2701 if (unlikely(pgoff >= max_off)) 2702 goto out_release; 2703 2704 ret = mem_cgroup_charge(folio, dst_vma->vm_mm, gfp); 2705 if (ret) 2706 goto out_release; 2707 ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL, gfp); 2708 if (ret) 2709 goto out_release; 2710 2711 ret = mfill_atomic_install_pte(dst_pmd, dst_vma, dst_addr, 2712 &folio->page, true, flags); 2713 if (ret) 2714 goto out_delete_from_cache; 2715 2716 shmem_recalc_inode(inode, 1, 0); 2717 folio_unlock(folio); 2718 return 0; 2719 out_delete_from_cache: 2720 filemap_remove_folio(folio); 2721 out_release: 2722 folio_unlock(folio); 2723 folio_put(folio); 2724 out_unacct_blocks: 2725 shmem_inode_unacct_blocks(inode, 1); 2726 return ret; 2727 } 2728 #endif /* CONFIG_USERFAULTFD */ 2729 2730 #ifdef CONFIG_TMPFS 2731 static const struct inode_operations shmem_symlink_inode_operations; 2732 static const struct inode_operations shmem_short_symlink_operations; 2733 2734 static int 2735 shmem_write_begin(struct file *file, struct address_space *mapping, 2736 loff_t pos, unsigned len, 2737 struct page **pagep, void **fsdata) 2738 { 2739 struct inode *inode = mapping->host; 2740 struct shmem_inode_info *info = SHMEM_I(inode); 2741 pgoff_t index = pos >> PAGE_SHIFT; 2742 struct folio *folio; 2743 int ret = 0; 2744 2745 /* i_rwsem is held by caller */ 2746 if (unlikely(info->seals & (F_SEAL_GROW | 2747 F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) { 2748 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) 2749 return -EPERM; 2750 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size) 2751 return -EPERM; 2752 } 2753 2754 ret = shmem_get_folio(inode, index, &folio, SGP_WRITE); 2755 if (ret) 2756 return ret; 2757 2758 *pagep = folio_file_page(folio, index); 2759 if (PageHWPoison(*pagep)) { 2760 folio_unlock(folio); 2761 folio_put(folio); 2762 *pagep = NULL; 2763 return -EIO; 2764 } 2765 2766 return 0; 2767 } 2768 2769 static int 2770 shmem_write_end(struct file *file, struct address_space *mapping, 2771 loff_t pos, unsigned len, unsigned copied, 2772 struct page *page, void *fsdata) 2773 { 2774 struct folio *folio = page_folio(page); 2775 struct inode *inode = mapping->host; 2776 2777 if (pos + copied > inode->i_size) 2778 i_size_write(inode, pos + copied); 2779 2780 if (!folio_test_uptodate(folio)) { 2781 if (copied < folio_size(folio)) { 2782 size_t from = offset_in_folio(folio, pos); 2783 folio_zero_segments(folio, 0, from, 2784 from + copied, folio_size(folio)); 2785 } 2786 folio_mark_uptodate(folio); 2787 } 2788 folio_mark_dirty(folio); 2789 folio_unlock(folio); 2790 folio_put(folio); 2791 2792 return copied; 2793 } 2794 2795 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 2796 { 2797 struct file *file = iocb->ki_filp; 2798 struct inode *inode = file_inode(file); 2799 struct address_space *mapping = inode->i_mapping; 2800 pgoff_t index; 2801 unsigned long offset; 2802 int error = 0; 2803 ssize_t retval = 0; 2804 loff_t *ppos = &iocb->ki_pos; 2805 2806 index = *ppos >> PAGE_SHIFT; 2807 offset = *ppos & ~PAGE_MASK; 2808 2809 for (;;) { 2810 struct folio *folio = NULL; 2811 struct page *page = NULL; 2812 pgoff_t end_index; 2813 unsigned long nr, ret; 2814 loff_t i_size = i_size_read(inode); 2815 2816 end_index = i_size >> PAGE_SHIFT; 2817 if (index > end_index) 2818 break; 2819 if (index == end_index) { 2820 nr = i_size & ~PAGE_MASK; 2821 if (nr <= offset) 2822 break; 2823 } 2824 2825 error = shmem_get_folio(inode, index, &folio, SGP_READ); 2826 if (error) { 2827 if (error == -EINVAL) 2828 error = 0; 2829 break; 2830 } 2831 if (folio) { 2832 folio_unlock(folio); 2833 2834 page = folio_file_page(folio, index); 2835 if (PageHWPoison(page)) { 2836 folio_put(folio); 2837 error = -EIO; 2838 break; 2839 } 2840 } 2841 2842 /* 2843 * We must evaluate after, since reads (unlike writes) 2844 * are called without i_rwsem protection against truncate 2845 */ 2846 nr = PAGE_SIZE; 2847 i_size = i_size_read(inode); 2848 end_index = i_size >> PAGE_SHIFT; 2849 if (index == end_index) { 2850 nr = i_size & ~PAGE_MASK; 2851 if (nr <= offset) { 2852 if (folio) 2853 folio_put(folio); 2854 break; 2855 } 2856 } 2857 nr -= offset; 2858 2859 if (folio) { 2860 /* 2861 * If users can be writing to this page using arbitrary 2862 * virtual addresses, take care about potential aliasing 2863 * before reading the page on the kernel side. 2864 */ 2865 if (mapping_writably_mapped(mapping)) 2866 flush_dcache_page(page); 2867 /* 2868 * Mark the page accessed if we read the beginning. 2869 */ 2870 if (!offset) 2871 folio_mark_accessed(folio); 2872 /* 2873 * Ok, we have the page, and it's up-to-date, so 2874 * now we can copy it to user space... 2875 */ 2876 ret = copy_page_to_iter(page, offset, nr, to); 2877 folio_put(folio); 2878 2879 } else if (user_backed_iter(to)) { 2880 /* 2881 * Copy to user tends to be so well optimized, but 2882 * clear_user() not so much, that it is noticeably 2883 * faster to copy the zero page instead of clearing. 2884 */ 2885 ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to); 2886 } else { 2887 /* 2888 * But submitting the same page twice in a row to 2889 * splice() - or others? - can result in confusion: 2890 * so don't attempt that optimization on pipes etc. 2891 */ 2892 ret = iov_iter_zero(nr, to); 2893 } 2894 2895 retval += ret; 2896 offset += ret; 2897 index += offset >> PAGE_SHIFT; 2898 offset &= ~PAGE_MASK; 2899 2900 if (!iov_iter_count(to)) 2901 break; 2902 if (ret < nr) { 2903 error = -EFAULT; 2904 break; 2905 } 2906 cond_resched(); 2907 } 2908 2909 *ppos = ((loff_t) index << PAGE_SHIFT) + offset; 2910 file_accessed(file); 2911 return retval ? retval : error; 2912 } 2913 2914 static ssize_t shmem_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 2915 { 2916 struct file *file = iocb->ki_filp; 2917 struct inode *inode = file->f_mapping->host; 2918 ssize_t ret; 2919 2920 inode_lock(inode); 2921 ret = generic_write_checks(iocb, from); 2922 if (ret <= 0) 2923 goto unlock; 2924 ret = file_remove_privs(file); 2925 if (ret) 2926 goto unlock; 2927 ret = file_update_time(file); 2928 if (ret) 2929 goto unlock; 2930 ret = generic_perform_write(iocb, from); 2931 unlock: 2932 inode_unlock(inode); 2933 return ret; 2934 } 2935 2936 static bool zero_pipe_buf_get(struct pipe_inode_info *pipe, 2937 struct pipe_buffer *buf) 2938 { 2939 return true; 2940 } 2941 2942 static void zero_pipe_buf_release(struct pipe_inode_info *pipe, 2943 struct pipe_buffer *buf) 2944 { 2945 } 2946 2947 static bool zero_pipe_buf_try_steal(struct pipe_inode_info *pipe, 2948 struct pipe_buffer *buf) 2949 { 2950 return false; 2951 } 2952 2953 static const struct pipe_buf_operations zero_pipe_buf_ops = { 2954 .release = zero_pipe_buf_release, 2955 .try_steal = zero_pipe_buf_try_steal, 2956 .get = zero_pipe_buf_get, 2957 }; 2958 2959 static size_t splice_zeropage_into_pipe(struct pipe_inode_info *pipe, 2960 loff_t fpos, size_t size) 2961 { 2962 size_t offset = fpos & ~PAGE_MASK; 2963 2964 size = min_t(size_t, size, PAGE_SIZE - offset); 2965 2966 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage)) { 2967 struct pipe_buffer *buf = pipe_head_buf(pipe); 2968 2969 *buf = (struct pipe_buffer) { 2970 .ops = &zero_pipe_buf_ops, 2971 .page = ZERO_PAGE(0), 2972 .offset = offset, 2973 .len = size, 2974 }; 2975 pipe->head++; 2976 } 2977 2978 return size; 2979 } 2980 2981 static ssize_t shmem_file_splice_read(struct file *in, loff_t *ppos, 2982 struct pipe_inode_info *pipe, 2983 size_t len, unsigned int flags) 2984 { 2985 struct inode *inode = file_inode(in); 2986 struct address_space *mapping = inode->i_mapping; 2987 struct folio *folio = NULL; 2988 size_t total_spliced = 0, used, npages, n, part; 2989 loff_t isize; 2990 int error = 0; 2991 2992 /* Work out how much data we can actually add into the pipe */ 2993 used = pipe_occupancy(pipe->head, pipe->tail); 2994 npages = max_t(ssize_t, pipe->max_usage - used, 0); 2995 len = min_t(size_t, len, npages * PAGE_SIZE); 2996 2997 do { 2998 if (*ppos >= i_size_read(inode)) 2999 break; 3000 3001 error = shmem_get_folio(inode, *ppos / PAGE_SIZE, &folio, 3002 SGP_READ); 3003 if (error) { 3004 if (error == -EINVAL) 3005 error = 0; 3006 break; 3007 } 3008 if (folio) { 3009 folio_unlock(folio); 3010 3011 if (folio_test_hwpoison(folio) || 3012 (folio_test_large(folio) && 3013 folio_test_has_hwpoisoned(folio))) { 3014 error = -EIO; 3015 break; 3016 } 3017 } 3018 3019 /* 3020 * i_size must be checked after we know the pages are Uptodate. 3021 * 3022 * Checking i_size after the check allows us to calculate 3023 * the correct value for "nr", which means the zero-filled 3024 * part of the page is not copied back to userspace (unless 3025 * another truncate extends the file - this is desired though). 3026 */ 3027 isize = i_size_read(inode); 3028 if (unlikely(*ppos >= isize)) 3029 break; 3030 part = min_t(loff_t, isize - *ppos, len); 3031 3032 if (folio) { 3033 /* 3034 * If users can be writing to this page using arbitrary 3035 * virtual addresses, take care about potential aliasing 3036 * before reading the page on the kernel side. 3037 */ 3038 if (mapping_writably_mapped(mapping)) 3039 flush_dcache_folio(folio); 3040 folio_mark_accessed(folio); 3041 /* 3042 * Ok, we have the page, and it's up-to-date, so we can 3043 * now splice it into the pipe. 3044 */ 3045 n = splice_folio_into_pipe(pipe, folio, *ppos, part); 3046 folio_put(folio); 3047 folio = NULL; 3048 } else { 3049 n = splice_zeropage_into_pipe(pipe, *ppos, part); 3050 } 3051 3052 if (!n) 3053 break; 3054 len -= n; 3055 total_spliced += n; 3056 *ppos += n; 3057 in->f_ra.prev_pos = *ppos; 3058 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) 3059 break; 3060 3061 cond_resched(); 3062 } while (len); 3063 3064 if (folio) 3065 folio_put(folio); 3066 3067 file_accessed(in); 3068 return total_spliced ? total_spliced : error; 3069 } 3070 3071 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence) 3072 { 3073 struct address_space *mapping = file->f_mapping; 3074 struct inode *inode = mapping->host; 3075 3076 if (whence != SEEK_DATA && whence != SEEK_HOLE) 3077 return generic_file_llseek_size(file, offset, whence, 3078 MAX_LFS_FILESIZE, i_size_read(inode)); 3079 if (offset < 0) 3080 return -ENXIO; 3081 3082 inode_lock(inode); 3083 /* We're holding i_rwsem so we can access i_size directly */ 3084 offset = mapping_seek_hole_data(mapping, offset, inode->i_size, whence); 3085 if (offset >= 0) 3086 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE); 3087 inode_unlock(inode); 3088 return offset; 3089 } 3090 3091 static long shmem_fallocate(struct file *file, int mode, loff_t offset, 3092 loff_t len) 3093 { 3094 struct inode *inode = file_inode(file); 3095 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 3096 struct shmem_inode_info *info = SHMEM_I(inode); 3097 struct shmem_falloc shmem_falloc; 3098 pgoff_t start, index, end, undo_fallocend; 3099 int error; 3100 3101 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) 3102 return -EOPNOTSUPP; 3103 3104 inode_lock(inode); 3105 3106 if (mode & FALLOC_FL_PUNCH_HOLE) { 3107 struct address_space *mapping = file->f_mapping; 3108 loff_t unmap_start = round_up(offset, PAGE_SIZE); 3109 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1; 3110 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq); 3111 3112 /* protected by i_rwsem */ 3113 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) { 3114 error = -EPERM; 3115 goto out; 3116 } 3117 3118 shmem_falloc.waitq = &shmem_falloc_waitq; 3119 shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT; 3120 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; 3121 spin_lock(&inode->i_lock); 3122 inode->i_private = &shmem_falloc; 3123 spin_unlock(&inode->i_lock); 3124 3125 if ((u64)unmap_end > (u64)unmap_start) 3126 unmap_mapping_range(mapping, unmap_start, 3127 1 + unmap_end - unmap_start, 0); 3128 shmem_truncate_range(inode, offset, offset + len - 1); 3129 /* No need to unmap again: hole-punching leaves COWed pages */ 3130 3131 spin_lock(&inode->i_lock); 3132 inode->i_private = NULL; 3133 wake_up_all(&shmem_falloc_waitq); 3134 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head)); 3135 spin_unlock(&inode->i_lock); 3136 error = 0; 3137 goto out; 3138 } 3139 3140 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */ 3141 error = inode_newsize_ok(inode, offset + len); 3142 if (error) 3143 goto out; 3144 3145 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) { 3146 error = -EPERM; 3147 goto out; 3148 } 3149 3150 start = offset >> PAGE_SHIFT; 3151 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 3152 /* Try to avoid a swapstorm if len is impossible to satisfy */ 3153 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) { 3154 error = -ENOSPC; 3155 goto out; 3156 } 3157 3158 shmem_falloc.waitq = NULL; 3159 shmem_falloc.start = start; 3160 shmem_falloc.next = start; 3161 shmem_falloc.nr_falloced = 0; 3162 shmem_falloc.nr_unswapped = 0; 3163 spin_lock(&inode->i_lock); 3164 inode->i_private = &shmem_falloc; 3165 spin_unlock(&inode->i_lock); 3166 3167 /* 3168 * info->fallocend is only relevant when huge pages might be 3169 * involved: to prevent split_huge_page() freeing fallocated 3170 * pages when FALLOC_FL_KEEP_SIZE committed beyond i_size. 3171 */ 3172 undo_fallocend = info->fallocend; 3173 if (info->fallocend < end) 3174 info->fallocend = end; 3175 3176 for (index = start; index < end; ) { 3177 struct folio *folio; 3178 3179 /* 3180 * Good, the fallocate(2) manpage permits EINTR: we may have 3181 * been interrupted because we are using up too much memory. 3182 */ 3183 if (signal_pending(current)) 3184 error = -EINTR; 3185 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) 3186 error = -ENOMEM; 3187 else 3188 error = shmem_get_folio(inode, index, &folio, 3189 SGP_FALLOC); 3190 if (error) { 3191 info->fallocend = undo_fallocend; 3192 /* Remove the !uptodate folios we added */ 3193 if (index > start) { 3194 shmem_undo_range(inode, 3195 (loff_t)start << PAGE_SHIFT, 3196 ((loff_t)index << PAGE_SHIFT) - 1, true); 3197 } 3198 goto undone; 3199 } 3200 3201 /* 3202 * Here is a more important optimization than it appears: 3203 * a second SGP_FALLOC on the same large folio will clear it, 3204 * making it uptodate and un-undoable if we fail later. 3205 */ 3206 index = folio_next_index(folio); 3207 /* Beware 32-bit wraparound */ 3208 if (!index) 3209 index--; 3210 3211 /* 3212 * Inform shmem_writepage() how far we have reached. 3213 * No need for lock or barrier: we have the page lock. 3214 */ 3215 if (!folio_test_uptodate(folio)) 3216 shmem_falloc.nr_falloced += index - shmem_falloc.next; 3217 shmem_falloc.next = index; 3218 3219 /* 3220 * If !uptodate, leave it that way so that freeable folios 3221 * can be recognized if we need to rollback on error later. 3222 * But mark it dirty so that memory pressure will swap rather 3223 * than free the folios we are allocating (and SGP_CACHE folios 3224 * might still be clean: we now need to mark those dirty too). 3225 */ 3226 folio_mark_dirty(folio); 3227 folio_unlock(folio); 3228 folio_put(folio); 3229 cond_resched(); 3230 } 3231 3232 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) 3233 i_size_write(inode, offset + len); 3234 undone: 3235 spin_lock(&inode->i_lock); 3236 inode->i_private = NULL; 3237 spin_unlock(&inode->i_lock); 3238 out: 3239 if (!error) 3240 file_modified(file); 3241 inode_unlock(inode); 3242 return error; 3243 } 3244 3245 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 3246 { 3247 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 3248 3249 buf->f_type = TMPFS_MAGIC; 3250 buf->f_bsize = PAGE_SIZE; 3251 buf->f_namelen = NAME_MAX; 3252 if (sbinfo->max_blocks) { 3253 buf->f_blocks = sbinfo->max_blocks; 3254 buf->f_bavail = 3255 buf->f_bfree = sbinfo->max_blocks - 3256 percpu_counter_sum(&sbinfo->used_blocks); 3257 } 3258 if (sbinfo->max_inodes) { 3259 buf->f_files = sbinfo->max_inodes; 3260 buf->f_ffree = sbinfo->free_ispace / BOGO_INODE_SIZE; 3261 } 3262 /* else leave those fields 0 like simple_statfs */ 3263 3264 buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b); 3265 3266 return 0; 3267 } 3268 3269 /* 3270 * File creation. Allocate an inode, and we're done.. 3271 */ 3272 static int 3273 shmem_mknod(struct mnt_idmap *idmap, struct inode *dir, 3274 struct dentry *dentry, umode_t mode, dev_t dev) 3275 { 3276 struct inode *inode; 3277 int error; 3278 3279 inode = shmem_get_inode(idmap, dir->i_sb, dir, mode, dev, VM_NORESERVE); 3280 if (IS_ERR(inode)) 3281 return PTR_ERR(inode); 3282 3283 error = simple_acl_create(dir, inode); 3284 if (error) 3285 goto out_iput; 3286 error = security_inode_init_security(inode, dir, &dentry->d_name, 3287 shmem_initxattrs, NULL); 3288 if (error && error != -EOPNOTSUPP) 3289 goto out_iput; 3290 3291 error = simple_offset_add(shmem_get_offset_ctx(dir), dentry); 3292 if (error) 3293 goto out_iput; 3294 3295 dir->i_size += BOGO_DIRENT_SIZE; 3296 inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); 3297 inode_inc_iversion(dir); 3298 d_instantiate(dentry, inode); 3299 dget(dentry); /* Extra count - pin the dentry in core */ 3300 return error; 3301 3302 out_iput: 3303 iput(inode); 3304 return error; 3305 } 3306 3307 static int 3308 shmem_tmpfile(struct mnt_idmap *idmap, struct inode *dir, 3309 struct file *file, umode_t mode) 3310 { 3311 struct inode *inode; 3312 int error; 3313 3314 inode = shmem_get_inode(idmap, dir->i_sb, dir, mode, 0, VM_NORESERVE); 3315 if (IS_ERR(inode)) { 3316 error = PTR_ERR(inode); 3317 goto err_out; 3318 } 3319 error = security_inode_init_security(inode, dir, NULL, 3320 shmem_initxattrs, NULL); 3321 if (error && error != -EOPNOTSUPP) 3322 goto out_iput; 3323 error = simple_acl_create(dir, inode); 3324 if (error) 3325 goto out_iput; 3326 d_tmpfile(file, inode); 3327 3328 err_out: 3329 return finish_open_simple(file, error); 3330 out_iput: 3331 iput(inode); 3332 return error; 3333 } 3334 3335 static int shmem_mkdir(struct mnt_idmap *idmap, struct inode *dir, 3336 struct dentry *dentry, umode_t mode) 3337 { 3338 int error; 3339 3340 error = shmem_mknod(idmap, dir, dentry, mode | S_IFDIR, 0); 3341 if (error) 3342 return error; 3343 inc_nlink(dir); 3344 return 0; 3345 } 3346 3347 static int shmem_create(struct mnt_idmap *idmap, struct inode *dir, 3348 struct dentry *dentry, umode_t mode, bool excl) 3349 { 3350 return shmem_mknod(idmap, dir, dentry, mode | S_IFREG, 0); 3351 } 3352 3353 /* 3354 * Link a file.. 3355 */ 3356 static int shmem_link(struct dentry *old_dentry, struct inode *dir, 3357 struct dentry *dentry) 3358 { 3359 struct inode *inode = d_inode(old_dentry); 3360 int ret = 0; 3361 3362 /* 3363 * No ordinary (disk based) filesystem counts links as inodes; 3364 * but each new link needs a new dentry, pinning lowmem, and 3365 * tmpfs dentries cannot be pruned until they are unlinked. 3366 * But if an O_TMPFILE file is linked into the tmpfs, the 3367 * first link must skip that, to get the accounting right. 3368 */ 3369 if (inode->i_nlink) { 3370 ret = shmem_reserve_inode(inode->i_sb, NULL); 3371 if (ret) 3372 goto out; 3373 } 3374 3375 ret = simple_offset_add(shmem_get_offset_ctx(dir), dentry); 3376 if (ret) { 3377 if (inode->i_nlink) 3378 shmem_free_inode(inode->i_sb, 0); 3379 goto out; 3380 } 3381 3382 dir->i_size += BOGO_DIRENT_SIZE; 3383 inode_set_mtime_to_ts(dir, 3384 inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode))); 3385 inode_inc_iversion(dir); 3386 inc_nlink(inode); 3387 ihold(inode); /* New dentry reference */ 3388 dget(dentry); /* Extra pinning count for the created dentry */ 3389 d_instantiate(dentry, inode); 3390 out: 3391 return ret; 3392 } 3393 3394 static int shmem_unlink(struct inode *dir, struct dentry *dentry) 3395 { 3396 struct inode *inode = d_inode(dentry); 3397 3398 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 3399 shmem_free_inode(inode->i_sb, 0); 3400 3401 simple_offset_remove(shmem_get_offset_ctx(dir), dentry); 3402 3403 dir->i_size -= BOGO_DIRENT_SIZE; 3404 inode_set_mtime_to_ts(dir, 3405 inode_set_ctime_to_ts(dir, inode_set_ctime_current(inode))); 3406 inode_inc_iversion(dir); 3407 drop_nlink(inode); 3408 dput(dentry); /* Undo the count from "create" - does all the work */ 3409 return 0; 3410 } 3411 3412 static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 3413 { 3414 if (!simple_offset_empty(dentry)) 3415 return -ENOTEMPTY; 3416 3417 drop_nlink(d_inode(dentry)); 3418 drop_nlink(dir); 3419 return shmem_unlink(dir, dentry); 3420 } 3421 3422 static int shmem_whiteout(struct mnt_idmap *idmap, 3423 struct inode *old_dir, struct dentry *old_dentry) 3424 { 3425 struct dentry *whiteout; 3426 int error; 3427 3428 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name); 3429 if (!whiteout) 3430 return -ENOMEM; 3431 3432 error = shmem_mknod(idmap, old_dir, whiteout, 3433 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); 3434 dput(whiteout); 3435 if (error) 3436 return error; 3437 3438 /* 3439 * Cheat and hash the whiteout while the old dentry is still in 3440 * place, instead of playing games with FS_RENAME_DOES_D_MOVE. 3441 * 3442 * d_lookup() will consistently find one of them at this point, 3443 * not sure which one, but that isn't even important. 3444 */ 3445 d_rehash(whiteout); 3446 return 0; 3447 } 3448 3449 /* 3450 * The VFS layer already does all the dentry stuff for rename, 3451 * we just have to decrement the usage count for the target if 3452 * it exists so that the VFS layer correctly free's it when it 3453 * gets overwritten. 3454 */ 3455 static int shmem_rename2(struct mnt_idmap *idmap, 3456 struct inode *old_dir, struct dentry *old_dentry, 3457 struct inode *new_dir, struct dentry *new_dentry, 3458 unsigned int flags) 3459 { 3460 struct inode *inode = d_inode(old_dentry); 3461 int they_are_dirs = S_ISDIR(inode->i_mode); 3462 int error; 3463 3464 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 3465 return -EINVAL; 3466 3467 if (flags & RENAME_EXCHANGE) 3468 return simple_offset_rename_exchange(old_dir, old_dentry, 3469 new_dir, new_dentry); 3470 3471 if (!simple_offset_empty(new_dentry)) 3472 return -ENOTEMPTY; 3473 3474 if (flags & RENAME_WHITEOUT) { 3475 error = shmem_whiteout(idmap, old_dir, old_dentry); 3476 if (error) 3477 return error; 3478 } 3479 3480 error = simple_offset_rename(old_dir, old_dentry, new_dir, new_dentry); 3481 if (error) 3482 return error; 3483 3484 if (d_really_is_positive(new_dentry)) { 3485 (void) shmem_unlink(new_dir, new_dentry); 3486 if (they_are_dirs) { 3487 drop_nlink(d_inode(new_dentry)); 3488 drop_nlink(old_dir); 3489 } 3490 } else if (they_are_dirs) { 3491 drop_nlink(old_dir); 3492 inc_nlink(new_dir); 3493 } 3494 3495 old_dir->i_size -= BOGO_DIRENT_SIZE; 3496 new_dir->i_size += BOGO_DIRENT_SIZE; 3497 simple_rename_timestamp(old_dir, old_dentry, new_dir, new_dentry); 3498 inode_inc_iversion(old_dir); 3499 inode_inc_iversion(new_dir); 3500 return 0; 3501 } 3502 3503 static int shmem_symlink(struct mnt_idmap *idmap, struct inode *dir, 3504 struct dentry *dentry, const char *symname) 3505 { 3506 int error; 3507 int len; 3508 struct inode *inode; 3509 struct folio *folio; 3510 3511 len = strlen(symname) + 1; 3512 if (len > PAGE_SIZE) 3513 return -ENAMETOOLONG; 3514 3515 inode = shmem_get_inode(idmap, dir->i_sb, dir, S_IFLNK | 0777, 0, 3516 VM_NORESERVE); 3517 if (IS_ERR(inode)) 3518 return PTR_ERR(inode); 3519 3520 error = security_inode_init_security(inode, dir, &dentry->d_name, 3521 shmem_initxattrs, NULL); 3522 if (error && error != -EOPNOTSUPP) 3523 goto out_iput; 3524 3525 error = simple_offset_add(shmem_get_offset_ctx(dir), dentry); 3526 if (error) 3527 goto out_iput; 3528 3529 inode->i_size = len-1; 3530 if (len <= SHORT_SYMLINK_LEN) { 3531 inode->i_link = kmemdup(symname, len, GFP_KERNEL); 3532 if (!inode->i_link) { 3533 error = -ENOMEM; 3534 goto out_remove_offset; 3535 } 3536 inode->i_op = &shmem_short_symlink_operations; 3537 } else { 3538 inode_nohighmem(inode); 3539 inode->i_mapping->a_ops = &shmem_aops; 3540 error = shmem_get_folio(inode, 0, &folio, SGP_WRITE); 3541 if (error) 3542 goto out_remove_offset; 3543 inode->i_op = &shmem_symlink_inode_operations; 3544 memcpy(folio_address(folio), symname, len); 3545 folio_mark_uptodate(folio); 3546 folio_mark_dirty(folio); 3547 folio_unlock(folio); 3548 folio_put(folio); 3549 } 3550 dir->i_size += BOGO_DIRENT_SIZE; 3551 inode_set_mtime_to_ts(dir, inode_set_ctime_current(dir)); 3552 inode_inc_iversion(dir); 3553 d_instantiate(dentry, inode); 3554 dget(dentry); 3555 return 0; 3556 3557 out_remove_offset: 3558 simple_offset_remove(shmem_get_offset_ctx(dir), dentry); 3559 out_iput: 3560 iput(inode); 3561 return error; 3562 } 3563 3564 static void shmem_put_link(void *arg) 3565 { 3566 folio_mark_accessed(arg); 3567 folio_put(arg); 3568 } 3569 3570 static const char *shmem_get_link(struct dentry *dentry, struct inode *inode, 3571 struct delayed_call *done) 3572 { 3573 struct folio *folio = NULL; 3574 int error; 3575 3576 if (!dentry) { 3577 folio = filemap_get_folio(inode->i_mapping, 0); 3578 if (IS_ERR(folio)) 3579 return ERR_PTR(-ECHILD); 3580 if (PageHWPoison(folio_page(folio, 0)) || 3581 !folio_test_uptodate(folio)) { 3582 folio_put(folio); 3583 return ERR_PTR(-ECHILD); 3584 } 3585 } else { 3586 error = shmem_get_folio(inode, 0, &folio, SGP_READ); 3587 if (error) 3588 return ERR_PTR(error); 3589 if (!folio) 3590 return ERR_PTR(-ECHILD); 3591 if (PageHWPoison(folio_page(folio, 0))) { 3592 folio_unlock(folio); 3593 folio_put(folio); 3594 return ERR_PTR(-ECHILD); 3595 } 3596 folio_unlock(folio); 3597 } 3598 set_delayed_call(done, shmem_put_link, folio); 3599 return folio_address(folio); 3600 } 3601 3602 #ifdef CONFIG_TMPFS_XATTR 3603 3604 static int shmem_fileattr_get(struct dentry *dentry, struct fileattr *fa) 3605 { 3606 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3607 3608 fileattr_fill_flags(fa, info->fsflags & SHMEM_FL_USER_VISIBLE); 3609 3610 return 0; 3611 } 3612 3613 static int shmem_fileattr_set(struct mnt_idmap *idmap, 3614 struct dentry *dentry, struct fileattr *fa) 3615 { 3616 struct inode *inode = d_inode(dentry); 3617 struct shmem_inode_info *info = SHMEM_I(inode); 3618 3619 if (fileattr_has_fsx(fa)) 3620 return -EOPNOTSUPP; 3621 if (fa->flags & ~SHMEM_FL_USER_MODIFIABLE) 3622 return -EOPNOTSUPP; 3623 3624 info->fsflags = (info->fsflags & ~SHMEM_FL_USER_MODIFIABLE) | 3625 (fa->flags & SHMEM_FL_USER_MODIFIABLE); 3626 3627 shmem_set_inode_flags(inode, info->fsflags); 3628 inode_set_ctime_current(inode); 3629 inode_inc_iversion(inode); 3630 return 0; 3631 } 3632 3633 /* 3634 * Superblocks without xattr inode operations may get some security.* xattr 3635 * support from the LSM "for free". As soon as we have any other xattrs 3636 * like ACLs, we also need to implement the security.* handlers at 3637 * filesystem level, though. 3638 */ 3639 3640 /* 3641 * Callback for security_inode_init_security() for acquiring xattrs. 3642 */ 3643 static int shmem_initxattrs(struct inode *inode, 3644 const struct xattr *xattr_array, void *fs_info) 3645 { 3646 struct shmem_inode_info *info = SHMEM_I(inode); 3647 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 3648 const struct xattr *xattr; 3649 struct simple_xattr *new_xattr; 3650 size_t ispace = 0; 3651 size_t len; 3652 3653 if (sbinfo->max_inodes) { 3654 for (xattr = xattr_array; xattr->name != NULL; xattr++) { 3655 ispace += simple_xattr_space(xattr->name, 3656 xattr->value_len + XATTR_SECURITY_PREFIX_LEN); 3657 } 3658 if (ispace) { 3659 raw_spin_lock(&sbinfo->stat_lock); 3660 if (sbinfo->free_ispace < ispace) 3661 ispace = 0; 3662 else 3663 sbinfo->free_ispace -= ispace; 3664 raw_spin_unlock(&sbinfo->stat_lock); 3665 if (!ispace) 3666 return -ENOSPC; 3667 } 3668 } 3669 3670 for (xattr = xattr_array; xattr->name != NULL; xattr++) { 3671 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len); 3672 if (!new_xattr) 3673 break; 3674 3675 len = strlen(xattr->name) + 1; 3676 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, 3677 GFP_KERNEL_ACCOUNT); 3678 if (!new_xattr->name) { 3679 kvfree(new_xattr); 3680 break; 3681 } 3682 3683 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, 3684 XATTR_SECURITY_PREFIX_LEN); 3685 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, 3686 xattr->name, len); 3687 3688 simple_xattr_add(&info->xattrs, new_xattr); 3689 } 3690 3691 if (xattr->name != NULL) { 3692 if (ispace) { 3693 raw_spin_lock(&sbinfo->stat_lock); 3694 sbinfo->free_ispace += ispace; 3695 raw_spin_unlock(&sbinfo->stat_lock); 3696 } 3697 simple_xattrs_free(&info->xattrs, NULL); 3698 return -ENOMEM; 3699 } 3700 3701 return 0; 3702 } 3703 3704 static int shmem_xattr_handler_get(const struct xattr_handler *handler, 3705 struct dentry *unused, struct inode *inode, 3706 const char *name, void *buffer, size_t size) 3707 { 3708 struct shmem_inode_info *info = SHMEM_I(inode); 3709 3710 name = xattr_full_name(handler, name); 3711 return simple_xattr_get(&info->xattrs, name, buffer, size); 3712 } 3713 3714 static int shmem_xattr_handler_set(const struct xattr_handler *handler, 3715 struct mnt_idmap *idmap, 3716 struct dentry *unused, struct inode *inode, 3717 const char *name, const void *value, 3718 size_t size, int flags) 3719 { 3720 struct shmem_inode_info *info = SHMEM_I(inode); 3721 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 3722 struct simple_xattr *old_xattr; 3723 size_t ispace = 0; 3724 3725 name = xattr_full_name(handler, name); 3726 if (value && sbinfo->max_inodes) { 3727 ispace = simple_xattr_space(name, size); 3728 raw_spin_lock(&sbinfo->stat_lock); 3729 if (sbinfo->free_ispace < ispace) 3730 ispace = 0; 3731 else 3732 sbinfo->free_ispace -= ispace; 3733 raw_spin_unlock(&sbinfo->stat_lock); 3734 if (!ispace) 3735 return -ENOSPC; 3736 } 3737 3738 old_xattr = simple_xattr_set(&info->xattrs, name, value, size, flags); 3739 if (!IS_ERR(old_xattr)) { 3740 ispace = 0; 3741 if (old_xattr && sbinfo->max_inodes) 3742 ispace = simple_xattr_space(old_xattr->name, 3743 old_xattr->size); 3744 simple_xattr_free(old_xattr); 3745 old_xattr = NULL; 3746 inode_set_ctime_current(inode); 3747 inode_inc_iversion(inode); 3748 } 3749 if (ispace) { 3750 raw_spin_lock(&sbinfo->stat_lock); 3751 sbinfo->free_ispace += ispace; 3752 raw_spin_unlock(&sbinfo->stat_lock); 3753 } 3754 return PTR_ERR(old_xattr); 3755 } 3756 3757 static const struct xattr_handler shmem_security_xattr_handler = { 3758 .prefix = XATTR_SECURITY_PREFIX, 3759 .get = shmem_xattr_handler_get, 3760 .set = shmem_xattr_handler_set, 3761 }; 3762 3763 static const struct xattr_handler shmem_trusted_xattr_handler = { 3764 .prefix = XATTR_TRUSTED_PREFIX, 3765 .get = shmem_xattr_handler_get, 3766 .set = shmem_xattr_handler_set, 3767 }; 3768 3769 static const struct xattr_handler shmem_user_xattr_handler = { 3770 .prefix = XATTR_USER_PREFIX, 3771 .get = shmem_xattr_handler_get, 3772 .set = shmem_xattr_handler_set, 3773 }; 3774 3775 static const struct xattr_handler * const shmem_xattr_handlers[] = { 3776 &shmem_security_xattr_handler, 3777 &shmem_trusted_xattr_handler, 3778 &shmem_user_xattr_handler, 3779 NULL 3780 }; 3781 3782 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 3783 { 3784 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3785 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size); 3786 } 3787 #endif /* CONFIG_TMPFS_XATTR */ 3788 3789 static const struct inode_operations shmem_short_symlink_operations = { 3790 .getattr = shmem_getattr, 3791 .setattr = shmem_setattr, 3792 .get_link = simple_get_link, 3793 #ifdef CONFIG_TMPFS_XATTR 3794 .listxattr = shmem_listxattr, 3795 #endif 3796 }; 3797 3798 static const struct inode_operations shmem_symlink_inode_operations = { 3799 .getattr = shmem_getattr, 3800 .setattr = shmem_setattr, 3801 .get_link = shmem_get_link, 3802 #ifdef CONFIG_TMPFS_XATTR 3803 .listxattr = shmem_listxattr, 3804 #endif 3805 }; 3806 3807 static struct dentry *shmem_get_parent(struct dentry *child) 3808 { 3809 return ERR_PTR(-ESTALE); 3810 } 3811 3812 static int shmem_match(struct inode *ino, void *vfh) 3813 { 3814 __u32 *fh = vfh; 3815 __u64 inum = fh[2]; 3816 inum = (inum << 32) | fh[1]; 3817 return ino->i_ino == inum && fh[0] == ino->i_generation; 3818 } 3819 3820 /* Find any alias of inode, but prefer a hashed alias */ 3821 static struct dentry *shmem_find_alias(struct inode *inode) 3822 { 3823 struct dentry *alias = d_find_alias(inode); 3824 3825 return alias ?: d_find_any_alias(inode); 3826 } 3827 3828 static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 3829 struct fid *fid, int fh_len, int fh_type) 3830 { 3831 struct inode *inode; 3832 struct dentry *dentry = NULL; 3833 u64 inum; 3834 3835 if (fh_len < 3) 3836 return NULL; 3837 3838 inum = fid->raw[2]; 3839 inum = (inum << 32) | fid->raw[1]; 3840 3841 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 3842 shmem_match, fid->raw); 3843 if (inode) { 3844 dentry = shmem_find_alias(inode); 3845 iput(inode); 3846 } 3847 3848 return dentry; 3849 } 3850 3851 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len, 3852 struct inode *parent) 3853 { 3854 if (*len < 3) { 3855 *len = 3; 3856 return FILEID_INVALID; 3857 } 3858 3859 if (inode_unhashed(inode)) { 3860 /* Unfortunately insert_inode_hash is not idempotent, 3861 * so as we hash inodes here rather than at creation 3862 * time, we need a lock to ensure we only try 3863 * to do it once 3864 */ 3865 static DEFINE_SPINLOCK(lock); 3866 spin_lock(&lock); 3867 if (inode_unhashed(inode)) 3868 __insert_inode_hash(inode, 3869 inode->i_ino + inode->i_generation); 3870 spin_unlock(&lock); 3871 } 3872 3873 fh[0] = inode->i_generation; 3874 fh[1] = inode->i_ino; 3875 fh[2] = ((__u64)inode->i_ino) >> 32; 3876 3877 *len = 3; 3878 return 1; 3879 } 3880 3881 static const struct export_operations shmem_export_ops = { 3882 .get_parent = shmem_get_parent, 3883 .encode_fh = shmem_encode_fh, 3884 .fh_to_dentry = shmem_fh_to_dentry, 3885 }; 3886 3887 enum shmem_param { 3888 Opt_gid, 3889 Opt_huge, 3890 Opt_mode, 3891 Opt_mpol, 3892 Opt_nr_blocks, 3893 Opt_nr_inodes, 3894 Opt_size, 3895 Opt_uid, 3896 Opt_inode32, 3897 Opt_inode64, 3898 Opt_noswap, 3899 Opt_quota, 3900 Opt_usrquota, 3901 Opt_grpquota, 3902 Opt_usrquota_block_hardlimit, 3903 Opt_usrquota_inode_hardlimit, 3904 Opt_grpquota_block_hardlimit, 3905 Opt_grpquota_inode_hardlimit, 3906 }; 3907 3908 static const struct constant_table shmem_param_enums_huge[] = { 3909 {"never", SHMEM_HUGE_NEVER }, 3910 {"always", SHMEM_HUGE_ALWAYS }, 3911 {"within_size", SHMEM_HUGE_WITHIN_SIZE }, 3912 {"advise", SHMEM_HUGE_ADVISE }, 3913 {} 3914 }; 3915 3916 const struct fs_parameter_spec shmem_fs_parameters[] = { 3917 fsparam_u32 ("gid", Opt_gid), 3918 fsparam_enum ("huge", Opt_huge, shmem_param_enums_huge), 3919 fsparam_u32oct("mode", Opt_mode), 3920 fsparam_string("mpol", Opt_mpol), 3921 fsparam_string("nr_blocks", Opt_nr_blocks), 3922 fsparam_string("nr_inodes", Opt_nr_inodes), 3923 fsparam_string("size", Opt_size), 3924 fsparam_u32 ("uid", Opt_uid), 3925 fsparam_flag ("inode32", Opt_inode32), 3926 fsparam_flag ("inode64", Opt_inode64), 3927 fsparam_flag ("noswap", Opt_noswap), 3928 #ifdef CONFIG_TMPFS_QUOTA 3929 fsparam_flag ("quota", Opt_quota), 3930 fsparam_flag ("usrquota", Opt_usrquota), 3931 fsparam_flag ("grpquota", Opt_grpquota), 3932 fsparam_string("usrquota_block_hardlimit", Opt_usrquota_block_hardlimit), 3933 fsparam_string("usrquota_inode_hardlimit", Opt_usrquota_inode_hardlimit), 3934 fsparam_string("grpquota_block_hardlimit", Opt_grpquota_block_hardlimit), 3935 fsparam_string("grpquota_inode_hardlimit", Opt_grpquota_inode_hardlimit), 3936 #endif 3937 {} 3938 }; 3939 3940 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param) 3941 { 3942 struct shmem_options *ctx = fc->fs_private; 3943 struct fs_parse_result result; 3944 unsigned long long size; 3945 char *rest; 3946 int opt; 3947 kuid_t kuid; 3948 kgid_t kgid; 3949 3950 opt = fs_parse(fc, shmem_fs_parameters, param, &result); 3951 if (opt < 0) 3952 return opt; 3953 3954 switch (opt) { 3955 case Opt_size: 3956 size = memparse(param->string, &rest); 3957 if (*rest == '%') { 3958 size <<= PAGE_SHIFT; 3959 size *= totalram_pages(); 3960 do_div(size, 100); 3961 rest++; 3962 } 3963 if (*rest) 3964 goto bad_value; 3965 ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE); 3966 ctx->seen |= SHMEM_SEEN_BLOCKS; 3967 break; 3968 case Opt_nr_blocks: 3969 ctx->blocks = memparse(param->string, &rest); 3970 if (*rest || ctx->blocks > LONG_MAX) 3971 goto bad_value; 3972 ctx->seen |= SHMEM_SEEN_BLOCKS; 3973 break; 3974 case Opt_nr_inodes: 3975 ctx->inodes = memparse(param->string, &rest); 3976 if (*rest || ctx->inodes > ULONG_MAX / BOGO_INODE_SIZE) 3977 goto bad_value; 3978 ctx->seen |= SHMEM_SEEN_INODES; 3979 break; 3980 case Opt_mode: 3981 ctx->mode = result.uint_32 & 07777; 3982 break; 3983 case Opt_uid: 3984 kuid = make_kuid(current_user_ns(), result.uint_32); 3985 if (!uid_valid(kuid)) 3986 goto bad_value; 3987 3988 /* 3989 * The requested uid must be representable in the 3990 * filesystem's idmapping. 3991 */ 3992 if (!kuid_has_mapping(fc->user_ns, kuid)) 3993 goto bad_value; 3994 3995 ctx->uid = kuid; 3996 break; 3997 case Opt_gid: 3998 kgid = make_kgid(current_user_ns(), result.uint_32); 3999 if (!gid_valid(kgid)) 4000 goto bad_value; 4001 4002 /* 4003 * The requested gid must be representable in the 4004 * filesystem's idmapping. 4005 */ 4006 if (!kgid_has_mapping(fc->user_ns, kgid)) 4007 goto bad_value; 4008 4009 ctx->gid = kgid; 4010 break; 4011 case Opt_huge: 4012 ctx->huge = result.uint_32; 4013 if (ctx->huge != SHMEM_HUGE_NEVER && 4014 !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 4015 has_transparent_hugepage())) 4016 goto unsupported_parameter; 4017 ctx->seen |= SHMEM_SEEN_HUGE; 4018 break; 4019 case Opt_mpol: 4020 if (IS_ENABLED(CONFIG_NUMA)) { 4021 mpol_put(ctx->mpol); 4022 ctx->mpol = NULL; 4023 if (mpol_parse_str(param->string, &ctx->mpol)) 4024 goto bad_value; 4025 break; 4026 } 4027 goto unsupported_parameter; 4028 case Opt_inode32: 4029 ctx->full_inums = false; 4030 ctx->seen |= SHMEM_SEEN_INUMS; 4031 break; 4032 case Opt_inode64: 4033 if (sizeof(ino_t) < 8) { 4034 return invalfc(fc, 4035 "Cannot use inode64 with <64bit inums in kernel\n"); 4036 } 4037 ctx->full_inums = true; 4038 ctx->seen |= SHMEM_SEEN_INUMS; 4039 break; 4040 case Opt_noswap: 4041 if ((fc->user_ns != &init_user_ns) || !capable(CAP_SYS_ADMIN)) { 4042 return invalfc(fc, 4043 "Turning off swap in unprivileged tmpfs mounts unsupported"); 4044 } 4045 ctx->noswap = true; 4046 ctx->seen |= SHMEM_SEEN_NOSWAP; 4047 break; 4048 case Opt_quota: 4049 if (fc->user_ns != &init_user_ns) 4050 return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported"); 4051 ctx->seen |= SHMEM_SEEN_QUOTA; 4052 ctx->quota_types |= (QTYPE_MASK_USR | QTYPE_MASK_GRP); 4053 break; 4054 case Opt_usrquota: 4055 if (fc->user_ns != &init_user_ns) 4056 return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported"); 4057 ctx->seen |= SHMEM_SEEN_QUOTA; 4058 ctx->quota_types |= QTYPE_MASK_USR; 4059 break; 4060 case Opt_grpquota: 4061 if (fc->user_ns != &init_user_ns) 4062 return invalfc(fc, "Quotas in unprivileged tmpfs mounts are unsupported"); 4063 ctx->seen |= SHMEM_SEEN_QUOTA; 4064 ctx->quota_types |= QTYPE_MASK_GRP; 4065 break; 4066 case Opt_usrquota_block_hardlimit: 4067 size = memparse(param->string, &rest); 4068 if (*rest || !size) 4069 goto bad_value; 4070 if (size > SHMEM_QUOTA_MAX_SPC_LIMIT) 4071 return invalfc(fc, 4072 "User quota block hardlimit too large."); 4073 ctx->qlimits.usrquota_bhardlimit = size; 4074 break; 4075 case Opt_grpquota_block_hardlimit: 4076 size = memparse(param->string, &rest); 4077 if (*rest || !size) 4078 goto bad_value; 4079 if (size > SHMEM_QUOTA_MAX_SPC_LIMIT) 4080 return invalfc(fc, 4081 "Group quota block hardlimit too large."); 4082 ctx->qlimits.grpquota_bhardlimit = size; 4083 break; 4084 case Opt_usrquota_inode_hardlimit: 4085 size = memparse(param->string, &rest); 4086 if (*rest || !size) 4087 goto bad_value; 4088 if (size > SHMEM_QUOTA_MAX_INO_LIMIT) 4089 return invalfc(fc, 4090 "User quota inode hardlimit too large."); 4091 ctx->qlimits.usrquota_ihardlimit = size; 4092 break; 4093 case Opt_grpquota_inode_hardlimit: 4094 size = memparse(param->string, &rest); 4095 if (*rest || !size) 4096 goto bad_value; 4097 if (size > SHMEM_QUOTA_MAX_INO_LIMIT) 4098 return invalfc(fc, 4099 "Group quota inode hardlimit too large."); 4100 ctx->qlimits.grpquota_ihardlimit = size; 4101 break; 4102 } 4103 return 0; 4104 4105 unsupported_parameter: 4106 return invalfc(fc, "Unsupported parameter '%s'", param->key); 4107 bad_value: 4108 return invalfc(fc, "Bad value for '%s'", param->key); 4109 } 4110 4111 static int shmem_parse_options(struct fs_context *fc, void *data) 4112 { 4113 char *options = data; 4114 4115 if (options) { 4116 int err = security_sb_eat_lsm_opts(options, &fc->security); 4117 if (err) 4118 return err; 4119 } 4120 4121 while (options != NULL) { 4122 char *this_char = options; 4123 for (;;) { 4124 /* 4125 * NUL-terminate this option: unfortunately, 4126 * mount options form a comma-separated list, 4127 * but mpol's nodelist may also contain commas. 4128 */ 4129 options = strchr(options, ','); 4130 if (options == NULL) 4131 break; 4132 options++; 4133 if (!isdigit(*options)) { 4134 options[-1] = '\0'; 4135 break; 4136 } 4137 } 4138 if (*this_char) { 4139 char *value = strchr(this_char, '='); 4140 size_t len = 0; 4141 int err; 4142 4143 if (value) { 4144 *value++ = '\0'; 4145 len = strlen(value); 4146 } 4147 err = vfs_parse_fs_string(fc, this_char, value, len); 4148 if (err < 0) 4149 return err; 4150 } 4151 } 4152 return 0; 4153 } 4154 4155 /* 4156 * Reconfigure a shmem filesystem. 4157 */ 4158 static int shmem_reconfigure(struct fs_context *fc) 4159 { 4160 struct shmem_options *ctx = fc->fs_private; 4161 struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb); 4162 unsigned long used_isp; 4163 struct mempolicy *mpol = NULL; 4164 const char *err; 4165 4166 raw_spin_lock(&sbinfo->stat_lock); 4167 used_isp = sbinfo->max_inodes * BOGO_INODE_SIZE - sbinfo->free_ispace; 4168 4169 if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) { 4170 if (!sbinfo->max_blocks) { 4171 err = "Cannot retroactively limit size"; 4172 goto out; 4173 } 4174 if (percpu_counter_compare(&sbinfo->used_blocks, 4175 ctx->blocks) > 0) { 4176 err = "Too small a size for current use"; 4177 goto out; 4178 } 4179 } 4180 if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) { 4181 if (!sbinfo->max_inodes) { 4182 err = "Cannot retroactively limit inodes"; 4183 goto out; 4184 } 4185 if (ctx->inodes * BOGO_INODE_SIZE < used_isp) { 4186 err = "Too few inodes for current use"; 4187 goto out; 4188 } 4189 } 4190 4191 if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums && 4192 sbinfo->next_ino > UINT_MAX) { 4193 err = "Current inum too high to switch to 32-bit inums"; 4194 goto out; 4195 } 4196 if ((ctx->seen & SHMEM_SEEN_NOSWAP) && ctx->noswap && !sbinfo->noswap) { 4197 err = "Cannot disable swap on remount"; 4198 goto out; 4199 } 4200 if (!(ctx->seen & SHMEM_SEEN_NOSWAP) && !ctx->noswap && sbinfo->noswap) { 4201 err = "Cannot enable swap on remount if it was disabled on first mount"; 4202 goto out; 4203 } 4204 4205 if (ctx->seen & SHMEM_SEEN_QUOTA && 4206 !sb_any_quota_loaded(fc->root->d_sb)) { 4207 err = "Cannot enable quota on remount"; 4208 goto out; 4209 } 4210 4211 #ifdef CONFIG_TMPFS_QUOTA 4212 #define CHANGED_LIMIT(name) \ 4213 (ctx->qlimits.name## hardlimit && \ 4214 (ctx->qlimits.name## hardlimit != sbinfo->qlimits.name## hardlimit)) 4215 4216 if (CHANGED_LIMIT(usrquota_b) || CHANGED_LIMIT(usrquota_i) || 4217 CHANGED_LIMIT(grpquota_b) || CHANGED_LIMIT(grpquota_i)) { 4218 err = "Cannot change global quota limit on remount"; 4219 goto out; 4220 } 4221 #endif /* CONFIG_TMPFS_QUOTA */ 4222 4223 if (ctx->seen & SHMEM_SEEN_HUGE) 4224 sbinfo->huge = ctx->huge; 4225 if (ctx->seen & SHMEM_SEEN_INUMS) 4226 sbinfo->full_inums = ctx->full_inums; 4227 if (ctx->seen & SHMEM_SEEN_BLOCKS) 4228 sbinfo->max_blocks = ctx->blocks; 4229 if (ctx->seen & SHMEM_SEEN_INODES) { 4230 sbinfo->max_inodes = ctx->inodes; 4231 sbinfo->free_ispace = ctx->inodes * BOGO_INODE_SIZE - used_isp; 4232 } 4233 4234 /* 4235 * Preserve previous mempolicy unless mpol remount option was specified. 4236 */ 4237 if (ctx->mpol) { 4238 mpol = sbinfo->mpol; 4239 sbinfo->mpol = ctx->mpol; /* transfers initial ref */ 4240 ctx->mpol = NULL; 4241 } 4242 4243 if (ctx->noswap) 4244 sbinfo->noswap = true; 4245 4246 raw_spin_unlock(&sbinfo->stat_lock); 4247 mpol_put(mpol); 4248 return 0; 4249 out: 4250 raw_spin_unlock(&sbinfo->stat_lock); 4251 return invalfc(fc, "%s", err); 4252 } 4253 4254 static int shmem_show_options(struct seq_file *seq, struct dentry *root) 4255 { 4256 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb); 4257 struct mempolicy *mpol; 4258 4259 if (sbinfo->max_blocks != shmem_default_max_blocks()) 4260 seq_printf(seq, ",size=%luk", K(sbinfo->max_blocks)); 4261 if (sbinfo->max_inodes != shmem_default_max_inodes()) 4262 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 4263 if (sbinfo->mode != (0777 | S_ISVTX)) 4264 seq_printf(seq, ",mode=%03ho", sbinfo->mode); 4265 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID)) 4266 seq_printf(seq, ",uid=%u", 4267 from_kuid_munged(&init_user_ns, sbinfo->uid)); 4268 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) 4269 seq_printf(seq, ",gid=%u", 4270 from_kgid_munged(&init_user_ns, sbinfo->gid)); 4271 4272 /* 4273 * Showing inode{64,32} might be useful even if it's the system default, 4274 * since then people don't have to resort to checking both here and 4275 * /proc/config.gz to confirm 64-bit inums were successfully applied 4276 * (which may not even exist if IKCONFIG_PROC isn't enabled). 4277 * 4278 * We hide it when inode64 isn't the default and we are using 32-bit 4279 * inodes, since that probably just means the feature isn't even under 4280 * consideration. 4281 * 4282 * As such: 4283 * 4284 * +-----------------+-----------------+ 4285 * | TMPFS_INODE64=y | TMPFS_INODE64=n | 4286 * +------------------+-----------------+-----------------+ 4287 * | full_inums=true | show | show | 4288 * | full_inums=false | show | hide | 4289 * +------------------+-----------------+-----------------+ 4290 * 4291 */ 4292 if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums) 4293 seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32)); 4294 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4295 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ 4296 if (sbinfo->huge) 4297 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); 4298 #endif 4299 mpol = shmem_get_sbmpol(sbinfo); 4300 shmem_show_mpol(seq, mpol); 4301 mpol_put(mpol); 4302 if (sbinfo->noswap) 4303 seq_printf(seq, ",noswap"); 4304 #ifdef CONFIG_TMPFS_QUOTA 4305 if (sb_has_quota_active(root->d_sb, USRQUOTA)) 4306 seq_printf(seq, ",usrquota"); 4307 if (sb_has_quota_active(root->d_sb, GRPQUOTA)) 4308 seq_printf(seq, ",grpquota"); 4309 if (sbinfo->qlimits.usrquota_bhardlimit) 4310 seq_printf(seq, ",usrquota_block_hardlimit=%lld", 4311 sbinfo->qlimits.usrquota_bhardlimit); 4312 if (sbinfo->qlimits.grpquota_bhardlimit) 4313 seq_printf(seq, ",grpquota_block_hardlimit=%lld", 4314 sbinfo->qlimits.grpquota_bhardlimit); 4315 if (sbinfo->qlimits.usrquota_ihardlimit) 4316 seq_printf(seq, ",usrquota_inode_hardlimit=%lld", 4317 sbinfo->qlimits.usrquota_ihardlimit); 4318 if (sbinfo->qlimits.grpquota_ihardlimit) 4319 seq_printf(seq, ",grpquota_inode_hardlimit=%lld", 4320 sbinfo->qlimits.grpquota_ihardlimit); 4321 #endif 4322 return 0; 4323 } 4324 4325 #endif /* CONFIG_TMPFS */ 4326 4327 static void shmem_put_super(struct super_block *sb) 4328 { 4329 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 4330 4331 #ifdef CONFIG_TMPFS_QUOTA 4332 shmem_disable_quotas(sb); 4333 #endif 4334 free_percpu(sbinfo->ino_batch); 4335 percpu_counter_destroy(&sbinfo->used_blocks); 4336 mpol_put(sbinfo->mpol); 4337 kfree(sbinfo); 4338 sb->s_fs_info = NULL; 4339 } 4340 4341 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc) 4342 { 4343 struct shmem_options *ctx = fc->fs_private; 4344 struct inode *inode; 4345 struct shmem_sb_info *sbinfo; 4346 int error = -ENOMEM; 4347 4348 /* Round up to L1_CACHE_BYTES to resist false sharing */ 4349 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 4350 L1_CACHE_BYTES), GFP_KERNEL); 4351 if (!sbinfo) 4352 return error; 4353 4354 sb->s_fs_info = sbinfo; 4355 4356 #ifdef CONFIG_TMPFS 4357 /* 4358 * Per default we only allow half of the physical ram per 4359 * tmpfs instance, limiting inodes to one per page of lowmem; 4360 * but the internal instance is left unlimited. 4361 */ 4362 if (!(sb->s_flags & SB_KERNMOUNT)) { 4363 if (!(ctx->seen & SHMEM_SEEN_BLOCKS)) 4364 ctx->blocks = shmem_default_max_blocks(); 4365 if (!(ctx->seen & SHMEM_SEEN_INODES)) 4366 ctx->inodes = shmem_default_max_inodes(); 4367 if (!(ctx->seen & SHMEM_SEEN_INUMS)) 4368 ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64); 4369 sbinfo->noswap = ctx->noswap; 4370 } else { 4371 sb->s_flags |= SB_NOUSER; 4372 } 4373 sb->s_export_op = &shmem_export_ops; 4374 sb->s_flags |= SB_NOSEC | SB_I_VERSION; 4375 #else 4376 sb->s_flags |= SB_NOUSER; 4377 #endif 4378 sbinfo->max_blocks = ctx->blocks; 4379 sbinfo->max_inodes = ctx->inodes; 4380 sbinfo->free_ispace = sbinfo->max_inodes * BOGO_INODE_SIZE; 4381 if (sb->s_flags & SB_KERNMOUNT) { 4382 sbinfo->ino_batch = alloc_percpu(ino_t); 4383 if (!sbinfo->ino_batch) 4384 goto failed; 4385 } 4386 sbinfo->uid = ctx->uid; 4387 sbinfo->gid = ctx->gid; 4388 sbinfo->full_inums = ctx->full_inums; 4389 sbinfo->mode = ctx->mode; 4390 sbinfo->huge = ctx->huge; 4391 sbinfo->mpol = ctx->mpol; 4392 ctx->mpol = NULL; 4393 4394 raw_spin_lock_init(&sbinfo->stat_lock); 4395 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) 4396 goto failed; 4397 spin_lock_init(&sbinfo->shrinklist_lock); 4398 INIT_LIST_HEAD(&sbinfo->shrinklist); 4399 4400 sb->s_maxbytes = MAX_LFS_FILESIZE; 4401 sb->s_blocksize = PAGE_SIZE; 4402 sb->s_blocksize_bits = PAGE_SHIFT; 4403 sb->s_magic = TMPFS_MAGIC; 4404 sb->s_op = &shmem_ops; 4405 sb->s_time_gran = 1; 4406 #ifdef CONFIG_TMPFS_XATTR 4407 sb->s_xattr = shmem_xattr_handlers; 4408 #endif 4409 #ifdef CONFIG_TMPFS_POSIX_ACL 4410 sb->s_flags |= SB_POSIXACL; 4411 #endif 4412 uuid_t uuid; 4413 uuid_gen(&uuid); 4414 super_set_uuid(sb, uuid.b, sizeof(uuid)); 4415 4416 #ifdef CONFIG_TMPFS_QUOTA 4417 if (ctx->seen & SHMEM_SEEN_QUOTA) { 4418 sb->dq_op = &shmem_quota_operations; 4419 sb->s_qcop = &dquot_quotactl_sysfile_ops; 4420 sb->s_quota_types = QTYPE_MASK_USR | QTYPE_MASK_GRP; 4421 4422 /* Copy the default limits from ctx into sbinfo */ 4423 memcpy(&sbinfo->qlimits, &ctx->qlimits, 4424 sizeof(struct shmem_quota_limits)); 4425 4426 if (shmem_enable_quotas(sb, ctx->quota_types)) 4427 goto failed; 4428 } 4429 #endif /* CONFIG_TMPFS_QUOTA */ 4430 4431 inode = shmem_get_inode(&nop_mnt_idmap, sb, NULL, 4432 S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 4433 if (IS_ERR(inode)) { 4434 error = PTR_ERR(inode); 4435 goto failed; 4436 } 4437 inode->i_uid = sbinfo->uid; 4438 inode->i_gid = sbinfo->gid; 4439 sb->s_root = d_make_root(inode); 4440 if (!sb->s_root) 4441 goto failed; 4442 return 0; 4443 4444 failed: 4445 shmem_put_super(sb); 4446 return error; 4447 } 4448 4449 static int shmem_get_tree(struct fs_context *fc) 4450 { 4451 return get_tree_nodev(fc, shmem_fill_super); 4452 } 4453 4454 static void shmem_free_fc(struct fs_context *fc) 4455 { 4456 struct shmem_options *ctx = fc->fs_private; 4457 4458 if (ctx) { 4459 mpol_put(ctx->mpol); 4460 kfree(ctx); 4461 } 4462 } 4463 4464 static const struct fs_context_operations shmem_fs_context_ops = { 4465 .free = shmem_free_fc, 4466 .get_tree = shmem_get_tree, 4467 #ifdef CONFIG_TMPFS 4468 .parse_monolithic = shmem_parse_options, 4469 .parse_param = shmem_parse_one, 4470 .reconfigure = shmem_reconfigure, 4471 #endif 4472 }; 4473 4474 static struct kmem_cache *shmem_inode_cachep __ro_after_init; 4475 4476 static struct inode *shmem_alloc_inode(struct super_block *sb) 4477 { 4478 struct shmem_inode_info *info; 4479 info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL); 4480 if (!info) 4481 return NULL; 4482 return &info->vfs_inode; 4483 } 4484 4485 static void shmem_free_in_core_inode(struct inode *inode) 4486 { 4487 if (S_ISLNK(inode->i_mode)) 4488 kfree(inode->i_link); 4489 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 4490 } 4491 4492 static void shmem_destroy_inode(struct inode *inode) 4493 { 4494 if (S_ISREG(inode->i_mode)) 4495 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 4496 if (S_ISDIR(inode->i_mode)) 4497 simple_offset_destroy(shmem_get_offset_ctx(inode)); 4498 } 4499 4500 static void shmem_init_inode(void *foo) 4501 { 4502 struct shmem_inode_info *info = foo; 4503 inode_init_once(&info->vfs_inode); 4504 } 4505 4506 static void __init shmem_init_inodecache(void) 4507 { 4508 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 4509 sizeof(struct shmem_inode_info), 4510 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode); 4511 } 4512 4513 static void __init shmem_destroy_inodecache(void) 4514 { 4515 kmem_cache_destroy(shmem_inode_cachep); 4516 } 4517 4518 /* Keep the page in page cache instead of truncating it */ 4519 static int shmem_error_remove_folio(struct address_space *mapping, 4520 struct folio *folio) 4521 { 4522 return 0; 4523 } 4524 4525 static const struct address_space_operations shmem_aops = { 4526 .writepage = shmem_writepage, 4527 .dirty_folio = noop_dirty_folio, 4528 #ifdef CONFIG_TMPFS 4529 .write_begin = shmem_write_begin, 4530 .write_end = shmem_write_end, 4531 #endif 4532 #ifdef CONFIG_MIGRATION 4533 .migrate_folio = migrate_folio, 4534 #endif 4535 .error_remove_folio = shmem_error_remove_folio, 4536 }; 4537 4538 static const struct file_operations shmem_file_operations = { 4539 .mmap = shmem_mmap, 4540 .open = shmem_file_open, 4541 .get_unmapped_area = shmem_get_unmapped_area, 4542 #ifdef CONFIG_TMPFS 4543 .llseek = shmem_file_llseek, 4544 .read_iter = shmem_file_read_iter, 4545 .write_iter = shmem_file_write_iter, 4546 .fsync = noop_fsync, 4547 .splice_read = shmem_file_splice_read, 4548 .splice_write = iter_file_splice_write, 4549 .fallocate = shmem_fallocate, 4550 #endif 4551 }; 4552 4553 static const struct inode_operations shmem_inode_operations = { 4554 .getattr = shmem_getattr, 4555 .setattr = shmem_setattr, 4556 #ifdef CONFIG_TMPFS_XATTR 4557 .listxattr = shmem_listxattr, 4558 .set_acl = simple_set_acl, 4559 .fileattr_get = shmem_fileattr_get, 4560 .fileattr_set = shmem_fileattr_set, 4561 #endif 4562 }; 4563 4564 static const struct inode_operations shmem_dir_inode_operations = { 4565 #ifdef CONFIG_TMPFS 4566 .getattr = shmem_getattr, 4567 .create = shmem_create, 4568 .lookup = simple_lookup, 4569 .link = shmem_link, 4570 .unlink = shmem_unlink, 4571 .symlink = shmem_symlink, 4572 .mkdir = shmem_mkdir, 4573 .rmdir = shmem_rmdir, 4574 .mknod = shmem_mknod, 4575 .rename = shmem_rename2, 4576 .tmpfile = shmem_tmpfile, 4577 .get_offset_ctx = shmem_get_offset_ctx, 4578 #endif 4579 #ifdef CONFIG_TMPFS_XATTR 4580 .listxattr = shmem_listxattr, 4581 .fileattr_get = shmem_fileattr_get, 4582 .fileattr_set = shmem_fileattr_set, 4583 #endif 4584 #ifdef CONFIG_TMPFS_POSIX_ACL 4585 .setattr = shmem_setattr, 4586 .set_acl = simple_set_acl, 4587 #endif 4588 }; 4589 4590 static const struct inode_operations shmem_special_inode_operations = { 4591 .getattr = shmem_getattr, 4592 #ifdef CONFIG_TMPFS_XATTR 4593 .listxattr = shmem_listxattr, 4594 #endif 4595 #ifdef CONFIG_TMPFS_POSIX_ACL 4596 .setattr = shmem_setattr, 4597 .set_acl = simple_set_acl, 4598 #endif 4599 }; 4600 4601 static const struct super_operations shmem_ops = { 4602 .alloc_inode = shmem_alloc_inode, 4603 .free_inode = shmem_free_in_core_inode, 4604 .destroy_inode = shmem_destroy_inode, 4605 #ifdef CONFIG_TMPFS 4606 .statfs = shmem_statfs, 4607 .show_options = shmem_show_options, 4608 #endif 4609 #ifdef CONFIG_TMPFS_QUOTA 4610 .get_dquots = shmem_get_dquots, 4611 #endif 4612 .evict_inode = shmem_evict_inode, 4613 .drop_inode = generic_delete_inode, 4614 .put_super = shmem_put_super, 4615 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4616 .nr_cached_objects = shmem_unused_huge_count, 4617 .free_cached_objects = shmem_unused_huge_scan, 4618 #endif 4619 }; 4620 4621 static const struct vm_operations_struct shmem_vm_ops = { 4622 .fault = shmem_fault, 4623 .map_pages = filemap_map_pages, 4624 #ifdef CONFIG_NUMA 4625 .set_policy = shmem_set_policy, 4626 .get_policy = shmem_get_policy, 4627 #endif 4628 }; 4629 4630 static const struct vm_operations_struct shmem_anon_vm_ops = { 4631 .fault = shmem_fault, 4632 .map_pages = filemap_map_pages, 4633 #ifdef CONFIG_NUMA 4634 .set_policy = shmem_set_policy, 4635 .get_policy = shmem_get_policy, 4636 #endif 4637 }; 4638 4639 int shmem_init_fs_context(struct fs_context *fc) 4640 { 4641 struct shmem_options *ctx; 4642 4643 ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL); 4644 if (!ctx) 4645 return -ENOMEM; 4646 4647 ctx->mode = 0777 | S_ISVTX; 4648 ctx->uid = current_fsuid(); 4649 ctx->gid = current_fsgid(); 4650 4651 fc->fs_private = ctx; 4652 fc->ops = &shmem_fs_context_ops; 4653 return 0; 4654 } 4655 4656 static struct file_system_type shmem_fs_type = { 4657 .owner = THIS_MODULE, 4658 .name = "tmpfs", 4659 .init_fs_context = shmem_init_fs_context, 4660 #ifdef CONFIG_TMPFS 4661 .parameters = shmem_fs_parameters, 4662 #endif 4663 .kill_sb = kill_litter_super, 4664 .fs_flags = FS_USERNS_MOUNT | FS_ALLOW_IDMAP, 4665 }; 4666 4667 void __init shmem_init(void) 4668 { 4669 int error; 4670 4671 shmem_init_inodecache(); 4672 4673 #ifdef CONFIG_TMPFS_QUOTA 4674 error = register_quota_format(&shmem_quota_format); 4675 if (error < 0) { 4676 pr_err("Could not register quota format\n"); 4677 goto out3; 4678 } 4679 #endif 4680 4681 error = register_filesystem(&shmem_fs_type); 4682 if (error) { 4683 pr_err("Could not register tmpfs\n"); 4684 goto out2; 4685 } 4686 4687 shm_mnt = kern_mount(&shmem_fs_type); 4688 if (IS_ERR(shm_mnt)) { 4689 error = PTR_ERR(shm_mnt); 4690 pr_err("Could not kern_mount tmpfs\n"); 4691 goto out1; 4692 } 4693 4694 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 4695 if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY) 4696 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 4697 else 4698 shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */ 4699 #endif 4700 return; 4701 4702 out1: 4703 unregister_filesystem(&shmem_fs_type); 4704 out2: 4705 #ifdef CONFIG_TMPFS_QUOTA 4706 unregister_quota_format(&shmem_quota_format); 4707 out3: 4708 #endif 4709 shmem_destroy_inodecache(); 4710 shm_mnt = ERR_PTR(error); 4711 } 4712 4713 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS) 4714 static ssize_t shmem_enabled_show(struct kobject *kobj, 4715 struct kobj_attribute *attr, char *buf) 4716 { 4717 static const int values[] = { 4718 SHMEM_HUGE_ALWAYS, 4719 SHMEM_HUGE_WITHIN_SIZE, 4720 SHMEM_HUGE_ADVISE, 4721 SHMEM_HUGE_NEVER, 4722 SHMEM_HUGE_DENY, 4723 SHMEM_HUGE_FORCE, 4724 }; 4725 int len = 0; 4726 int i; 4727 4728 for (i = 0; i < ARRAY_SIZE(values); i++) { 4729 len += sysfs_emit_at(buf, len, 4730 shmem_huge == values[i] ? "%s[%s]" : "%s%s", 4731 i ? " " : "", shmem_format_huge(values[i])); 4732 } 4733 len += sysfs_emit_at(buf, len, "\n"); 4734 4735 return len; 4736 } 4737 4738 static ssize_t shmem_enabled_store(struct kobject *kobj, 4739 struct kobj_attribute *attr, const char *buf, size_t count) 4740 { 4741 char tmp[16]; 4742 int huge; 4743 4744 if (count + 1 > sizeof(tmp)) 4745 return -EINVAL; 4746 memcpy(tmp, buf, count); 4747 tmp[count] = '\0'; 4748 if (count && tmp[count - 1] == '\n') 4749 tmp[count - 1] = '\0'; 4750 4751 huge = shmem_parse_huge(tmp); 4752 if (huge == -EINVAL) 4753 return -EINVAL; 4754 if (!has_transparent_hugepage() && 4755 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY) 4756 return -EINVAL; 4757 4758 shmem_huge = huge; 4759 if (shmem_huge > SHMEM_HUGE_DENY) 4760 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 4761 return count; 4762 } 4763 4764 struct kobj_attribute shmem_enabled_attr = __ATTR_RW(shmem_enabled); 4765 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */ 4766 4767 #else /* !CONFIG_SHMEM */ 4768 4769 /* 4770 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 4771 * 4772 * This is intended for small system where the benefits of the full 4773 * shmem code (swap-backed and resource-limited) are outweighed by 4774 * their complexity. On systems without swap this code should be 4775 * effectively equivalent, but much lighter weight. 4776 */ 4777 4778 static struct file_system_type shmem_fs_type = { 4779 .name = "tmpfs", 4780 .init_fs_context = ramfs_init_fs_context, 4781 .parameters = ramfs_fs_parameters, 4782 .kill_sb = ramfs_kill_sb, 4783 .fs_flags = FS_USERNS_MOUNT, 4784 }; 4785 4786 void __init shmem_init(void) 4787 { 4788 BUG_ON(register_filesystem(&shmem_fs_type) != 0); 4789 4790 shm_mnt = kern_mount(&shmem_fs_type); 4791 BUG_ON(IS_ERR(shm_mnt)); 4792 } 4793 4794 int shmem_unuse(unsigned int type) 4795 { 4796 return 0; 4797 } 4798 4799 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts) 4800 { 4801 return 0; 4802 } 4803 4804 void shmem_unlock_mapping(struct address_space *mapping) 4805 { 4806 } 4807 4808 #ifdef CONFIG_MMU 4809 unsigned long shmem_get_unmapped_area(struct file *file, 4810 unsigned long addr, unsigned long len, 4811 unsigned long pgoff, unsigned long flags) 4812 { 4813 return mm_get_unmapped_area(current->mm, file, addr, len, pgoff, flags); 4814 } 4815 #endif 4816 4817 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 4818 { 4819 truncate_inode_pages_range(inode->i_mapping, lstart, lend); 4820 } 4821 EXPORT_SYMBOL_GPL(shmem_truncate_range); 4822 4823 #define shmem_vm_ops generic_file_vm_ops 4824 #define shmem_anon_vm_ops generic_file_vm_ops 4825 #define shmem_file_operations ramfs_file_operations 4826 #define shmem_acct_size(flags, size) 0 4827 #define shmem_unacct_size(flags, size) do {} while (0) 4828 4829 static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap, 4830 struct super_block *sb, struct inode *dir, 4831 umode_t mode, dev_t dev, unsigned long flags) 4832 { 4833 struct inode *inode = ramfs_get_inode(sb, dir, mode, dev); 4834 return inode ? inode : ERR_PTR(-ENOSPC); 4835 } 4836 4837 #endif /* CONFIG_SHMEM */ 4838 4839 /* common code */ 4840 4841 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, 4842 loff_t size, unsigned long flags, unsigned int i_flags) 4843 { 4844 struct inode *inode; 4845 struct file *res; 4846 4847 if (IS_ERR(mnt)) 4848 return ERR_CAST(mnt); 4849 4850 if (size < 0 || size > MAX_LFS_FILESIZE) 4851 return ERR_PTR(-EINVAL); 4852 4853 if (shmem_acct_size(flags, size)) 4854 return ERR_PTR(-ENOMEM); 4855 4856 if (is_idmapped_mnt(mnt)) 4857 return ERR_PTR(-EINVAL); 4858 4859 inode = shmem_get_inode(&nop_mnt_idmap, mnt->mnt_sb, NULL, 4860 S_IFREG | S_IRWXUGO, 0, flags); 4861 if (IS_ERR(inode)) { 4862 shmem_unacct_size(flags, size); 4863 return ERR_CAST(inode); 4864 } 4865 inode->i_flags |= i_flags; 4866 inode->i_size = size; 4867 clear_nlink(inode); /* It is unlinked */ 4868 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); 4869 if (!IS_ERR(res)) 4870 res = alloc_file_pseudo(inode, mnt, name, O_RDWR, 4871 &shmem_file_operations); 4872 if (IS_ERR(res)) 4873 iput(inode); 4874 return res; 4875 } 4876 4877 /** 4878 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be 4879 * kernel internal. There will be NO LSM permission checks against the 4880 * underlying inode. So users of this interface must do LSM checks at a 4881 * higher layer. The users are the big_key and shm implementations. LSM 4882 * checks are provided at the key or shm level rather than the inode. 4883 * @name: name for dentry (to be seen in /proc/<pid>/maps 4884 * @size: size to be set for the file 4885 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4886 */ 4887 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags) 4888 { 4889 return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE); 4890 } 4891 EXPORT_SYMBOL_GPL(shmem_kernel_file_setup); 4892 4893 /** 4894 * shmem_file_setup - get an unlinked file living in tmpfs 4895 * @name: name for dentry (to be seen in /proc/<pid>/maps 4896 * @size: size to be set for the file 4897 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4898 */ 4899 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 4900 { 4901 return __shmem_file_setup(shm_mnt, name, size, flags, 0); 4902 } 4903 EXPORT_SYMBOL_GPL(shmem_file_setup); 4904 4905 /** 4906 * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs 4907 * @mnt: the tmpfs mount where the file will be created 4908 * @name: name for dentry (to be seen in /proc/<pid>/maps 4909 * @size: size to be set for the file 4910 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4911 */ 4912 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name, 4913 loff_t size, unsigned long flags) 4914 { 4915 return __shmem_file_setup(mnt, name, size, flags, 0); 4916 } 4917 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt); 4918 4919 /** 4920 * shmem_zero_setup - setup a shared anonymous mapping 4921 * @vma: the vma to be mmapped is prepared by do_mmap 4922 */ 4923 int shmem_zero_setup(struct vm_area_struct *vma) 4924 { 4925 struct file *file; 4926 loff_t size = vma->vm_end - vma->vm_start; 4927 4928 /* 4929 * Cloning a new file under mmap_lock leads to a lock ordering conflict 4930 * between XFS directory reading and selinux: since this file is only 4931 * accessible to the user through its mapping, use S_PRIVATE flag to 4932 * bypass file security, in the same way as shmem_kernel_file_setup(). 4933 */ 4934 file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags); 4935 if (IS_ERR(file)) 4936 return PTR_ERR(file); 4937 4938 if (vma->vm_file) 4939 fput(vma->vm_file); 4940 vma->vm_file = file; 4941 vma->vm_ops = &shmem_anon_vm_ops; 4942 4943 return 0; 4944 } 4945 4946 /** 4947 * shmem_read_folio_gfp - read into page cache, using specified page allocation flags. 4948 * @mapping: the folio's address_space 4949 * @index: the folio index 4950 * @gfp: the page allocator flags to use if allocating 4951 * 4952 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", 4953 * with any new page allocations done using the specified allocation flags. 4954 * But read_cache_page_gfp() uses the ->read_folio() method: which does not 4955 * suit tmpfs, since it may have pages in swapcache, and needs to find those 4956 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. 4957 * 4958 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in 4959 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. 4960 */ 4961 struct folio *shmem_read_folio_gfp(struct address_space *mapping, 4962 pgoff_t index, gfp_t gfp) 4963 { 4964 #ifdef CONFIG_SHMEM 4965 struct inode *inode = mapping->host; 4966 struct folio *folio; 4967 int error; 4968 4969 error = shmem_get_folio_gfp(inode, index, &folio, SGP_CACHE, 4970 gfp, NULL, NULL); 4971 if (error) 4972 return ERR_PTR(error); 4973 4974 folio_unlock(folio); 4975 return folio; 4976 #else 4977 /* 4978 * The tiny !SHMEM case uses ramfs without swap 4979 */ 4980 return mapping_read_folio_gfp(mapping, index, gfp); 4981 #endif 4982 } 4983 EXPORT_SYMBOL_GPL(shmem_read_folio_gfp); 4984 4985 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, 4986 pgoff_t index, gfp_t gfp) 4987 { 4988 struct folio *folio = shmem_read_folio_gfp(mapping, index, gfp); 4989 struct page *page; 4990 4991 if (IS_ERR(folio)) 4992 return &folio->page; 4993 4994 page = folio_file_page(folio, index); 4995 if (PageHWPoison(page)) { 4996 folio_put(folio); 4997 return ERR_PTR(-EIO); 4998 } 4999 5000 return page; 5001 } 5002 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); 5003