1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/pagewalk.h> 3 #include <linux/mm_inline.h> 4 #include <linux/hugetlb.h> 5 #include <linux/huge_mm.h> 6 #include <linux/mount.h> 7 #include <linux/ksm.h> 8 #include <linux/seq_file.h> 9 #include <linux/highmem.h> 10 #include <linux/ptrace.h> 11 #include <linux/slab.h> 12 #include <linux/pagemap.h> 13 #include <linux/mempolicy.h> 14 #include <linux/rmap.h> 15 #include <linux/swap.h> 16 #include <linux/sched/mm.h> 17 #include <linux/leafops.h> 18 #include <linux/mmu_notifier.h> 19 #include <linux/page_idle.h> 20 #include <linux/shmem_fs.h> 21 #include <linux/uaccess.h> 22 #include <linux/pkeys.h> 23 #include <linux/minmax.h> 24 #include <linux/overflow.h> 25 #include <linux/buildid.h> 26 27 #include <asm/elf.h> 28 #include <asm/tlb.h> 29 #include <asm/tlbflush.h> 30 #include "internal.h" 31 32 #define SENTINEL_VMA_END -1 33 #define SENTINEL_VMA_GATE -2 34 35 #define SEQ_PUT_DEC(str, val) \ 36 seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8) 37 void task_mem(struct seq_file *m, struct mm_struct *mm) 38 { 39 unsigned long text, lib, swap, anon, file, shmem; 40 unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss; 41 42 anon = get_mm_counter_sum(mm, MM_ANONPAGES); 43 file = get_mm_counter_sum(mm, MM_FILEPAGES); 44 shmem = get_mm_counter_sum(mm, MM_SHMEMPAGES); 45 46 /* 47 * Note: to minimize their overhead, mm maintains hiwater_vm and 48 * hiwater_rss only when about to *lower* total_vm or rss. Any 49 * collector of these hiwater stats must therefore get total_vm 50 * and rss too, which will usually be the higher. Barriers? not 51 * worth the effort, such snapshots can always be inconsistent. 52 */ 53 hiwater_vm = total_vm = mm->total_vm; 54 if (hiwater_vm < mm->hiwater_vm) 55 hiwater_vm = mm->hiwater_vm; 56 hiwater_rss = total_rss = anon + file + shmem; 57 if (hiwater_rss < mm->hiwater_rss) 58 hiwater_rss = mm->hiwater_rss; 59 60 /* split executable areas between text and lib */ 61 text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK); 62 text = min(text, mm->exec_vm << PAGE_SHIFT); 63 lib = (mm->exec_vm << PAGE_SHIFT) - text; 64 65 swap = get_mm_counter_sum(mm, MM_SWAPENTS); 66 SEQ_PUT_DEC("VmPeak:\t", hiwater_vm); 67 SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm); 68 SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm); 69 SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm)); 70 SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss); 71 SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss); 72 SEQ_PUT_DEC(" kB\nRssAnon:\t", anon); 73 SEQ_PUT_DEC(" kB\nRssFile:\t", file); 74 SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem); 75 SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm); 76 SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm); 77 seq_put_decimal_ull_width(m, 78 " kB\nVmExe:\t", text >> 10, 8); 79 seq_put_decimal_ull_width(m, 80 " kB\nVmLib:\t", lib >> 10, 8); 81 seq_put_decimal_ull_width(m, 82 " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8); 83 SEQ_PUT_DEC(" kB\nVmSwap:\t", swap); 84 seq_puts(m, " kB\n"); 85 hugetlb_report_usage(m, mm); 86 } 87 #undef SEQ_PUT_DEC 88 89 unsigned long task_vsize(struct mm_struct *mm) 90 { 91 return PAGE_SIZE * mm->total_vm; 92 } 93 94 unsigned long task_statm(struct mm_struct *mm, 95 unsigned long *shared, unsigned long *text, 96 unsigned long *data, unsigned long *resident) 97 { 98 *shared = get_mm_counter_sum(mm, MM_FILEPAGES) + 99 get_mm_counter_sum(mm, MM_SHMEMPAGES); 100 *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) 101 >> PAGE_SHIFT; 102 *data = mm->data_vm + mm->stack_vm; 103 *resident = *shared + get_mm_counter_sum(mm, MM_ANONPAGES); 104 return mm->total_vm; 105 } 106 107 #ifdef CONFIG_NUMA 108 /* 109 * Save get_task_policy() for show_numa_map(). 110 */ 111 static void hold_task_mempolicy(struct proc_maps_private *priv) 112 { 113 struct task_struct *task = priv->task; 114 115 task_lock(task); 116 priv->task_mempolicy = get_task_policy(task); 117 mpol_get(priv->task_mempolicy); 118 task_unlock(task); 119 } 120 static void release_task_mempolicy(struct proc_maps_private *priv) 121 { 122 mpol_put(priv->task_mempolicy); 123 } 124 #else 125 static void hold_task_mempolicy(struct proc_maps_private *priv) 126 { 127 } 128 static void release_task_mempolicy(struct proc_maps_private *priv) 129 { 130 } 131 #endif 132 133 #ifdef CONFIG_PER_VMA_LOCK 134 135 static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) 136 { 137 int ret = mmap_read_lock_killable(lock_ctx->mm); 138 139 if (!ret) 140 lock_ctx->mmap_locked = true; 141 142 return ret; 143 } 144 145 static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) 146 { 147 mmap_read_unlock(lock_ctx->mm); 148 lock_ctx->mmap_locked = false; 149 } 150 151 static void reset_lock_ctx(struct proc_maps_locking_ctx *lock_ctx) 152 { 153 lock_ctx->locked_vma = NULL; 154 lock_ctx->mmap_locked = false; 155 } 156 157 static void unlock_ctx_vma(struct proc_maps_locking_ctx *lock_ctx) 158 { 159 if (lock_ctx->locked_vma) { 160 vma_end_read(lock_ctx->locked_vma); 161 lock_ctx->locked_vma = NULL; 162 } 163 } 164 165 static inline bool lock_vma_range(struct seq_file *m, 166 struct proc_maps_locking_ctx *lock_ctx) 167 { 168 rcu_read_lock(); 169 reset_lock_ctx(lock_ctx); 170 171 return true; 172 } 173 174 static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx) 175 { 176 if (lock_ctx->mmap_locked) { 177 unlock_ctx_mm(lock_ctx); 178 } else { 179 unlock_ctx_vma(lock_ctx); 180 rcu_read_unlock(); 181 } 182 } 183 184 static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, 185 loff_t last_pos) 186 { 187 struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx; 188 struct vm_area_struct *vma; 189 190 if (lock_ctx->mmap_locked) 191 return vma_next(&priv->iter); 192 193 unlock_ctx_vma(lock_ctx); 194 vma = lock_next_vma(lock_ctx->mm, &priv->iter, last_pos); 195 if (!IS_ERR_OR_NULL(vma)) 196 lock_ctx->locked_vma = vma; 197 198 return vma; 199 } 200 201 static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, 202 loff_t pos) 203 { 204 struct proc_maps_locking_ctx *lock_ctx = &priv->lock_ctx; 205 206 if (lock_ctx->mmap_locked) 207 return false; 208 209 rcu_read_unlock(); 210 mmap_read_lock(lock_ctx->mm); 211 /* Reinitialize the iterator after taking mmap_lock */ 212 vma_iter_set(&priv->iter, pos); 213 lock_ctx->mmap_locked = true; 214 215 return true; 216 } 217 218 static inline void drop_rcu(struct proc_maps_private *priv) 219 { 220 if (priv->lock_ctx.mmap_locked) 221 return; 222 223 rcu_read_unlock(); 224 } 225 226 static inline void reacquire_rcu(struct proc_maps_private *priv) 227 { 228 if (priv->lock_ctx.mmap_locked) 229 return; 230 231 rcu_read_lock(); 232 /* Reinitialize the iterator. */ 233 vma_iter_set(&priv->iter, priv->lock_ctx.locked_vma->vm_end); 234 } 235 236 #else /* CONFIG_PER_VMA_LOCK */ 237 238 static inline int lock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) 239 { 240 return mmap_read_lock_killable(lock_ctx->mm); 241 } 242 243 static inline void unlock_ctx_mm(struct proc_maps_locking_ctx *lock_ctx) 244 { 245 mmap_read_unlock(lock_ctx->mm); 246 } 247 248 static inline bool lock_vma_range(struct seq_file *m, 249 struct proc_maps_locking_ctx *lock_ctx) 250 { 251 return lock_ctx_mm(lock_ctx) == 0; 252 } 253 254 static inline void unlock_vma_range(struct proc_maps_locking_ctx *lock_ctx) 255 { 256 unlock_ctx_mm(lock_ctx); 257 } 258 259 static struct vm_area_struct *get_next_vma(struct proc_maps_private *priv, 260 loff_t last_pos) 261 { 262 return vma_next(&priv->iter); 263 } 264 265 static inline bool fallback_to_mmap_lock(struct proc_maps_private *priv, 266 loff_t pos) 267 { 268 return false; 269 } 270 271 static inline void drop_rcu(struct proc_maps_private *priv) {} 272 static inline void reacquire_rcu(struct proc_maps_private *priv) {} 273 274 #endif /* CONFIG_PER_VMA_LOCK */ 275 276 static struct vm_area_struct *proc_get_vma(struct seq_file *m, loff_t *ppos) 277 { 278 struct proc_maps_private *priv = m->private; 279 struct vm_area_struct *vma; 280 281 retry: 282 vma = get_next_vma(priv, *ppos); 283 /* EINTR of EAGAIN is possible */ 284 if (IS_ERR(vma)) { 285 if (PTR_ERR(vma) == -EAGAIN && fallback_to_mmap_lock(priv, *ppos)) 286 goto retry; 287 288 return vma; 289 } 290 291 /* Store previous position to be able to restart if needed */ 292 priv->last_pos = *ppos; 293 if (vma) { 294 /* 295 * Track the end of the reported vma to ensure position changes 296 * even if previous vma was merged with the next vma and we 297 * found the extended vma with the same vm_start. 298 */ 299 *ppos = vma->vm_end; 300 } else { 301 *ppos = SENTINEL_VMA_GATE; 302 vma = get_gate_vma(priv->lock_ctx.mm); 303 } 304 305 return vma; 306 } 307 308 static void *m_start(struct seq_file *m, loff_t *ppos) 309 { 310 struct proc_maps_private *priv = m->private; 311 struct proc_maps_locking_ctx *lock_ctx; 312 loff_t last_addr = *ppos; 313 struct mm_struct *mm; 314 315 /* See m_next(). Zero at the start or after lseek. */ 316 if (last_addr == SENTINEL_VMA_END) 317 return NULL; 318 319 priv->task = get_proc_task(priv->inode); 320 if (!priv->task) 321 return ERR_PTR(-ESRCH); 322 323 lock_ctx = &priv->lock_ctx; 324 mm = lock_ctx->mm; 325 if (!mm || !mmget_not_zero(mm)) { 326 put_task_struct(priv->task); 327 priv->task = NULL; 328 return NULL; 329 } 330 331 if (!lock_vma_range(m, lock_ctx)) { 332 mmput(mm); 333 put_task_struct(priv->task); 334 priv->task = NULL; 335 return ERR_PTR(-EINTR); 336 } 337 338 /* 339 * Reset current position if last_addr was set before 340 * and it's not a sentinel. 341 */ 342 if (last_addr > 0) 343 *ppos = last_addr = priv->last_pos; 344 vma_iter_init(&priv->iter, mm, (unsigned long)last_addr); 345 hold_task_mempolicy(priv); 346 if (last_addr == SENTINEL_VMA_GATE) 347 return get_gate_vma(mm); 348 349 return proc_get_vma(m, ppos); 350 } 351 352 static void *m_next(struct seq_file *m, void *v, loff_t *ppos) 353 { 354 if (*ppos == SENTINEL_VMA_GATE) { 355 *ppos = SENTINEL_VMA_END; 356 return NULL; 357 } 358 return proc_get_vma(m, ppos); 359 } 360 361 static void m_stop(struct seq_file *m, void *v) 362 { 363 struct proc_maps_private *priv = m->private; 364 struct mm_struct *mm = priv->lock_ctx.mm; 365 366 if (!priv->task) 367 return; 368 369 release_task_mempolicy(priv); 370 unlock_vma_range(&priv->lock_ctx); 371 mmput(mm); 372 put_task_struct(priv->task); 373 priv->task = NULL; 374 } 375 376 static int proc_maps_open(struct inode *inode, struct file *file, 377 const struct seq_operations *ops, int psize) 378 { 379 struct proc_maps_private *priv = __seq_open_private(file, ops, psize); 380 381 if (!priv) 382 return -ENOMEM; 383 384 priv->inode = inode; 385 priv->lock_ctx.mm = proc_mem_open(inode, PTRACE_MODE_READ); 386 if (IS_ERR(priv->lock_ctx.mm)) { 387 int err = PTR_ERR(priv->lock_ctx.mm); 388 389 seq_release_private(inode, file); 390 return err; 391 } 392 393 return 0; 394 } 395 396 static int proc_map_release(struct inode *inode, struct file *file) 397 { 398 struct seq_file *seq = file->private_data; 399 struct proc_maps_private *priv = seq->private; 400 401 if (priv->lock_ctx.mm) 402 mmdrop(priv->lock_ctx.mm); 403 404 return seq_release_private(inode, file); 405 } 406 407 static int do_maps_open(struct inode *inode, struct file *file, 408 const struct seq_operations *ops) 409 { 410 return proc_maps_open(inode, file, ops, 411 sizeof(struct proc_maps_private)); 412 } 413 414 static void get_vma_name(struct vm_area_struct *vma, 415 const struct path **path, 416 const char **name, 417 const char **name_fmt) 418 { 419 struct anon_vma_name *anon_name = vma->vm_mm ? anon_vma_name(vma) : NULL; 420 421 *name = NULL; 422 *path = NULL; 423 *name_fmt = NULL; 424 425 /* 426 * Print the dentry name for named mappings, and a 427 * special [heap] marker for the heap: 428 */ 429 if (vma->vm_file) { 430 /* 431 * If user named this anon shared memory via 432 * prctl(PR_SET_VMA ..., use the provided name. 433 */ 434 if (anon_name) { 435 *name_fmt = "[anon_shmem:%s]"; 436 *name = anon_name->name; 437 } else { 438 *path = file_user_path(vma->vm_file); 439 } 440 return; 441 } 442 443 if (vma->vm_ops && vma->vm_ops->name) { 444 *name = vma->vm_ops->name(vma); 445 if (*name) 446 return; 447 } 448 449 *name = arch_vma_name(vma); 450 if (*name) 451 return; 452 453 if (!vma->vm_mm) { 454 *name = "[vdso]"; 455 return; 456 } 457 458 if (vma_is_initial_heap(vma)) { 459 *name = "[heap]"; 460 return; 461 } 462 463 if (vma_is_initial_stack(vma)) { 464 *name = "[stack]"; 465 return; 466 } 467 468 if (anon_name) { 469 *name_fmt = "[anon:%s]"; 470 *name = anon_name->name; 471 return; 472 } 473 } 474 475 static void show_vma_header_prefix(struct seq_file *m, 476 unsigned long start, unsigned long end, 477 vm_flags_t flags, unsigned long long pgoff, 478 dev_t dev, u64 ino) 479 { 480 seq_setwidth(m, 25 + sizeof(void *) * 6 - 1); 481 seq_put_hex_ll(m, NULL, start, 8); 482 seq_put_hex_ll(m, "-", end, 8); 483 seq_putc(m, ' '); 484 seq_putc(m, flags & VM_READ ? 'r' : '-'); 485 seq_putc(m, flags & VM_WRITE ? 'w' : '-'); 486 seq_putc(m, flags & VM_EXEC ? 'x' : '-'); 487 seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p'); 488 seq_put_hex_ll(m, " ", pgoff, 8); 489 seq_put_hex_ll(m, " ", MAJOR(dev), 2); 490 seq_put_hex_ll(m, ":", MINOR(dev), 2); 491 seq_put_decimal_ull(m, " ", ino); 492 seq_putc(m, ' '); 493 } 494 495 static void 496 show_map_vma(struct seq_file *m, struct vm_area_struct *vma) 497 { 498 const struct path *path; 499 const char *name_fmt, *name; 500 vm_flags_t flags = vma->vm_flags; 501 u64 ino = 0; 502 unsigned long long pgoff = 0; 503 unsigned long start, end; 504 dev_t dev = 0; 505 506 if (vma->vm_file) { 507 const struct inode *inode = file_user_inode(vma->vm_file); 508 509 dev = inode->i_sb->s_dev; 510 ino = inode->i_ino; 511 pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT; 512 } 513 514 start = vma->vm_start; 515 end = vma->vm_end; 516 show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino); 517 518 get_vma_name(vma, &path, &name, &name_fmt); 519 if (path) { 520 seq_pad(m, ' '); 521 seq_path(m, path, "\n"); 522 } else if (name_fmt) { 523 seq_pad(m, ' '); 524 seq_printf(m, name_fmt, name); 525 } else if (name) { 526 seq_pad(m, ' '); 527 seq_puts(m, name); 528 } 529 seq_putc(m, '\n'); 530 } 531 532 static int show_map(struct seq_file *m, void *v) 533 { 534 show_map_vma(m, v); 535 return 0; 536 } 537 538 static const struct seq_operations proc_pid_maps_op = { 539 .start = m_start, 540 .next = m_next, 541 .stop = m_stop, 542 .show = show_map 543 }; 544 545 static int pid_maps_open(struct inode *inode, struct file *file) 546 { 547 return do_maps_open(inode, file, &proc_pid_maps_op); 548 } 549 550 #define PROCMAP_QUERY_VMA_FLAGS ( \ 551 PROCMAP_QUERY_VMA_READABLE | \ 552 PROCMAP_QUERY_VMA_WRITABLE | \ 553 PROCMAP_QUERY_VMA_EXECUTABLE | \ 554 PROCMAP_QUERY_VMA_SHARED \ 555 ) 556 557 #define PROCMAP_QUERY_VALID_FLAGS_MASK ( \ 558 PROCMAP_QUERY_COVERING_OR_NEXT_VMA | \ 559 PROCMAP_QUERY_FILE_BACKED_VMA | \ 560 PROCMAP_QUERY_VMA_FLAGS \ 561 ) 562 563 #ifdef CONFIG_PER_VMA_LOCK 564 565 static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx) 566 { 567 reset_lock_ctx(lock_ctx); 568 569 return 0; 570 } 571 572 static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx) 573 { 574 if (lock_ctx->mmap_locked) 575 unlock_ctx_mm(lock_ctx); 576 else 577 unlock_ctx_vma(lock_ctx); 578 } 579 580 static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx, 581 unsigned long addr) 582 { 583 struct mm_struct *mm = lock_ctx->mm; 584 struct vm_area_struct *vma; 585 struct vma_iterator vmi; 586 587 if (lock_ctx->mmap_locked) 588 return find_vma(mm, addr); 589 590 /* Unlock previously locked VMA and find the next one under RCU */ 591 unlock_ctx_vma(lock_ctx); 592 rcu_read_lock(); 593 vma_iter_init(&vmi, mm, addr); 594 vma = lock_next_vma(mm, &vmi, addr); 595 rcu_read_unlock(); 596 597 if (!vma) 598 return NULL; 599 600 if (!IS_ERR(vma)) { 601 lock_ctx->locked_vma = vma; 602 return vma; 603 } 604 605 if (PTR_ERR(vma) == -EAGAIN) { 606 /* Fallback to mmap_lock on vma->vm_refcnt overflow */ 607 mmap_read_lock(mm); 608 vma = find_vma(mm, addr); 609 lock_ctx->mmap_locked = true; 610 } 611 612 return vma; 613 } 614 615 #else /* CONFIG_PER_VMA_LOCK */ 616 617 static int query_vma_setup(struct proc_maps_locking_ctx *lock_ctx) 618 { 619 return mmap_read_lock_killable(lock_ctx->mm); 620 } 621 622 static void query_vma_teardown(struct proc_maps_locking_ctx *lock_ctx) 623 { 624 mmap_read_unlock(lock_ctx->mm); 625 } 626 627 static struct vm_area_struct *query_vma_find_by_addr(struct proc_maps_locking_ctx *lock_ctx, 628 unsigned long addr) 629 { 630 return find_vma(lock_ctx->mm, addr); 631 } 632 633 #endif /* CONFIG_PER_VMA_LOCK */ 634 635 static struct vm_area_struct *query_matching_vma(struct proc_maps_locking_ctx *lock_ctx, 636 unsigned long addr, u32 flags) 637 { 638 struct vm_area_struct *vma; 639 640 next_vma: 641 vma = query_vma_find_by_addr(lock_ctx, addr); 642 if (IS_ERR(vma)) 643 return vma; 644 645 if (!vma) 646 goto no_vma; 647 648 /* user requested only file-backed VMA, keep iterating */ 649 if ((flags & PROCMAP_QUERY_FILE_BACKED_VMA) && !vma->vm_file) 650 goto skip_vma; 651 652 /* VMA permissions should satisfy query flags */ 653 if (flags & PROCMAP_QUERY_VMA_FLAGS) { 654 u32 perm = 0; 655 656 if (flags & PROCMAP_QUERY_VMA_READABLE) 657 perm |= VM_READ; 658 if (flags & PROCMAP_QUERY_VMA_WRITABLE) 659 perm |= VM_WRITE; 660 if (flags & PROCMAP_QUERY_VMA_EXECUTABLE) 661 perm |= VM_EXEC; 662 if (flags & PROCMAP_QUERY_VMA_SHARED) 663 perm |= VM_MAYSHARE; 664 665 if ((vma->vm_flags & perm) != perm) 666 goto skip_vma; 667 } 668 669 /* found covering VMA or user is OK with the matching next VMA */ 670 if ((flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA) || vma->vm_start <= addr) 671 return vma; 672 673 skip_vma: 674 /* 675 * If the user needs closest matching VMA, keep iterating. 676 */ 677 addr = vma->vm_end; 678 if (flags & PROCMAP_QUERY_COVERING_OR_NEXT_VMA) 679 goto next_vma; 680 681 no_vma: 682 return ERR_PTR(-ENOENT); 683 } 684 685 static int do_procmap_query(struct mm_struct *mm, void __user *uarg) 686 { 687 struct proc_maps_locking_ctx lock_ctx = { .mm = mm }; 688 struct procmap_query karg; 689 struct vm_area_struct *vma; 690 struct file *vm_file = NULL; 691 const char *name = NULL; 692 char build_id_buf[BUILD_ID_SIZE_MAX], *name_buf = NULL; 693 __u64 usize; 694 int err; 695 696 if (copy_from_user(&usize, (void __user *)uarg, sizeof(usize))) 697 return -EFAULT; 698 /* argument struct can never be that large, reject abuse */ 699 if (usize > PAGE_SIZE) 700 return -E2BIG; 701 /* argument struct should have at least query_flags and query_addr fields */ 702 if (usize < offsetofend(struct procmap_query, query_addr)) 703 return -EINVAL; 704 err = copy_struct_from_user(&karg, sizeof(karg), uarg, usize); 705 if (err) 706 return err; 707 708 /* reject unknown flags */ 709 if (karg.query_flags & ~PROCMAP_QUERY_VALID_FLAGS_MASK) 710 return -EINVAL; 711 /* either both buffer address and size are set, or both should be zero */ 712 if (!!karg.vma_name_size != !!karg.vma_name_addr) 713 return -EINVAL; 714 if (!!karg.build_id_size != !!karg.build_id_addr) 715 return -EINVAL; 716 717 if (!mm || !mmget_not_zero(mm)) 718 return -ESRCH; 719 720 err = query_vma_setup(&lock_ctx); 721 if (err) { 722 mmput(mm); 723 return err; 724 } 725 726 vma = query_matching_vma(&lock_ctx, karg.query_addr, karg.query_flags); 727 if (IS_ERR(vma)) { 728 err = PTR_ERR(vma); 729 vma = NULL; 730 goto out; 731 } 732 733 karg.vma_start = vma->vm_start; 734 karg.vma_end = vma->vm_end; 735 736 karg.vma_flags = 0; 737 if (vma->vm_flags & VM_READ) 738 karg.vma_flags |= PROCMAP_QUERY_VMA_READABLE; 739 if (vma->vm_flags & VM_WRITE) 740 karg.vma_flags |= PROCMAP_QUERY_VMA_WRITABLE; 741 if (vma->vm_flags & VM_EXEC) 742 karg.vma_flags |= PROCMAP_QUERY_VMA_EXECUTABLE; 743 if (vma->vm_flags & VM_MAYSHARE) 744 karg.vma_flags |= PROCMAP_QUERY_VMA_SHARED; 745 746 karg.vma_page_size = vma_kernel_pagesize(vma); 747 748 if (vma->vm_file) { 749 const struct inode *inode = file_user_inode(vma->vm_file); 750 751 karg.vma_offset = ((__u64)vma->vm_pgoff) << PAGE_SHIFT; 752 karg.dev_major = MAJOR(inode->i_sb->s_dev); 753 karg.dev_minor = MINOR(inode->i_sb->s_dev); 754 karg.inode = inode->i_ino; 755 } else { 756 karg.vma_offset = 0; 757 karg.dev_major = 0; 758 karg.dev_minor = 0; 759 karg.inode = 0; 760 } 761 762 if (karg.vma_name_size) { 763 size_t name_buf_sz = min_t(size_t, PATH_MAX, karg.vma_name_size); 764 const struct path *path; 765 const char *name_fmt; 766 size_t name_sz = 0; 767 768 get_vma_name(vma, &path, &name, &name_fmt); 769 770 if (path || name_fmt || name) { 771 name_buf = kmalloc(name_buf_sz, GFP_KERNEL); 772 if (!name_buf) { 773 err = -ENOMEM; 774 goto out; 775 } 776 } 777 if (path) { 778 name = d_path(path, name_buf, name_buf_sz); 779 if (IS_ERR(name)) { 780 err = PTR_ERR(name); 781 goto out; 782 } 783 name_sz = name_buf + name_buf_sz - name; 784 } else if (name || name_fmt) { 785 name_sz = 1 + snprintf(name_buf, name_buf_sz, name_fmt ?: "%s", name); 786 name = name_buf; 787 } 788 if (name_sz > name_buf_sz) { 789 err = -ENAMETOOLONG; 790 goto out; 791 } 792 karg.vma_name_size = name_sz; 793 } 794 795 if (karg.build_id_size && vma->vm_file) 796 vm_file = get_file(vma->vm_file); 797 798 /* unlock vma or mmap_lock, and put mm_struct before copying data to user */ 799 query_vma_teardown(&lock_ctx); 800 mmput(mm); 801 802 if (karg.build_id_size) { 803 __u32 build_id_sz; 804 805 if (vm_file) 806 err = build_id_parse_file(vm_file, build_id_buf, &build_id_sz); 807 else 808 err = -ENOENT; 809 if (err) { 810 karg.build_id_size = 0; 811 } else { 812 if (karg.build_id_size < build_id_sz) { 813 err = -ENAMETOOLONG; 814 goto out_file; 815 } 816 karg.build_id_size = build_id_sz; 817 } 818 } 819 820 if (vm_file) 821 fput(vm_file); 822 823 if (karg.vma_name_size && copy_to_user(u64_to_user_ptr(karg.vma_name_addr), 824 name, karg.vma_name_size)) { 825 kfree(name_buf); 826 return -EFAULT; 827 } 828 kfree(name_buf); 829 830 if (karg.build_id_size && copy_to_user(u64_to_user_ptr(karg.build_id_addr), 831 build_id_buf, karg.build_id_size)) 832 return -EFAULT; 833 834 if (copy_to_user(uarg, &karg, min_t(size_t, sizeof(karg), usize))) 835 return -EFAULT; 836 837 return 0; 838 839 out: 840 query_vma_teardown(&lock_ctx); 841 mmput(mm); 842 out_file: 843 if (vm_file) 844 fput(vm_file); 845 kfree(name_buf); 846 return err; 847 } 848 849 static long procfs_procmap_ioctl(struct file *file, unsigned int cmd, unsigned long arg) 850 { 851 struct seq_file *seq = file->private_data; 852 struct proc_maps_private *priv = seq->private; 853 854 switch (cmd) { 855 case PROCMAP_QUERY: 856 /* priv->lock_ctx.mm is set during file open operation */ 857 return do_procmap_query(priv->lock_ctx.mm, (void __user *)arg); 858 default: 859 return -ENOIOCTLCMD; 860 } 861 } 862 863 const struct file_operations proc_pid_maps_operations = { 864 .open = pid_maps_open, 865 .read = seq_read, 866 .llseek = seq_lseek, 867 .release = proc_map_release, 868 .unlocked_ioctl = procfs_procmap_ioctl, 869 .compat_ioctl = compat_ptr_ioctl, 870 }; 871 872 /* 873 * Proportional Set Size(PSS): my share of RSS. 874 * 875 * PSS of a process is the count of pages it has in memory, where each 876 * page is divided by the number of processes sharing it. So if a 877 * process has 1000 pages all to itself, and 1000 shared with one other 878 * process, its PSS will be 1500. 879 * 880 * To keep (accumulated) division errors low, we adopt a 64bit 881 * fixed-point pss counter to minimize division errors. So (pss >> 882 * PSS_SHIFT) would be the real byte count. 883 * 884 * A shift of 12 before division means (assuming 4K page size): 885 * - 1M 3-user-pages add up to 8KB errors; 886 * - supports mapcount up to 2^24, or 16M; 887 * - supports PSS up to 2^52 bytes, or 4PB. 888 */ 889 #define PSS_SHIFT 12 890 891 #ifdef CONFIG_PROC_PAGE_MONITOR 892 struct mem_size_stats { 893 unsigned long resident; 894 unsigned long shared_clean; 895 unsigned long shared_dirty; 896 unsigned long private_clean; 897 unsigned long private_dirty; 898 unsigned long referenced; 899 unsigned long anonymous; 900 unsigned long lazyfree; 901 unsigned long anonymous_thp; 902 unsigned long shmem_thp; 903 unsigned long file_thp; 904 unsigned long swap; 905 unsigned long shared_hugetlb; 906 unsigned long private_hugetlb; 907 unsigned long ksm; 908 u64 pss; 909 u64 pss_anon; 910 u64 pss_file; 911 u64 pss_shmem; 912 u64 pss_dirty; 913 u64 pss_locked; 914 u64 swap_pss; 915 }; 916 917 static void smaps_page_accumulate(struct mem_size_stats *mss, 918 struct folio *folio, unsigned long size, unsigned long pss, 919 bool dirty, bool locked, bool private) 920 { 921 mss->pss += pss; 922 923 if (folio_test_anon(folio)) 924 mss->pss_anon += pss; 925 else if (folio_test_swapbacked(folio)) 926 mss->pss_shmem += pss; 927 else 928 mss->pss_file += pss; 929 930 if (locked) 931 mss->pss_locked += pss; 932 933 if (dirty || folio_test_dirty(folio)) { 934 mss->pss_dirty += pss; 935 if (private) 936 mss->private_dirty += size; 937 else 938 mss->shared_dirty += size; 939 } else { 940 if (private) 941 mss->private_clean += size; 942 else 943 mss->shared_clean += size; 944 } 945 } 946 947 static void smaps_account(struct mem_size_stats *mss, struct page *page, 948 bool compound, bool young, bool dirty, bool locked, 949 bool present) 950 { 951 struct folio *folio = page_folio(page); 952 int i, nr = compound ? compound_nr(page) : 1; 953 unsigned long size = nr * PAGE_SIZE; 954 bool exclusive; 955 int mapcount; 956 957 /* 958 * First accumulate quantities that depend only on |size| and the type 959 * of the compound page. 960 */ 961 if (folio_test_anon(folio)) { 962 mss->anonymous += size; 963 if (!folio_test_swapbacked(folio) && !dirty && 964 !folio_test_dirty(folio)) 965 mss->lazyfree += size; 966 } 967 968 if (folio_test_ksm(folio)) 969 mss->ksm += size; 970 971 mss->resident += size; 972 /* Accumulate the size in pages that have been accessed. */ 973 if (young || folio_test_young(folio) || folio_test_referenced(folio)) 974 mss->referenced += size; 975 976 /* 977 * Then accumulate quantities that may depend on sharing, or that may 978 * differ page-by-page. 979 * 980 * refcount == 1 for present entries guarantees that the folio is mapped 981 * exactly once. For large folios this implies that exactly one 982 * PTE/PMD/... maps (a part of) this folio. 983 * 984 * Treat all non-present entries (where relying on the mapcount and 985 * refcount doesn't make sense) as "maybe shared, but not sure how 986 * often". We treat device private entries as being fake-present. 987 * 988 * Note that it would not be safe to read the mapcount especially for 989 * pages referenced by migration entries, even with the PTL held. 990 */ 991 if (folio_ref_count(folio) == 1 || !present) { 992 smaps_page_accumulate(mss, folio, size, size << PSS_SHIFT, 993 dirty, locked, present); 994 return; 995 } 996 997 if (IS_ENABLED(CONFIG_NO_PAGE_MAPCOUNT)) { 998 mapcount = folio_average_page_mapcount(folio); 999 exclusive = !folio_maybe_mapped_shared(folio); 1000 } 1001 1002 /* 1003 * We obtain a snapshot of the mapcount. Without holding the folio lock 1004 * this snapshot can be slightly wrong as we cannot always read the 1005 * mapcount atomically. 1006 */ 1007 for (i = 0; i < nr; i++, page++) { 1008 unsigned long pss = PAGE_SIZE << PSS_SHIFT; 1009 1010 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) { 1011 mapcount = folio_precise_page_mapcount(folio, page); 1012 exclusive = mapcount < 2; 1013 } 1014 1015 if (mapcount >= 2) 1016 pss /= mapcount; 1017 smaps_page_accumulate(mss, folio, PAGE_SIZE, pss, 1018 dirty, locked, exclusive); 1019 } 1020 } 1021 1022 #ifdef CONFIG_SHMEM 1023 static int smaps_pte_hole(unsigned long addr, unsigned long end, 1024 __always_unused int depth, struct mm_walk *walk) 1025 { 1026 struct mem_size_stats *mss = walk->private; 1027 struct vm_area_struct *vma = walk->vma; 1028 1029 mss->swap += shmem_partial_swap_usage(walk->vma->vm_file->f_mapping, 1030 linear_page_index(vma, addr), 1031 linear_page_index(vma, end)); 1032 1033 return 0; 1034 } 1035 #else 1036 #define smaps_pte_hole NULL 1037 #endif /* CONFIG_SHMEM */ 1038 1039 static void smaps_pte_hole_lookup(unsigned long addr, struct mm_walk *walk) 1040 { 1041 #ifdef CONFIG_SHMEM 1042 if (walk->ops->pte_hole) { 1043 /* depth is not used */ 1044 smaps_pte_hole(addr, addr + PAGE_SIZE, 0, walk); 1045 } 1046 #endif 1047 } 1048 1049 static void smaps_pte_entry(pte_t *pte, unsigned long addr, 1050 struct mm_walk *walk) 1051 { 1052 struct mem_size_stats *mss = walk->private; 1053 struct vm_area_struct *vma = walk->vma; 1054 bool locked = !!(vma->vm_flags & VM_LOCKED); 1055 struct page *page = NULL; 1056 bool present = false, young = false, dirty = false; 1057 pte_t ptent = ptep_get(pte); 1058 1059 if (pte_present(ptent)) { 1060 page = vm_normal_page(vma, addr, ptent); 1061 young = pte_young(ptent); 1062 dirty = pte_dirty(ptent); 1063 present = true; 1064 } else if (pte_none(ptent)) { 1065 smaps_pte_hole_lookup(addr, walk); 1066 } else { 1067 const softleaf_t entry = softleaf_from_pte(ptent); 1068 1069 if (softleaf_is_swap(entry)) { 1070 int mapcount; 1071 1072 mss->swap += PAGE_SIZE; 1073 mapcount = swp_swapcount(entry); 1074 if (mapcount >= 2) { 1075 u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT; 1076 1077 do_div(pss_delta, mapcount); 1078 mss->swap_pss += pss_delta; 1079 } else { 1080 mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT; 1081 } 1082 } else if (softleaf_has_pfn(entry)) { 1083 if (softleaf_is_device_private(entry)) 1084 present = true; 1085 page = softleaf_to_page(entry); 1086 } 1087 } 1088 1089 if (!page) 1090 return; 1091 1092 smaps_account(mss, page, false, young, dirty, locked, present); 1093 } 1094 1095 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1096 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, 1097 struct mm_walk *walk) 1098 { 1099 struct mem_size_stats *mss = walk->private; 1100 struct vm_area_struct *vma = walk->vma; 1101 bool locked = !!(vma->vm_flags & VM_LOCKED); 1102 struct page *page = NULL; 1103 bool present = false; 1104 struct folio *folio; 1105 1106 if (pmd_none(*pmd)) 1107 return; 1108 if (pmd_present(*pmd)) { 1109 page = vm_normal_page_pmd(vma, addr, *pmd); 1110 present = true; 1111 } else if (unlikely(thp_migration_supported())) { 1112 const softleaf_t entry = softleaf_from_pmd(*pmd); 1113 1114 if (softleaf_has_pfn(entry)) 1115 page = softleaf_to_page(entry); 1116 } 1117 if (IS_ERR_OR_NULL(page)) 1118 return; 1119 folio = page_folio(page); 1120 if (folio_test_anon(folio)) 1121 mss->anonymous_thp += HPAGE_PMD_SIZE; 1122 else if (folio_test_swapbacked(folio)) 1123 mss->shmem_thp += HPAGE_PMD_SIZE; 1124 else if (folio_is_zone_device(folio)) 1125 /* pass */; 1126 else 1127 mss->file_thp += HPAGE_PMD_SIZE; 1128 1129 smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd), 1130 locked, present); 1131 } 1132 #else 1133 static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, 1134 struct mm_walk *walk) 1135 { 1136 } 1137 #endif 1138 1139 static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, 1140 struct mm_walk *walk) 1141 { 1142 struct vm_area_struct *vma = walk->vma; 1143 pte_t *pte; 1144 spinlock_t *ptl; 1145 1146 ptl = pmd_trans_huge_lock(pmd, vma); 1147 if (ptl) { 1148 smaps_pmd_entry(pmd, addr, walk); 1149 spin_unlock(ptl); 1150 goto out; 1151 } 1152 1153 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 1154 if (!pte) { 1155 walk->action = ACTION_AGAIN; 1156 return 0; 1157 } 1158 for (; addr != end; pte++, addr += PAGE_SIZE) 1159 smaps_pte_entry(pte, addr, walk); 1160 pte_unmap_unlock(pte - 1, ptl); 1161 out: 1162 cond_resched(); 1163 return 0; 1164 } 1165 1166 static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma) 1167 { 1168 /* 1169 * Don't forget to update Documentation/ on changes. 1170 * 1171 * The length of the second argument of mnemonics[] 1172 * needs to be 3 instead of previously set 2 1173 * (i.e. from [BITS_PER_LONG][2] to [BITS_PER_LONG][3]) 1174 * to avoid spurious 1175 * -Werror=unterminated-string-initialization warning 1176 * with GCC 15 1177 */ 1178 static const char mnemonics[BITS_PER_LONG][3] = { 1179 /* 1180 * In case if we meet a flag we don't know about. 1181 */ 1182 [0 ... (BITS_PER_LONG-1)] = "??", 1183 1184 [ilog2(VM_READ)] = "rd", 1185 [ilog2(VM_WRITE)] = "wr", 1186 [ilog2(VM_EXEC)] = "ex", 1187 [ilog2(VM_SHARED)] = "sh", 1188 [ilog2(VM_MAYREAD)] = "mr", 1189 [ilog2(VM_MAYWRITE)] = "mw", 1190 [ilog2(VM_MAYEXEC)] = "me", 1191 [ilog2(VM_MAYSHARE)] = "ms", 1192 [ilog2(VM_GROWSDOWN)] = "gd", 1193 [ilog2(VM_PFNMAP)] = "pf", 1194 [ilog2(VM_MAYBE_GUARD)] = "gu", 1195 [ilog2(VM_LOCKED)] = "lo", 1196 [ilog2(VM_IO)] = "io", 1197 [ilog2(VM_SEQ_READ)] = "sr", 1198 [ilog2(VM_RAND_READ)] = "rr", 1199 [ilog2(VM_DONTCOPY)] = "dc", 1200 [ilog2(VM_DONTEXPAND)] = "de", 1201 [ilog2(VM_LOCKONFAULT)] = "lf", 1202 [ilog2(VM_ACCOUNT)] = "ac", 1203 [ilog2(VM_NORESERVE)] = "nr", 1204 [ilog2(VM_HUGETLB)] = "ht", 1205 [ilog2(VM_SYNC)] = "sf", 1206 [ilog2(VM_ARCH_1)] = "ar", 1207 [ilog2(VM_WIPEONFORK)] = "wf", 1208 [ilog2(VM_DONTDUMP)] = "dd", 1209 #ifdef CONFIG_ARM64_BTI 1210 [ilog2(VM_ARM64_BTI)] = "bt", 1211 #endif 1212 #ifdef CONFIG_MEM_SOFT_DIRTY 1213 [ilog2(VM_SOFTDIRTY)] = "sd", 1214 #endif 1215 [ilog2(VM_MIXEDMAP)] = "mm", 1216 [ilog2(VM_HUGEPAGE)] = "hg", 1217 [ilog2(VM_NOHUGEPAGE)] = "nh", 1218 [ilog2(VM_MERGEABLE)] = "mg", 1219 [ilog2(VM_UFFD_MISSING)]= "um", 1220 [ilog2(VM_UFFD_WP)] = "uw", 1221 #ifdef CONFIG_ARM64_MTE 1222 [ilog2(VM_MTE)] = "mt", 1223 [ilog2(VM_MTE_ALLOWED)] = "", 1224 #endif 1225 #ifdef CONFIG_ARCH_HAS_PKEYS 1226 /* These come out via ProtectionKey: */ 1227 [ilog2(VM_PKEY_BIT0)] = "", 1228 [ilog2(VM_PKEY_BIT1)] = "", 1229 [ilog2(VM_PKEY_BIT2)] = "", 1230 #if CONFIG_ARCH_PKEY_BITS > 3 1231 [ilog2(VM_PKEY_BIT3)] = "", 1232 #endif 1233 #if CONFIG_ARCH_PKEY_BITS > 4 1234 [ilog2(VM_PKEY_BIT4)] = "", 1235 #endif 1236 #endif /* CONFIG_ARCH_HAS_PKEYS */ 1237 #ifdef CONFIG_HAVE_ARCH_USERFAULTFD_MINOR 1238 [ilog2(VM_UFFD_MINOR)] = "ui", 1239 #endif /* CONFIG_HAVE_ARCH_USERFAULTFD_MINOR */ 1240 #ifdef CONFIG_USERFAULTFD_RWP 1241 [ilog2(VM_UFFD_RWP)] = "ur", 1242 #endif 1243 #ifdef CONFIG_ARCH_HAS_USER_SHADOW_STACK 1244 [ilog2(VM_SHADOW_STACK)] = "ss", 1245 #endif 1246 #if defined(CONFIG_64BIT) || defined(CONFIG_PPC32) 1247 [ilog2(VM_DROPPABLE)] = "dp", 1248 #endif 1249 #ifdef CONFIG_64BIT 1250 [ilog2(VM_SEALED)] = "sl", 1251 #endif 1252 }; 1253 size_t i; 1254 1255 seq_puts(m, "VmFlags: "); 1256 for (i = 0; i < BITS_PER_LONG; i++) { 1257 if (!mnemonics[i][0]) 1258 continue; 1259 if (vma->vm_flags & (1UL << i)) 1260 seq_printf(m, "%s ", mnemonics[i]); 1261 } 1262 seq_putc(m, '\n'); 1263 } 1264 1265 #ifdef CONFIG_HUGETLB_PAGE 1266 static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask, 1267 unsigned long addr, unsigned long end, 1268 struct mm_walk *walk) 1269 { 1270 struct mem_size_stats *mss = walk->private; 1271 struct vm_area_struct *vma = walk->vma; 1272 struct folio *folio = NULL; 1273 bool present = false; 1274 spinlock_t *ptl; 1275 pte_t ptent; 1276 1277 ptl = huge_pte_lock(hstate_vma(vma), walk->mm, pte); 1278 ptent = huge_ptep_get(walk->mm, addr, pte); 1279 if (pte_present(ptent)) { 1280 folio = page_folio(pte_page(ptent)); 1281 present = true; 1282 } else { 1283 const softleaf_t entry = softleaf_from_pte(ptent); 1284 1285 if (softleaf_has_pfn(entry)) 1286 folio = softleaf_to_folio(entry); 1287 } 1288 1289 if (folio) { 1290 /* We treat non-present entries as "maybe shared". */ 1291 if (!present || folio_maybe_mapped_shared(folio) || 1292 hugetlb_pmd_shared(pte)) 1293 mss->shared_hugetlb += huge_page_size(hstate_vma(vma)); 1294 else 1295 mss->private_hugetlb += huge_page_size(hstate_vma(vma)); 1296 } 1297 spin_unlock(ptl); 1298 return 0; 1299 } 1300 #else 1301 #define smaps_hugetlb_range NULL 1302 #endif /* HUGETLB_PAGE */ 1303 1304 static const struct mm_walk_ops smaps_walk_ops = { 1305 .pmd_entry = smaps_pte_range, 1306 .hugetlb_entry = smaps_hugetlb_range, 1307 .walk_lock = PGWALK_RDLOCK, 1308 }; 1309 1310 static const struct mm_walk_ops smaps_shmem_walk_ops = { 1311 .pmd_entry = smaps_pte_range, 1312 .hugetlb_entry = smaps_hugetlb_range, 1313 .pte_hole = smaps_pte_hole, 1314 .walk_lock = PGWALK_RDLOCK, 1315 }; 1316 1317 #ifdef CONFIG_PER_VMA_LOCK 1318 1319 static const struct mm_walk_ops smaps_walk_vma_lock_ops = { 1320 .pmd_entry = smaps_pte_range, 1321 .hugetlb_entry = smaps_hugetlb_range, 1322 .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, 1323 }; 1324 1325 static const struct mm_walk_ops smaps_shmem_walk_vma_lock_ops = { 1326 .pmd_entry = smaps_pte_range, 1327 .hugetlb_entry = smaps_hugetlb_range, 1328 .pte_hole = smaps_pte_hole, 1329 .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, 1330 }; 1331 1332 static inline const struct mm_walk_ops * 1333 get_smaps_walk_ops(struct proc_maps_private *priv) 1334 { 1335 if (priv->lock_ctx.mmap_locked) 1336 return &smaps_walk_ops; 1337 return &smaps_walk_vma_lock_ops; 1338 } 1339 1340 static inline const struct mm_walk_ops * 1341 get_smaps_shmem_walk_ops(struct proc_maps_private *priv) 1342 { 1343 if (priv->lock_ctx.mmap_locked) 1344 return &smaps_shmem_walk_ops; 1345 return &smaps_shmem_walk_vma_lock_ops; 1346 } 1347 1348 #else /* CONFIG_PER_VMA_LOCK */ 1349 1350 static inline const struct mm_walk_ops * 1351 get_smaps_walk_ops(struct proc_maps_private *priv) 1352 { 1353 return &smaps_walk_ops; 1354 } 1355 1356 static inline const struct mm_walk_ops * 1357 get_smaps_shmem_walk_ops(struct proc_maps_private *priv) 1358 { 1359 return &smaps_shmem_walk_ops; 1360 } 1361 1362 #endif /* CONFIG_PER_VMA_LOCK */ 1363 1364 /* 1365 * Gather mem stats from @vma with the indicated beginning 1366 * address @start, and keep them in @mss. 1367 * 1368 * Use vm_start of @vma as the beginning address if @start is 0. 1369 */ 1370 static void smap_gather_stats(struct proc_maps_private *priv, 1371 struct vm_area_struct *vma, 1372 struct mem_size_stats *mss, unsigned long start) 1373 { 1374 const struct mm_walk_ops *ops = get_smaps_walk_ops(priv); 1375 1376 /* Invalid start */ 1377 if (start >= vma->vm_end) 1378 return; 1379 1380 if (vma == get_gate_vma(priv->lock_ctx.mm)) 1381 return; 1382 1383 /* Might sleep. Drop RCU read lock but keep the VMA locked. */ 1384 drop_rcu(priv); 1385 1386 if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) { 1387 /* 1388 * For shared or readonly shmem mappings we know that all 1389 * swapped out pages belong to the shmem object, and we can 1390 * obtain the swap value much more efficiently. For private 1391 * writable mappings, we might have COW pages that are 1392 * not affected by the parent swapped out pages of the shmem 1393 * object, so we have to distinguish them during the page walk. 1394 * Unless we know that the shmem object (or the part mapped by 1395 * our VMA) has no swapped out pages at all. 1396 */ 1397 unsigned long shmem_swapped = shmem_swap_usage(vma); 1398 1399 if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) || 1400 !(vma->vm_flags & VM_WRITE))) { 1401 mss->swap += shmem_swapped; 1402 } else { 1403 ops = get_smaps_shmem_walk_ops(priv); 1404 } 1405 } 1406 1407 if (!start) 1408 walk_page_vma(vma, ops, mss); 1409 else 1410 walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss); 1411 1412 reacquire_rcu(priv); 1413 } 1414 1415 #define SEQ_PUT_DEC(str, val) \ 1416 seq_put_decimal_ull_width(m, str, (val) >> 10, 8) 1417 1418 /* Show the contents common for smaps and smaps_rollup */ 1419 static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss, 1420 bool rollup_mode) 1421 { 1422 SEQ_PUT_DEC("Rss: ", mss->resident); 1423 SEQ_PUT_DEC(" kB\nPss: ", mss->pss >> PSS_SHIFT); 1424 SEQ_PUT_DEC(" kB\nPss_Dirty: ", mss->pss_dirty >> PSS_SHIFT); 1425 if (rollup_mode) { 1426 /* 1427 * These are meaningful only for smaps_rollup, otherwise two of 1428 * them are zero, and the other one is the same as Pss. 1429 */ 1430 SEQ_PUT_DEC(" kB\nPss_Anon: ", 1431 mss->pss_anon >> PSS_SHIFT); 1432 SEQ_PUT_DEC(" kB\nPss_File: ", 1433 mss->pss_file >> PSS_SHIFT); 1434 SEQ_PUT_DEC(" kB\nPss_Shmem: ", 1435 mss->pss_shmem >> PSS_SHIFT); 1436 } 1437 SEQ_PUT_DEC(" kB\nShared_Clean: ", mss->shared_clean); 1438 SEQ_PUT_DEC(" kB\nShared_Dirty: ", mss->shared_dirty); 1439 SEQ_PUT_DEC(" kB\nPrivate_Clean: ", mss->private_clean); 1440 SEQ_PUT_DEC(" kB\nPrivate_Dirty: ", mss->private_dirty); 1441 SEQ_PUT_DEC(" kB\nReferenced: ", mss->referenced); 1442 SEQ_PUT_DEC(" kB\nAnonymous: ", mss->anonymous); 1443 SEQ_PUT_DEC(" kB\nKSM: ", mss->ksm); 1444 SEQ_PUT_DEC(" kB\nLazyFree: ", mss->lazyfree); 1445 SEQ_PUT_DEC(" kB\nAnonHugePages: ", mss->anonymous_thp); 1446 SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp); 1447 SEQ_PUT_DEC(" kB\nFilePmdMapped: ", mss->file_thp); 1448 SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb); 1449 seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ", 1450 mss->private_hugetlb >> 10, 7); 1451 SEQ_PUT_DEC(" kB\nSwap: ", mss->swap); 1452 SEQ_PUT_DEC(" kB\nSwapPss: ", 1453 mss->swap_pss >> PSS_SHIFT); 1454 SEQ_PUT_DEC(" kB\nLocked: ", 1455 mss->pss_locked >> PSS_SHIFT); 1456 seq_puts(m, " kB\n"); 1457 } 1458 1459 static int show_smap(struct seq_file *m, void *v) 1460 { 1461 struct proc_maps_private *priv = m->private; 1462 struct vm_area_struct *vma = v; 1463 struct mem_size_stats mss = {}; 1464 1465 smap_gather_stats(priv, vma, &mss, 0); 1466 1467 show_map_vma(m, vma); 1468 1469 SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start); 1470 SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma)); 1471 SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma)); 1472 seq_puts(m, " kB\n"); 1473 1474 __show_smap(m, &mss, false); 1475 1476 seq_printf(m, "THPeligible: %8u\n", 1477 !!thp_vma_allowable_orders(vma, vma->vm_flags, TVA_SMAPS, 1478 THP_ORDERS_ALL)); 1479 1480 if (arch_pkeys_enabled()) 1481 seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma)); 1482 show_smap_vma_flags(m, vma); 1483 1484 return 0; 1485 } 1486 1487 static int show_smaps_rollup(struct seq_file *m, void *v) 1488 { 1489 struct proc_maps_private *priv = m->private; 1490 struct mem_size_stats mss = {}; 1491 struct mm_struct *mm = priv->lock_ctx.mm; 1492 struct vm_area_struct *vma; 1493 unsigned long vma_start = 0, last_vma_end = 0; 1494 int ret = 0; 1495 VMA_ITERATOR(vmi, mm, 0); 1496 1497 priv->task = get_proc_task(priv->inode); 1498 if (!priv->task) 1499 return -ESRCH; 1500 1501 if (!mm || !mmget_not_zero(mm)) { 1502 ret = -ESRCH; 1503 goto out_put_task; 1504 } 1505 1506 ret = lock_ctx_mm(&priv->lock_ctx); 1507 if (ret) 1508 goto out_put_mm; 1509 1510 hold_task_mempolicy(priv); 1511 vma = vma_next(&vmi); 1512 1513 if (unlikely(!vma)) 1514 goto empty_set; 1515 1516 vma_start = vma->vm_start; 1517 do { 1518 smap_gather_stats(priv, vma, &mss, 0); 1519 last_vma_end = vma->vm_end; 1520 1521 /* 1522 * Release mmap_lock temporarily if someone wants to 1523 * access it for write request. 1524 */ 1525 if (mmap_lock_is_contended(mm)) { 1526 vma_iter_invalidate(&vmi); 1527 unlock_ctx_mm(&priv->lock_ctx); 1528 ret = lock_ctx_mm(&priv->lock_ctx); 1529 if (ret) { 1530 release_task_mempolicy(priv); 1531 goto out_put_mm; 1532 } 1533 1534 /* 1535 * After dropping the lock, there are four cases to 1536 * consider. See the following example for explanation. 1537 * 1538 * +------+------+-----------+ 1539 * | VMA1 | VMA2 | VMA3 | 1540 * +------+------+-----------+ 1541 * | | | | 1542 * 4k 8k 16k 400k 1543 * 1544 * Suppose we drop the lock after reading VMA2 due to 1545 * contention, then we get: 1546 * 1547 * last_vma_end = 16k 1548 * 1549 * 1) VMA2 is freed, but VMA3 exists: 1550 * 1551 * vma_next(vmi) will return VMA3. 1552 * In this case, just continue from VMA3. 1553 * 1554 * 2) VMA2 still exists: 1555 * 1556 * vma_next(vmi) will return VMA3. 1557 * In this case, just continue from VMA3. 1558 * 1559 * 3) No more VMAs can be found: 1560 * 1561 * vma_next(vmi) will return NULL. 1562 * No more things to do, just break. 1563 * 1564 * 4) (last_vma_end - 1) is the middle of a vma (VMA'): 1565 * 1566 * vma_next(vmi) will return VMA' whose range 1567 * contains last_vma_end. 1568 * Iterate VMA' from last_vma_end. 1569 */ 1570 vma = vma_next(&vmi); 1571 /* Case 3 above */ 1572 if (!vma) 1573 break; 1574 1575 /* Case 1 and 2 above */ 1576 if (vma->vm_start >= last_vma_end) { 1577 smap_gather_stats(priv, vma, &mss, 0); 1578 last_vma_end = vma->vm_end; 1579 continue; 1580 } 1581 1582 /* Case 4 above */ 1583 if (vma->vm_end > last_vma_end) { 1584 smap_gather_stats(priv, vma, &mss, last_vma_end); 1585 last_vma_end = vma->vm_end; 1586 } 1587 } 1588 } for_each_vma(vmi, vma); 1589 1590 empty_set: 1591 show_vma_header_prefix(m, vma_start, last_vma_end, 0, 0, 0, 0); 1592 seq_pad(m, ' '); 1593 seq_puts(m, "[rollup]\n"); 1594 1595 __show_smap(m, &mss, true); 1596 1597 release_task_mempolicy(priv); 1598 unlock_ctx_mm(&priv->lock_ctx); 1599 1600 out_put_mm: 1601 mmput(mm); 1602 out_put_task: 1603 put_task_struct(priv->task); 1604 priv->task = NULL; 1605 1606 return ret; 1607 } 1608 #undef SEQ_PUT_DEC 1609 1610 static const struct seq_operations proc_pid_smaps_op = { 1611 .start = m_start, 1612 .next = m_next, 1613 .stop = m_stop, 1614 .show = show_smap 1615 }; 1616 1617 static int pid_smaps_open(struct inode *inode, struct file *file) 1618 { 1619 return do_maps_open(inode, file, &proc_pid_smaps_op); 1620 } 1621 1622 static int smaps_rollup_open(struct inode *inode, struct file *file) 1623 { 1624 int ret; 1625 struct proc_maps_private *priv; 1626 1627 priv = kzalloc_obj(*priv, GFP_KERNEL_ACCOUNT); 1628 if (!priv) 1629 return -ENOMEM; 1630 1631 ret = single_open(file, show_smaps_rollup, priv); 1632 if (ret) 1633 goto out_free; 1634 1635 priv->inode = inode; 1636 priv->lock_ctx.mm = proc_mem_open(inode, PTRACE_MODE_READ); 1637 if (IS_ERR_OR_NULL(priv->lock_ctx.mm)) { 1638 ret = priv->lock_ctx.mm ? PTR_ERR(priv->lock_ctx.mm) : -ESRCH; 1639 1640 single_release(inode, file); 1641 goto out_free; 1642 } 1643 1644 return 0; 1645 1646 out_free: 1647 kfree(priv); 1648 return ret; 1649 } 1650 1651 static int smaps_rollup_release(struct inode *inode, struct file *file) 1652 { 1653 struct seq_file *seq = file->private_data; 1654 struct proc_maps_private *priv = seq->private; 1655 1656 if (priv->lock_ctx.mm) 1657 mmdrop(priv->lock_ctx.mm); 1658 1659 kfree(priv); 1660 return single_release(inode, file); 1661 } 1662 1663 const struct file_operations proc_pid_smaps_operations = { 1664 .open = pid_smaps_open, 1665 .read = seq_read, 1666 .llseek = seq_lseek, 1667 .release = proc_map_release, 1668 }; 1669 1670 const struct file_operations proc_pid_smaps_rollup_operations = { 1671 .open = smaps_rollup_open, 1672 .read = seq_read, 1673 .llseek = seq_lseek, 1674 .release = smaps_rollup_release, 1675 }; 1676 1677 enum clear_refs_types { 1678 CLEAR_REFS_ALL = 1, 1679 CLEAR_REFS_ANON, 1680 CLEAR_REFS_MAPPED, 1681 CLEAR_REFS_SOFT_DIRTY, 1682 CLEAR_REFS_MM_HIWATER_RSS, 1683 CLEAR_REFS_LAST, 1684 }; 1685 1686 struct clear_refs_private { 1687 enum clear_refs_types type; 1688 }; 1689 1690 static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte) 1691 { 1692 struct folio *folio; 1693 1694 if (!pte_write(pte)) 1695 return false; 1696 if (!vma_is_cow_mapping(vma)) 1697 return false; 1698 if (likely(!mm_flags_test(MMF_HAS_PINNED, vma->vm_mm))) 1699 return false; 1700 folio = vm_normal_folio(vma, addr, pte); 1701 if (!folio) 1702 return false; 1703 return folio_maybe_dma_pinned(folio); 1704 } 1705 1706 static inline void clear_soft_dirty(struct vm_area_struct *vma, 1707 unsigned long addr, pte_t *pte) 1708 { 1709 if (!pgtable_supports_soft_dirty()) 1710 return; 1711 /* 1712 * The soft-dirty tracker uses #PF-s to catch writes 1713 * to pages, so write-protect the pte as well. See the 1714 * Documentation/admin-guide/mm/soft-dirty.rst for full description 1715 * of how soft-dirty works. 1716 */ 1717 pte_t ptent = ptep_get(pte); 1718 1719 if (pte_none(ptent)) 1720 return; 1721 1722 if (pte_present(ptent)) { 1723 pte_t old_pte; 1724 1725 if (pte_is_pinned(vma, addr, ptent)) 1726 return; 1727 old_pte = ptep_modify_prot_start(vma, addr, pte); 1728 ptent = pte_wrprotect(old_pte); 1729 ptent = pte_clear_soft_dirty(ptent); 1730 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); 1731 } else { 1732 ptent = pte_swp_clear_soft_dirty(ptent); 1733 set_pte_at(vma->vm_mm, addr, pte, ptent); 1734 } 1735 } 1736 1737 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) 1738 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, 1739 unsigned long addr, pmd_t *pmdp) 1740 { 1741 pmd_t old, pmd = *pmdp; 1742 1743 if (!pgtable_supports_soft_dirty()) 1744 return; 1745 1746 if (pmd_present(pmd)) { 1747 /* See comment in change_huge_pmd() */ 1748 old = pmdp_invalidate(vma, addr, pmdp); 1749 if (pmd_dirty(old)) 1750 pmd = pmd_mkdirty(pmd); 1751 if (pmd_young(old)) 1752 pmd = pmd_mkyoung(pmd); 1753 1754 pmd = pmd_wrprotect(pmd); 1755 pmd = pmd_clear_soft_dirty(pmd); 1756 1757 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 1758 } else if (pmd_is_migration_entry(pmd)) { 1759 pmd = pmd_swp_clear_soft_dirty(pmd); 1760 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 1761 } 1762 } 1763 #else 1764 static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, 1765 unsigned long addr, pmd_t *pmdp) 1766 { 1767 } 1768 #endif 1769 1770 static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, 1771 unsigned long end, struct mm_walk *walk) 1772 { 1773 struct clear_refs_private *cp = walk->private; 1774 struct vm_area_struct *vma = walk->vma; 1775 pte_t *pte, ptent; 1776 spinlock_t *ptl; 1777 struct folio *folio; 1778 1779 ptl = pmd_trans_huge_lock(pmd, vma); 1780 if (ptl) { 1781 if (cp->type == CLEAR_REFS_SOFT_DIRTY) { 1782 clear_soft_dirty_pmd(vma, addr, pmd); 1783 goto out; 1784 } 1785 1786 if (!pmd_present(*pmd)) 1787 goto out; 1788 1789 folio = pmd_folio(*pmd); 1790 1791 /* Clear accessed and referenced bits. */ 1792 pmdp_test_and_clear_young(vma, addr, pmd); 1793 folio_test_clear_young(folio); 1794 folio_clear_referenced(folio); 1795 out: 1796 spin_unlock(ptl); 1797 return 0; 1798 } 1799 1800 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl); 1801 if (!pte) { 1802 walk->action = ACTION_AGAIN; 1803 return 0; 1804 } 1805 for (; addr != end; pte++, addr += PAGE_SIZE) { 1806 ptent = ptep_get(pte); 1807 1808 if (cp->type == CLEAR_REFS_SOFT_DIRTY) { 1809 clear_soft_dirty(vma, addr, pte); 1810 continue; 1811 } 1812 1813 if (!pte_present(ptent)) 1814 continue; 1815 1816 folio = vm_normal_folio(vma, addr, ptent); 1817 if (!folio) 1818 continue; 1819 1820 /* Clear accessed and referenced bits. */ 1821 ptep_test_and_clear_young(vma, addr, pte); 1822 folio_test_clear_young(folio); 1823 folio_clear_referenced(folio); 1824 } 1825 pte_unmap_unlock(pte - 1, ptl); 1826 cond_resched(); 1827 return 0; 1828 } 1829 1830 static int clear_refs_test_walk(unsigned long start, unsigned long end, 1831 struct mm_walk *walk) 1832 { 1833 struct clear_refs_private *cp = walk->private; 1834 struct vm_area_struct *vma = walk->vma; 1835 1836 if (vma->vm_flags & VM_PFNMAP) 1837 return 1; 1838 1839 /* 1840 * Writing 1 to /proc/pid/clear_refs affects all pages. 1841 * Writing 2 to /proc/pid/clear_refs only affects anonymous pages. 1842 * Writing 3 to /proc/pid/clear_refs only affects file mapped pages. 1843 * Writing 4 to /proc/pid/clear_refs affects all pages. 1844 */ 1845 if (cp->type == CLEAR_REFS_ANON && vma->vm_file) 1846 return 1; 1847 if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file) 1848 return 1; 1849 return 0; 1850 } 1851 1852 static const struct mm_walk_ops clear_refs_walk_ops = { 1853 .pmd_entry = clear_refs_pte_range, 1854 .test_walk = clear_refs_test_walk, 1855 .walk_lock = PGWALK_WRLOCK, 1856 }; 1857 1858 static ssize_t clear_refs_write(struct file *file, const char __user *buf, 1859 size_t count, loff_t *ppos) 1860 { 1861 struct task_struct *task; 1862 char buffer[PROC_NUMBUF] = {}; 1863 struct mm_struct *mm; 1864 struct vm_area_struct *vma; 1865 enum clear_refs_types type; 1866 int itype; 1867 int rv; 1868 1869 if (count > sizeof(buffer) - 1) 1870 count = sizeof(buffer) - 1; 1871 if (copy_from_user(buffer, buf, count)) 1872 return -EFAULT; 1873 rv = kstrtoint(strstrip(buffer), 10, &itype); 1874 if (rv < 0) 1875 return rv; 1876 type = (enum clear_refs_types)itype; 1877 if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST) 1878 return -EINVAL; 1879 1880 task = get_proc_task(file_inode(file)); 1881 if (!task) 1882 return -ESRCH; 1883 mm = get_task_mm(task); 1884 if (mm) { 1885 VMA_ITERATOR(vmi, mm, 0); 1886 struct mmu_notifier_range range; 1887 struct clear_refs_private cp = { 1888 .type = type, 1889 }; 1890 1891 if (mmap_write_lock_killable(mm)) { 1892 count = -EINTR; 1893 goto out_mm; 1894 } 1895 if (type == CLEAR_REFS_MM_HIWATER_RSS) { 1896 /* 1897 * Writing 5 to /proc/pid/clear_refs resets the peak 1898 * resident set size to this mm's current rss value. 1899 */ 1900 reset_mm_hiwater_rss(mm); 1901 goto out_unlock; 1902 } 1903 1904 if (type == CLEAR_REFS_SOFT_DIRTY) { 1905 for_each_vma(vmi, vma) { 1906 if (!(vma->vm_flags & VM_SOFTDIRTY)) 1907 continue; 1908 vm_flags_clear(vma, VM_SOFTDIRTY); 1909 vma_set_page_prot(vma); 1910 } 1911 1912 inc_tlb_flush_pending(mm); 1913 mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY, 1914 0, mm, 0, -1UL); 1915 mmu_notifier_invalidate_range_start(&range); 1916 } 1917 walk_page_range(mm, 0, -1, &clear_refs_walk_ops, &cp); 1918 if (type == CLEAR_REFS_SOFT_DIRTY) { 1919 mmu_notifier_invalidate_range_end(&range); 1920 flush_tlb_mm(mm); 1921 dec_tlb_flush_pending(mm); 1922 } 1923 out_unlock: 1924 mmap_write_unlock(mm); 1925 out_mm: 1926 mmput(mm); 1927 } 1928 put_task_struct(task); 1929 1930 return count; 1931 } 1932 1933 const struct file_operations proc_clear_refs_operations = { 1934 .write = clear_refs_write, 1935 .llseek = noop_llseek, 1936 }; 1937 1938 typedef struct { 1939 u64 pme; 1940 } pagemap_entry_t; 1941 1942 struct pagemapread { 1943 int pos, len; /* units: PM_ENTRY_BYTES, not bytes */ 1944 pagemap_entry_t *buffer; 1945 bool show_pfn; 1946 }; 1947 1948 #define PAGEMAP_WALK_SIZE (PMD_SIZE) 1949 #define PAGEMAP_WALK_MASK (PMD_MASK) 1950 1951 #define PM_ENTRY_BYTES sizeof(pagemap_entry_t) 1952 #define PM_PFRAME_BITS 55 1953 #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0) 1954 #define PM_SOFT_DIRTY BIT_ULL(55) 1955 #define PM_MMAP_EXCLUSIVE BIT_ULL(56) 1956 #define PM_UFFD_WP BIT_ULL(57) 1957 #define PM_GUARD_REGION BIT_ULL(58) 1958 #define PM_FILE BIT_ULL(61) 1959 #define PM_SWAP BIT_ULL(62) 1960 #define PM_PRESENT BIT_ULL(63) 1961 1962 #define PM_END_OF_BUFFER 1 1963 1964 static inline pagemap_entry_t make_pme(u64 frame, u64 flags) 1965 { 1966 return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags }; 1967 } 1968 1969 static int add_to_pagemap(pagemap_entry_t *pme, struct pagemapread *pm) 1970 { 1971 pm->buffer[pm->pos++] = *pme; 1972 if (pm->pos >= pm->len) 1973 return PM_END_OF_BUFFER; 1974 return 0; 1975 } 1976 1977 static bool __folio_page_mapped_exclusively(struct folio *folio, struct page *page) 1978 { 1979 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) 1980 return folio_precise_page_mapcount(folio, page) == 1; 1981 return !folio_maybe_mapped_shared(folio); 1982 } 1983 1984 static int pagemap_pte_hole(unsigned long start, unsigned long end, 1985 __always_unused int depth, struct mm_walk *walk) 1986 { 1987 struct pagemapread *pm = walk->private; 1988 unsigned long addr = start; 1989 int err = 0; 1990 1991 while (addr < end) { 1992 struct vm_area_struct *vma = find_vma(walk->mm, addr); 1993 pagemap_entry_t pme = make_pme(0, 0); 1994 /* End of address space hole, which we mark as non-present. */ 1995 unsigned long hole_end; 1996 1997 if (vma) 1998 hole_end = min(end, vma->vm_start); 1999 else 2000 hole_end = end; 2001 2002 for (; addr < hole_end; addr += PAGE_SIZE) { 2003 err = add_to_pagemap(&pme, pm); 2004 if (err) 2005 goto out; 2006 } 2007 2008 if (!vma) 2009 break; 2010 2011 /* Addresses in the VMA. */ 2012 if (vma->vm_flags & VM_SOFTDIRTY) 2013 pme = make_pme(0, PM_SOFT_DIRTY); 2014 for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) { 2015 err = add_to_pagemap(&pme, pm); 2016 if (err) 2017 goto out; 2018 } 2019 } 2020 out: 2021 return err; 2022 } 2023 2024 static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm, 2025 struct vm_area_struct *vma, unsigned long addr, pte_t pte) 2026 { 2027 u64 frame = 0, flags = 0; 2028 struct page *page = NULL; 2029 struct folio *folio; 2030 2031 if (pte_none(pte)) 2032 goto out; 2033 2034 if (pte_present(pte)) { 2035 if (pm->show_pfn) 2036 frame = pte_pfn(pte); 2037 flags |= PM_PRESENT; 2038 page = vm_normal_page(vma, addr, pte); 2039 if (pte_soft_dirty(pte)) 2040 flags |= PM_SOFT_DIRTY; 2041 if (pte_uffd(pte)) 2042 flags |= PM_UFFD_WP; 2043 } else { 2044 softleaf_t entry; 2045 2046 if (pte_swp_soft_dirty(pte)) 2047 flags |= PM_SOFT_DIRTY; 2048 if (pte_swp_uffd(pte)) 2049 flags |= PM_UFFD_WP; 2050 entry = softleaf_from_pte(pte); 2051 if (pm->show_pfn) { 2052 pgoff_t offset; 2053 2054 /* 2055 * For PFN swap offsets, keeping the offset field 2056 * to be PFN only to be compatible with old smaps. 2057 */ 2058 if (softleaf_has_pfn(entry)) 2059 offset = softleaf_to_pfn(entry); 2060 else 2061 offset = swp_offset(entry); 2062 frame = swp_type(entry) | 2063 (offset << MAX_SWAPFILES_SHIFT); 2064 } 2065 flags |= PM_SWAP; 2066 if (softleaf_has_pfn(entry)) 2067 page = softleaf_to_page(entry); 2068 if (softleaf_is_uffd_wp_marker(entry)) 2069 flags |= PM_UFFD_WP; 2070 if (softleaf_is_guard_marker(entry)) 2071 flags |= PM_GUARD_REGION; 2072 } 2073 2074 if (page) { 2075 folio = page_folio(page); 2076 if (!folio_test_anon(folio)) 2077 flags |= PM_FILE; 2078 if ((flags & PM_PRESENT) && 2079 __folio_page_mapped_exclusively(folio, page)) 2080 flags |= PM_MMAP_EXCLUSIVE; 2081 } 2082 2083 out: 2084 if (vma->vm_flags & VM_SOFTDIRTY) 2085 flags |= PM_SOFT_DIRTY; 2086 2087 return make_pme(frame, flags); 2088 } 2089 2090 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2091 static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, 2092 unsigned long end, struct vm_area_struct *vma, 2093 struct pagemapread *pm) 2094 { 2095 unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT; 2096 u64 flags = 0, frame = 0; 2097 pmd_t pmd = *pmdp; 2098 struct page *page = NULL; 2099 struct folio *folio = NULL; 2100 int err = 0; 2101 2102 if (vma->vm_flags & VM_SOFTDIRTY) 2103 flags |= PM_SOFT_DIRTY; 2104 2105 if (pmd_none(pmd)) 2106 goto populate_pagemap; 2107 2108 if (pmd_present(pmd)) { 2109 page = pmd_page(pmd); 2110 2111 flags |= PM_PRESENT; 2112 if (pmd_soft_dirty(pmd)) 2113 flags |= PM_SOFT_DIRTY; 2114 if (pmd_uffd(pmd)) 2115 flags |= PM_UFFD_WP; 2116 if (pm->show_pfn) 2117 frame = pmd_pfn(pmd) + idx; 2118 } else if (thp_migration_supported()) { 2119 const softleaf_t entry = softleaf_from_pmd(pmd); 2120 unsigned long offset; 2121 2122 if (pm->show_pfn) { 2123 if (softleaf_has_pfn(entry)) 2124 offset = softleaf_to_pfn(entry) + idx; 2125 else 2126 offset = swp_offset(entry) + idx; 2127 frame = swp_type(entry) | 2128 (offset << MAX_SWAPFILES_SHIFT); 2129 } 2130 flags |= PM_SWAP; 2131 if (pmd_swp_soft_dirty(pmd)) 2132 flags |= PM_SOFT_DIRTY; 2133 if (pmd_swp_uffd(pmd)) 2134 flags |= PM_UFFD_WP; 2135 if (softleaf_has_pfn(entry)) 2136 page = softleaf_to_page(entry); 2137 } 2138 2139 if (page) { 2140 folio = page_folio(page); 2141 if (!folio_test_anon(folio)) 2142 flags |= PM_FILE; 2143 } 2144 2145 populate_pagemap: 2146 for (; addr != end; addr += PAGE_SIZE, idx++) { 2147 u64 cur_flags = flags; 2148 pagemap_entry_t pme; 2149 2150 if (folio && (flags & PM_PRESENT) && 2151 __folio_page_mapped_exclusively(folio, page)) 2152 cur_flags |= PM_MMAP_EXCLUSIVE; 2153 2154 pme = make_pme(frame, cur_flags); 2155 err = add_to_pagemap(&pme, pm); 2156 if (err) 2157 break; 2158 if (pm->show_pfn) { 2159 if (flags & PM_PRESENT) 2160 frame++; 2161 else if (flags & PM_SWAP) 2162 frame += (1 << MAX_SWAPFILES_SHIFT); 2163 } 2164 } 2165 return err; 2166 } 2167 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2168 2169 static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end, 2170 struct mm_walk *walk) 2171 { 2172 struct vm_area_struct *vma = walk->vma; 2173 struct pagemapread *pm = walk->private; 2174 spinlock_t *ptl; 2175 pte_t *pte, *orig_pte; 2176 int err = 0; 2177 2178 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2179 ptl = pmd_trans_huge_lock(pmdp, vma); 2180 if (ptl) { 2181 err = pagemap_pmd_range_thp(pmdp, addr, end, vma, pm); 2182 spin_unlock(ptl); 2183 return err; 2184 } 2185 #endif 2186 2187 /* 2188 * We can assume that @vma always points to a valid one and @end never 2189 * goes beyond vma->vm_end. 2190 */ 2191 orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl); 2192 if (!pte) { 2193 walk->action = ACTION_AGAIN; 2194 return err; 2195 } 2196 for (; addr < end; pte++, addr += PAGE_SIZE) { 2197 pagemap_entry_t pme; 2198 2199 pme = pte_to_pagemap_entry(pm, vma, addr, ptep_get(pte)); 2200 err = add_to_pagemap(&pme, pm); 2201 if (err) 2202 break; 2203 } 2204 pte_unmap_unlock(orig_pte, ptl); 2205 2206 cond_resched(); 2207 2208 return err; 2209 } 2210 2211 #ifdef CONFIG_HUGETLB_PAGE 2212 /* This function walks within one hugetlb entry in the single call */ 2213 static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask, 2214 unsigned long addr, unsigned long end, 2215 struct mm_walk *walk) 2216 { 2217 struct pagemapread *pm = walk->private; 2218 struct vm_area_struct *vma = walk->vma; 2219 u64 flags = 0, frame = 0; 2220 spinlock_t *ptl; 2221 int err = 0; 2222 pte_t pte; 2223 2224 if (vma->vm_flags & VM_SOFTDIRTY) 2225 flags |= PM_SOFT_DIRTY; 2226 2227 ptl = huge_pte_lock(hstate_vma(vma), walk->mm, ptep); 2228 pte = huge_ptep_get(walk->mm, addr, ptep); 2229 if (pte_present(pte)) { 2230 struct folio *folio = page_folio(pte_page(pte)); 2231 2232 if (!folio_test_anon(folio)) 2233 flags |= PM_FILE; 2234 2235 if (!folio_maybe_mapped_shared(folio) && 2236 !hugetlb_pmd_shared(ptep)) 2237 flags |= PM_MMAP_EXCLUSIVE; 2238 2239 if (huge_pte_uffd(pte)) 2240 flags |= PM_UFFD_WP; 2241 2242 flags |= PM_PRESENT; 2243 if (pm->show_pfn) 2244 frame = pte_pfn(pte) + 2245 ((addr & ~hmask) >> PAGE_SHIFT); 2246 } else if (pte_swp_uffd_any(pte)) { 2247 flags |= PM_UFFD_WP; 2248 } 2249 2250 for (; addr != end; addr += PAGE_SIZE) { 2251 pagemap_entry_t pme = make_pme(frame, flags); 2252 2253 err = add_to_pagemap(&pme, pm); 2254 if (err) 2255 break; 2256 if (pm->show_pfn && (flags & PM_PRESENT)) 2257 frame++; 2258 } 2259 2260 spin_unlock(ptl); 2261 cond_resched(); 2262 2263 return err; 2264 } 2265 #else 2266 #define pagemap_hugetlb_range NULL 2267 #endif /* HUGETLB_PAGE */ 2268 2269 static const struct mm_walk_ops pagemap_ops = { 2270 .pmd_entry = pagemap_pmd_range, 2271 .pte_hole = pagemap_pte_hole, 2272 .hugetlb_entry = pagemap_hugetlb_range, 2273 .walk_lock = PGWALK_RDLOCK, 2274 }; 2275 2276 /* 2277 * /proc/pid/pagemap - an array mapping virtual pages to pfns 2278 * 2279 * For each page in the address space, this file contains one 64-bit entry 2280 * consisting of the following: 2281 * 2282 * Bits 0-54 page frame number (PFN) if present 2283 * Bits 0-4 swap type if swapped 2284 * Bits 5-54 swap offset if swapped 2285 * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst) 2286 * Bit 56 page exclusively mapped 2287 * Bit 57 pte is tracked by userfaultfd (uffd-wp or RWP) 2288 * Bit 58 pte is a guard region 2289 * Bits 59-60 zero 2290 * Bit 61 page is file-page or shared-anon 2291 * Bit 62 page swapped 2292 * Bit 63 page present 2293 * 2294 * If the page is not present but in swap, then the PFN contains an 2295 * encoding of the swap file number and the page's offset into the 2296 * swap. Unmapped pages return a null PFN. This allows determining 2297 * precisely which pages are mapped (or in swap) and comparing mapped 2298 * pages between processes. 2299 * 2300 * Efficient users of this interface will use /proc/pid/maps to 2301 * determine which areas of memory are actually mapped and llseek to 2302 * skip over unmapped regions. 2303 */ 2304 static ssize_t pagemap_read(struct file *file, char __user *buf, 2305 size_t count, loff_t *ppos) 2306 { 2307 struct mm_struct *mm = file->private_data; 2308 struct pagemapread pm; 2309 unsigned long src; 2310 unsigned long svpfn; 2311 unsigned long start_vaddr; 2312 unsigned long end_vaddr; 2313 int ret = 0, copied = 0; 2314 2315 if (!mm || !mmget_not_zero(mm)) 2316 goto out; 2317 2318 ret = -EINVAL; 2319 /* file position must be aligned */ 2320 if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES)) 2321 goto out_mm; 2322 2323 ret = 0; 2324 if (!count) 2325 goto out_mm; 2326 2327 /* do not disclose physical addresses: attack vector */ 2328 pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN); 2329 2330 pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT); 2331 pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL); 2332 ret = -ENOMEM; 2333 if (!pm.buffer) 2334 goto out_mm; 2335 2336 src = *ppos; 2337 svpfn = src / PM_ENTRY_BYTES; 2338 end_vaddr = mm->task_size; 2339 2340 /* watch out for wraparound */ 2341 start_vaddr = end_vaddr; 2342 if (svpfn <= (ULONG_MAX >> PAGE_SHIFT)) { 2343 unsigned long end; 2344 2345 ret = mmap_read_lock_killable(mm); 2346 if (ret) 2347 goto out_free; 2348 start_vaddr = untagged_addr_remote(mm, svpfn << PAGE_SHIFT); 2349 mmap_read_unlock(mm); 2350 2351 end = start_vaddr + ((count / PM_ENTRY_BYTES) << PAGE_SHIFT); 2352 if (end >= start_vaddr && end < mm->task_size) 2353 end_vaddr = end; 2354 } 2355 2356 /* Ensure the address is inside the task */ 2357 if (start_vaddr > mm->task_size) 2358 start_vaddr = end_vaddr; 2359 2360 ret = 0; 2361 while (count && (start_vaddr < end_vaddr)) { 2362 int len; 2363 unsigned long end; 2364 2365 pm.pos = 0; 2366 end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK; 2367 /* overflow ? */ 2368 if (end < start_vaddr || end > end_vaddr) 2369 end = end_vaddr; 2370 ret = mmap_read_lock_killable(mm); 2371 if (ret) 2372 goto out_free; 2373 ret = walk_page_range(mm, start_vaddr, end, &pagemap_ops, &pm); 2374 mmap_read_unlock(mm); 2375 start_vaddr = end; 2376 2377 len = min(count, PM_ENTRY_BYTES * pm.pos); 2378 if (copy_to_user(buf, pm.buffer, len)) { 2379 ret = -EFAULT; 2380 goto out_free; 2381 } 2382 copied += len; 2383 buf += len; 2384 count -= len; 2385 } 2386 *ppos += copied; 2387 if (!ret || ret == PM_END_OF_BUFFER) 2388 ret = copied; 2389 2390 out_free: 2391 kfree(pm.buffer); 2392 out_mm: 2393 mmput(mm); 2394 out: 2395 return ret; 2396 } 2397 2398 static int pagemap_open(struct inode *inode, struct file *file) 2399 { 2400 struct mm_struct *mm; 2401 2402 mm = proc_mem_open(inode, PTRACE_MODE_READ); 2403 if (IS_ERR_OR_NULL(mm)) 2404 return mm ? PTR_ERR(mm) : -ESRCH; 2405 file->private_data = mm; 2406 return 0; 2407 } 2408 2409 static int pagemap_release(struct inode *inode, struct file *file) 2410 { 2411 struct mm_struct *mm = file->private_data; 2412 2413 if (mm) 2414 mmdrop(mm); 2415 return 0; 2416 } 2417 2418 #define PM_SCAN_CATEGORIES (PAGE_IS_WPALLOWED | PAGE_IS_WRITTEN | \ 2419 PAGE_IS_FILE | PAGE_IS_PRESENT | \ 2420 PAGE_IS_SWAPPED | PAGE_IS_PFNZERO | \ 2421 PAGE_IS_HUGE | PAGE_IS_SOFT_DIRTY | \ 2422 PAGE_IS_GUARD | PAGE_IS_ACCESSED) 2423 #define PM_SCAN_FLAGS (PM_SCAN_WP_MATCHING | PM_SCAN_CHECK_WPASYNC) 2424 2425 struct pagemap_scan_private { 2426 struct pm_scan_arg arg; 2427 unsigned long masks_of_interest, cur_vma_category; 2428 struct page_region *vec_buf; 2429 unsigned long vec_buf_len, vec_buf_index, found_pages; 2430 struct page_region __user *vec_out; 2431 }; 2432 2433 static unsigned long pagemap_page_category(struct pagemap_scan_private *p, 2434 struct vm_area_struct *vma, 2435 unsigned long addr, pte_t pte) 2436 { 2437 unsigned long categories; 2438 2439 if (pte_none(pte)) { 2440 /* 2441 * An unpopulated pte carries no uffd bit, i.e. it is not 2442 * write-protected. The PAGE_IS_WRITTEN fast path in 2443 * pagemap_scan_pmd_entry() is now gated on a VM_UFFD_WP VMA; 2444 * gate the pte_none report here the same way so the two paths 2445 * still agree. RWP has no such fast path and an unpopulated 2446 * page is not part of the RWP working set, so it is reported as 2447 * neither. 2448 */ 2449 if (userfaultfd_wp(vma)) 2450 return PAGE_IS_WRITTEN; 2451 return 0; 2452 } 2453 2454 if (pte_present(pte)) { 2455 struct page *page; 2456 2457 categories = PAGE_IS_PRESENT; 2458 2459 if (!pte_uffd(pte)) { 2460 if (userfaultfd_wp(vma)) 2461 categories |= PAGE_IS_WRITTEN; 2462 if (userfaultfd_rwp(vma)) 2463 categories |= PAGE_IS_ACCESSED; 2464 } 2465 2466 if (p->masks_of_interest & PAGE_IS_FILE) { 2467 page = vm_normal_page(vma, addr, pte); 2468 if (page && !PageAnon(page)) 2469 categories |= PAGE_IS_FILE; 2470 } 2471 2472 if (is_zero_pfn(pte_pfn(pte))) 2473 categories |= PAGE_IS_PFNZERO; 2474 if (pte_soft_dirty(pte)) 2475 categories |= PAGE_IS_SOFT_DIRTY; 2476 } else { 2477 softleaf_t entry; 2478 2479 categories = PAGE_IS_SWAPPED; 2480 2481 if (!pte_swp_uffd_any(pte)) { 2482 if (userfaultfd_wp(vma)) 2483 categories |= PAGE_IS_WRITTEN; 2484 if (userfaultfd_rwp(vma)) 2485 categories |= PAGE_IS_ACCESSED; 2486 } 2487 2488 entry = softleaf_from_pte(pte); 2489 if (softleaf_is_guard_marker(entry)) 2490 categories |= PAGE_IS_GUARD; 2491 else if ((p->masks_of_interest & PAGE_IS_FILE) && 2492 softleaf_has_pfn(entry) && 2493 !folio_test_anon(softleaf_to_folio(entry))) 2494 categories |= PAGE_IS_FILE; 2495 2496 if (pte_swp_soft_dirty(pte)) 2497 categories |= PAGE_IS_SOFT_DIRTY; 2498 } 2499 2500 return categories; 2501 } 2502 2503 static void make_uffd_wp_pte(struct vm_area_struct *vma, 2504 unsigned long addr, pte_t *pte, pte_t ptent) 2505 { 2506 if (pte_present(ptent)) { 2507 pte_t old_pte; 2508 2509 old_pte = ptep_modify_prot_start(vma, addr, pte); 2510 ptent = pte_mkuffd(old_pte); 2511 ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent); 2512 } else if (pte_none(ptent)) { 2513 set_pte_at(vma->vm_mm, addr, pte, 2514 make_pte_marker(PTE_MARKER_UFFD_WP)); 2515 } else { 2516 ptent = pte_swp_mkuffd(ptent); 2517 set_pte_at(vma->vm_mm, addr, pte, ptent); 2518 } 2519 } 2520 2521 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2522 static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, 2523 struct vm_area_struct *vma, 2524 unsigned long addr, pmd_t pmd) 2525 { 2526 unsigned long categories = PAGE_IS_HUGE; 2527 2528 if (pmd_none(pmd)) 2529 return categories; 2530 2531 if (pmd_present(pmd)) { 2532 struct page *page; 2533 2534 categories |= PAGE_IS_PRESENT; 2535 if (!pmd_uffd(pmd)) { 2536 if (userfaultfd_wp(vma)) 2537 categories |= PAGE_IS_WRITTEN; 2538 if (userfaultfd_rwp(vma)) 2539 categories |= PAGE_IS_ACCESSED; 2540 } 2541 2542 if (p->masks_of_interest & PAGE_IS_FILE) { 2543 page = vm_normal_page_pmd(vma, addr, pmd); 2544 if (page && !PageAnon(page)) 2545 categories |= PAGE_IS_FILE; 2546 } 2547 2548 if (is_huge_zero_pmd(pmd)) 2549 categories |= PAGE_IS_PFNZERO; 2550 if (pmd_soft_dirty(pmd)) 2551 categories |= PAGE_IS_SOFT_DIRTY; 2552 } else { 2553 categories |= PAGE_IS_SWAPPED; 2554 if (!pmd_swp_uffd(pmd)) { 2555 if (userfaultfd_wp(vma)) 2556 categories |= PAGE_IS_WRITTEN; 2557 if (userfaultfd_rwp(vma)) 2558 categories |= PAGE_IS_ACCESSED; 2559 } 2560 if (pmd_swp_soft_dirty(pmd)) 2561 categories |= PAGE_IS_SOFT_DIRTY; 2562 2563 if (p->masks_of_interest & PAGE_IS_FILE) { 2564 const softleaf_t entry = softleaf_from_pmd(pmd); 2565 2566 if (softleaf_has_pfn(entry) && 2567 !folio_test_anon(softleaf_to_folio(entry))) 2568 categories |= PAGE_IS_FILE; 2569 } 2570 } 2571 2572 return categories; 2573 } 2574 2575 static void make_uffd_wp_pmd(struct vm_area_struct *vma, 2576 unsigned long addr, pmd_t *pmdp) 2577 { 2578 pmd_t old, pmd = *pmdp; 2579 2580 if (pmd_present(pmd)) { 2581 old = pmdp_invalidate_ad(vma, addr, pmdp); 2582 pmd = pmd_mkuffd(old); 2583 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 2584 } else if (pmd_is_migration_entry(pmd)) { 2585 pmd = pmd_swp_mkuffd(pmd); 2586 set_pmd_at(vma->vm_mm, addr, pmdp, pmd); 2587 } 2588 } 2589 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 2590 2591 #ifdef CONFIG_HUGETLB_PAGE 2592 static unsigned long pagemap_hugetlb_category(struct vm_area_struct *vma, 2593 pte_t pte) 2594 { 2595 unsigned long categories = PAGE_IS_HUGE; 2596 2597 if (pte_none(pte)) 2598 return categories; 2599 2600 /* 2601 * According to pagemap_hugetlb_range(), file-backed HugeTLB 2602 * page cannot be swapped. So PAGE_IS_FILE is not checked for 2603 * swapped pages. 2604 */ 2605 if (pte_present(pte)) { 2606 categories |= PAGE_IS_PRESENT; 2607 2608 if (!huge_pte_uffd(pte)) { 2609 if (userfaultfd_wp(vma)) 2610 categories |= PAGE_IS_WRITTEN; 2611 if (userfaultfd_rwp(vma)) 2612 categories |= PAGE_IS_ACCESSED; 2613 } 2614 if (!PageAnon(pte_page(pte))) 2615 categories |= PAGE_IS_FILE; 2616 if (is_zero_pfn(pte_pfn(pte))) 2617 categories |= PAGE_IS_PFNZERO; 2618 if (pte_soft_dirty(pte)) 2619 categories |= PAGE_IS_SOFT_DIRTY; 2620 } else { 2621 categories |= PAGE_IS_SWAPPED; 2622 2623 if (!pte_swp_uffd_any(pte)) { 2624 if (userfaultfd_wp(vma)) 2625 categories |= PAGE_IS_WRITTEN; 2626 if (userfaultfd_rwp(vma)) 2627 categories |= PAGE_IS_ACCESSED; 2628 } 2629 if (pte_swp_soft_dirty(pte)) 2630 categories |= PAGE_IS_SOFT_DIRTY; 2631 } 2632 2633 return categories; 2634 } 2635 2636 static void make_uffd_wp_huge_pte(struct vm_area_struct *vma, 2637 unsigned long addr, pte_t *ptep, 2638 pte_t ptent) 2639 { 2640 const unsigned long psize = huge_page_size(hstate_vma(vma)); 2641 softleaf_t entry; 2642 2643 if (huge_pte_none(ptent)) { 2644 set_huge_pte_at(vma->vm_mm, addr, ptep, 2645 make_pte_marker(PTE_MARKER_UFFD_WP), psize); 2646 return; 2647 } 2648 2649 entry = softleaf_from_pte(ptent); 2650 if (softleaf_is_hwpoison(entry) || softleaf_is_marker(entry)) 2651 return; 2652 2653 if (softleaf_is_migration(entry)) { 2654 set_huge_pte_at(vma->vm_mm, addr, ptep, 2655 pte_swp_mkuffd(ptent), psize); 2656 } else { 2657 pte_t old_pte, new_pte; 2658 2659 old_pte = huge_ptep_modify_prot_start(vma, addr, ptep); 2660 new_pte = huge_pte_mkuffd(old_pte); 2661 huge_ptep_modify_prot_commit(vma, addr, ptep, old_pte, new_pte); 2662 } 2663 } 2664 #endif /* CONFIG_HUGETLB_PAGE */ 2665 2666 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) 2667 static void pagemap_scan_backout_range(struct pagemap_scan_private *p, 2668 unsigned long addr, unsigned long end) 2669 { 2670 struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index]; 2671 2672 if (!p->vec_buf) 2673 return; 2674 2675 if (cur_buf->start != addr) 2676 cur_buf->end = addr; 2677 else 2678 cur_buf->start = cur_buf->end = 0; 2679 2680 p->found_pages -= (end - addr) / PAGE_SIZE; 2681 } 2682 #endif 2683 2684 static bool pagemap_scan_is_interesting_page(unsigned long categories, 2685 const struct pagemap_scan_private *p) 2686 { 2687 categories ^= p->arg.category_inverted; 2688 if ((categories & p->arg.category_mask) != p->arg.category_mask) 2689 return false; 2690 if (p->arg.category_anyof_mask && !(categories & p->arg.category_anyof_mask)) 2691 return false; 2692 2693 return true; 2694 } 2695 2696 static bool pagemap_scan_is_interesting_vma(unsigned long categories, 2697 const struct pagemap_scan_private *p) 2698 { 2699 unsigned long required = p->arg.category_mask & PAGE_IS_WPALLOWED; 2700 2701 categories ^= p->arg.category_inverted; 2702 if ((categories & required) != required) 2703 return false; 2704 2705 return true; 2706 } 2707 2708 static int pagemap_scan_test_walk(unsigned long start, unsigned long end, 2709 struct mm_walk *walk) 2710 { 2711 struct pagemap_scan_private *p = walk->private; 2712 struct vm_area_struct *vma = walk->vma; 2713 unsigned long vma_category = 0; 2714 bool wp_allowed = userfaultfd_wp_async(vma) && 2715 userfaultfd_wp_use_markers(vma); 2716 2717 if (!wp_allowed) { 2718 /* User requested explicit failure over wp-async capability */ 2719 if (p->arg.flags & PM_SCAN_CHECK_WPASYNC) 2720 return -EPERM; 2721 /* 2722 * User requires wr-protect, and allows silently skipping 2723 * unsupported vmas. 2724 */ 2725 if (p->arg.flags & PM_SCAN_WP_MATCHING) 2726 return 1; 2727 /* 2728 * Then the request doesn't involve wr-protects at all, 2729 * fall through to the rest checks, and allow vma walk. 2730 */ 2731 } 2732 2733 if (vma->vm_flags & VM_PFNMAP) 2734 return 1; 2735 2736 if (wp_allowed) 2737 vma_category |= PAGE_IS_WPALLOWED; 2738 2739 if (vma->vm_flags & VM_SOFTDIRTY) 2740 vma_category |= PAGE_IS_SOFT_DIRTY; 2741 2742 if (!pagemap_scan_is_interesting_vma(vma_category, p)) 2743 return 1; 2744 2745 p->cur_vma_category = vma_category; 2746 2747 return 0; 2748 } 2749 2750 static bool pagemap_scan_push_range(unsigned long categories, 2751 struct pagemap_scan_private *p, 2752 unsigned long addr, unsigned long end) 2753 { 2754 struct page_region *cur_buf = &p->vec_buf[p->vec_buf_index]; 2755 2756 /* 2757 * When there is no output buffer provided at all, the sentinel values 2758 * won't match here. There is no other way for `cur_buf->end` to be 2759 * non-zero other than it being non-empty. 2760 */ 2761 if (addr == cur_buf->end && categories == cur_buf->categories) { 2762 cur_buf->end = end; 2763 return true; 2764 } 2765 2766 if (cur_buf->end) { 2767 if (p->vec_buf_index >= p->vec_buf_len - 1) 2768 return false; 2769 2770 cur_buf = &p->vec_buf[++p->vec_buf_index]; 2771 } 2772 2773 cur_buf->start = addr; 2774 cur_buf->end = end; 2775 cur_buf->categories = categories; 2776 2777 return true; 2778 } 2779 2780 static int pagemap_scan_output(unsigned long categories, 2781 struct pagemap_scan_private *p, 2782 unsigned long addr, unsigned long *end) 2783 { 2784 unsigned long n_pages, total_pages; 2785 int ret = 0; 2786 2787 if (!p->vec_buf) 2788 return 0; 2789 2790 categories &= p->arg.return_mask; 2791 2792 n_pages = (*end - addr) / PAGE_SIZE; 2793 if (check_add_overflow(p->found_pages, n_pages, &total_pages) || 2794 total_pages > p->arg.max_pages) { 2795 size_t n_too_much = total_pages - p->arg.max_pages; 2796 *end -= n_too_much * PAGE_SIZE; 2797 n_pages -= n_too_much; 2798 ret = -ENOSPC; 2799 } 2800 2801 if (!pagemap_scan_push_range(categories, p, addr, *end)) { 2802 *end = addr; 2803 n_pages = 0; 2804 ret = -ENOSPC; 2805 } 2806 2807 p->found_pages += n_pages; 2808 if (ret) 2809 p->arg.walk_end = *end; 2810 2811 return ret; 2812 } 2813 2814 static int pagemap_scan_thp_entry(pmd_t *pmd, unsigned long start, 2815 unsigned long end, struct mm_walk *walk) 2816 { 2817 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 2818 struct pagemap_scan_private *p = walk->private; 2819 struct vm_area_struct *vma = walk->vma; 2820 unsigned long categories; 2821 spinlock_t *ptl; 2822 int ret = 0; 2823 2824 ptl = pmd_trans_huge_lock(pmd, vma); 2825 if (!ptl) 2826 return -ENOENT; 2827 2828 categories = p->cur_vma_category | 2829 pagemap_thp_category(p, vma, start, *pmd); 2830 2831 if (!pagemap_scan_is_interesting_page(categories, p)) 2832 goto out_unlock; 2833 2834 ret = pagemap_scan_output(categories, p, start, &end); 2835 if (start == end) 2836 goto out_unlock; 2837 2838 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 2839 goto out_unlock; 2840 if (~categories & PAGE_IS_WRITTEN) 2841 goto out_unlock; 2842 2843 /* 2844 * Break huge page into small pages if the WP operation 2845 * needs to be performed on a portion of the huge page. 2846 */ 2847 if (end != start + HPAGE_SIZE) { 2848 spin_unlock(ptl); 2849 split_huge_pmd(vma, pmd, start); 2850 pagemap_scan_backout_range(p, start, end); 2851 /* Report as if there was no THP */ 2852 return -ENOENT; 2853 } 2854 2855 make_uffd_wp_pmd(vma, start, pmd); 2856 flush_tlb_range(vma, start, end); 2857 out_unlock: 2858 spin_unlock(ptl); 2859 return ret; 2860 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 2861 return -ENOENT; 2862 #endif 2863 } 2864 2865 static int pagemap_scan_pmd_entry(pmd_t *pmd, unsigned long start, 2866 unsigned long end, struct mm_walk *walk) 2867 { 2868 struct pagemap_scan_private *p = walk->private; 2869 struct vm_area_struct *vma = walk->vma; 2870 unsigned long addr, flush_end = 0; 2871 pte_t *pte, *start_pte; 2872 spinlock_t *ptl; 2873 int ret; 2874 2875 ret = pagemap_scan_thp_entry(pmd, start, end, walk); 2876 if (ret != -ENOENT) 2877 return ret; 2878 2879 ret = 0; 2880 start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, start, &ptl); 2881 if (!pte) { 2882 walk->action = ACTION_AGAIN; 2883 return 0; 2884 } 2885 2886 lazy_mmu_mode_enable(); 2887 2888 if ((p->arg.flags & PM_SCAN_WP_MATCHING) && !p->vec_out) { 2889 /* Fast path for performing exclusive WP */ 2890 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { 2891 pte_t ptent = ptep_get(pte); 2892 2893 if ((pte_present(ptent) && pte_uffd(ptent)) || 2894 pte_swp_uffd_any(ptent)) 2895 continue; 2896 make_uffd_wp_pte(vma, addr, pte, ptent); 2897 if (!flush_end) 2898 start = addr; 2899 flush_end = addr + PAGE_SIZE; 2900 } 2901 goto flush_and_return; 2902 } 2903 2904 if (userfaultfd_wp(vma) && !p->arg.category_anyof_mask && 2905 !p->arg.category_inverted && 2906 p->arg.category_mask == PAGE_IS_WRITTEN && 2907 p->arg.return_mask == PAGE_IS_WRITTEN) { 2908 for (addr = start; addr < end; pte++, addr += PAGE_SIZE) { 2909 unsigned long next = addr + PAGE_SIZE; 2910 pte_t ptent = ptep_get(pte); 2911 2912 if ((pte_present(ptent) && pte_uffd(ptent)) || 2913 pte_swp_uffd_any(ptent)) 2914 continue; 2915 ret = pagemap_scan_output(p->cur_vma_category | PAGE_IS_WRITTEN, 2916 p, addr, &next); 2917 if (next == addr) 2918 break; 2919 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 2920 continue; 2921 make_uffd_wp_pte(vma, addr, pte, ptent); 2922 if (!flush_end) 2923 start = addr; 2924 flush_end = next; 2925 } 2926 goto flush_and_return; 2927 } 2928 2929 for (addr = start; addr != end; pte++, addr += PAGE_SIZE) { 2930 pte_t ptent = ptep_get(pte); 2931 unsigned long categories = p->cur_vma_category | 2932 pagemap_page_category(p, vma, addr, ptent); 2933 unsigned long next = addr + PAGE_SIZE; 2934 2935 if (!pagemap_scan_is_interesting_page(categories, p)) 2936 continue; 2937 2938 ret = pagemap_scan_output(categories, p, addr, &next); 2939 if (next == addr) 2940 break; 2941 2942 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 2943 continue; 2944 if (~categories & PAGE_IS_WRITTEN) 2945 continue; 2946 2947 make_uffd_wp_pte(vma, addr, pte, ptent); 2948 if (!flush_end) 2949 start = addr; 2950 flush_end = next; 2951 } 2952 2953 flush_and_return: 2954 if (flush_end) 2955 flush_tlb_range(vma, start, addr); 2956 2957 lazy_mmu_mode_disable(); 2958 pte_unmap_unlock(start_pte, ptl); 2959 2960 cond_resched(); 2961 return ret; 2962 } 2963 2964 #ifdef CONFIG_HUGETLB_PAGE 2965 static int pagemap_scan_hugetlb_entry(pte_t *ptep, unsigned long hmask, 2966 unsigned long start, unsigned long end, 2967 struct mm_walk *walk) 2968 { 2969 struct pagemap_scan_private *p = walk->private; 2970 struct vm_area_struct *vma = walk->vma; 2971 unsigned long categories; 2972 spinlock_t *ptl; 2973 int ret = 0; 2974 pte_t pte; 2975 2976 if (~p->arg.flags & PM_SCAN_WP_MATCHING) { 2977 /* Go the short route when not write-protecting pages. */ 2978 2979 pte = huge_ptep_get(walk->mm, start, ptep); 2980 categories = p->cur_vma_category | 2981 pagemap_hugetlb_category(vma, pte); 2982 2983 if (!pagemap_scan_is_interesting_page(categories, p)) 2984 return 0; 2985 2986 return pagemap_scan_output(categories, p, start, &end); 2987 } 2988 2989 i_mmap_lock_write(vma->vm_file->f_mapping); 2990 ptl = huge_pte_lock(hstate_vma(vma), vma->vm_mm, ptep); 2991 2992 pte = huge_ptep_get(walk->mm, start, ptep); 2993 categories = p->cur_vma_category | pagemap_hugetlb_category(vma, pte); 2994 2995 if (!pagemap_scan_is_interesting_page(categories, p)) 2996 goto out_unlock; 2997 2998 ret = pagemap_scan_output(categories, p, start, &end); 2999 if (start == end) 3000 goto out_unlock; 3001 3002 if (~categories & PAGE_IS_WRITTEN) 3003 goto out_unlock; 3004 3005 if (end != start + huge_page_size(hstate_vma(vma))) { 3006 /* Partial HugeTLB page WP isn't possible. */ 3007 pagemap_scan_backout_range(p, start, end); 3008 p->arg.walk_end = start; 3009 ret = 0; 3010 goto out_unlock; 3011 } 3012 3013 make_uffd_wp_huge_pte(vma, start, ptep, pte); 3014 flush_hugetlb_tlb_range(vma, start, end); 3015 3016 out_unlock: 3017 spin_unlock(ptl); 3018 i_mmap_unlock_write(vma->vm_file->f_mapping); 3019 3020 return ret; 3021 } 3022 3023 /* 3024 * Write-protect the unpopulated hugetlb entries covering [addr, end) by 3025 * installing uffd-wp markers inline, exactly as pagemap_scan_hugetlb_entry() 3026 * does for populated entries. 3027 * 3028 * walk_hugetlb_range() currently calls ->pte_hole() once per huge page, so the 3029 * loop normally runs a single iteration; it is written to cover the full range 3030 * in case the walker ever coalesces adjacent holes. 3031 * 3032 * The obvious route -- uffd_wp_range() -> hugetlb_change_protection() -- 3033 * cannot be used here: it takes hugetlb_vma_lock_write(), but the page-table 3034 * walker (walk_hugetlb_range()) already holds hugetlb_vma_lock_read() on the 3035 * same VMA, so the scanning thread would deadlock against itself. PMD sharing 3036 * is disabled on uffd-wp VMAs (hugetlb_unshare_all_pmds() at registration), so 3037 * the vma lock guards nothing that matters for these entries anyway. 3038 */ 3039 static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, 3040 unsigned long addr, unsigned long end) 3041 { 3042 struct hstate *h = hstate_vma(vma); 3043 unsigned long psize = huge_page_size(h); 3044 struct mm_struct *mm = vma->vm_mm; 3045 spinlock_t *ptl; 3046 pte_t *ptep; 3047 pte_t pte; 3048 3049 for (addr = ALIGN_DOWN(addr, psize); addr < end; addr += psize) { 3050 ptep = huge_pte_alloc(mm, vma, addr, psize); 3051 if (!ptep) 3052 return -ENOMEM; 3053 3054 i_mmap_lock_write(vma->vm_file->f_mapping); 3055 ptl = huge_pte_lock(h, mm, ptep); 3056 pte = huge_ptep_get(mm, addr, ptep); 3057 make_uffd_wp_huge_pte(vma, addr, ptep, pte); 3058 /* 3059 * A none entry has no cached translation, so installing the 3060 * marker needs no TLB flush. Flush only if a fault populated 3061 * the entry between huge_pte_alloc() and the page table lock. 3062 */ 3063 if (!huge_pte_none(pte)) 3064 flush_hugetlb_tlb_range(vma, addr, addr + psize); 3065 spin_unlock(ptl); 3066 i_mmap_unlock_write(vma->vm_file->f_mapping); 3067 } 3068 3069 return 0; 3070 } 3071 #else 3072 #define pagemap_scan_hugetlb_entry NULL 3073 static int pagemap_scan_hugetlb_hole_wp(struct vm_area_struct *vma, 3074 unsigned long addr, unsigned long end) 3075 { 3076 return 0; 3077 } 3078 #endif 3079 3080 static int pagemap_scan_pte_hole(unsigned long addr, unsigned long end, 3081 int depth, struct mm_walk *walk) 3082 { 3083 struct pagemap_scan_private *p = walk->private; 3084 struct vm_area_struct *vma = walk->vma; 3085 unsigned long categories; 3086 int ret, err; 3087 3088 if (!vma) 3089 return 0; 3090 3091 /* 3092 * In a uffd-wp VMA an unpopulated range is treated as written: 3093 * uffd-wp registration populates page tables and installs markers 3094 * with WP_UNPOPULATED, so a missing marker means the range was 3095 * zapped. See the pte_none() handling in pagemap_page_category(). 3096 * 3097 * hugetlb differs, see pagemap_hugetlb_category(). 3098 */ 3099 categories = p->cur_vma_category; 3100 if (userfaultfd_wp(vma) && !is_vm_hugetlb_page(vma)) 3101 categories |= PAGE_IS_WRITTEN; 3102 3103 if (!pagemap_scan_is_interesting_page(categories, p)) 3104 return 0; 3105 3106 ret = pagemap_scan_output(categories, p, addr, &end); 3107 if (addr == end) 3108 return ret; 3109 3110 if (~p->arg.flags & PM_SCAN_WP_MATCHING) 3111 return ret; 3112 3113 if (is_vm_hugetlb_page(vma)) 3114 err = pagemap_scan_hugetlb_hole_wp(vma, addr, end); 3115 else 3116 err = uffd_wp_range(vma, addr, end - addr, true); 3117 if (err < 0) 3118 ret = err; 3119 3120 return ret; 3121 } 3122 3123 static const struct mm_walk_ops pagemap_scan_ops = { 3124 .test_walk = pagemap_scan_test_walk, 3125 .pmd_entry = pagemap_scan_pmd_entry, 3126 .pte_hole = pagemap_scan_pte_hole, 3127 .hugetlb_entry = pagemap_scan_hugetlb_entry, 3128 }; 3129 3130 static int pagemap_scan_get_args(struct pm_scan_arg *arg, 3131 unsigned long uarg) 3132 { 3133 if (copy_from_user(arg, (void __user *)uarg, sizeof(*arg))) 3134 return -EFAULT; 3135 3136 if (arg->size != sizeof(struct pm_scan_arg)) 3137 return -EINVAL; 3138 3139 /* Validate requested features */ 3140 if (arg->flags & ~PM_SCAN_FLAGS) 3141 return -EINVAL; 3142 if ((arg->category_inverted | arg->category_mask | 3143 arg->category_anyof_mask | arg->return_mask) & ~PM_SCAN_CATEGORIES) 3144 return -EINVAL; 3145 3146 arg->start = untagged_addr((unsigned long)arg->start); 3147 arg->end = untagged_addr((unsigned long)arg->end); 3148 arg->vec = untagged_addr((unsigned long)arg->vec); 3149 3150 /* Validate memory pointers */ 3151 if (!IS_ALIGNED(arg->start, PAGE_SIZE)) 3152 return -EINVAL; 3153 if (!access_ok((void __user *)(long)arg->start, arg->end - arg->start)) 3154 return -EFAULT; 3155 if (!arg->vec && arg->vec_len) 3156 return -EINVAL; 3157 if (UINT_MAX == SIZE_MAX && arg->vec_len > SIZE_MAX) 3158 return -EINVAL; 3159 if (arg->vec && !access_ok((void __user *)(long)arg->vec, 3160 size_mul(arg->vec_len, sizeof(struct page_region)))) 3161 return -EFAULT; 3162 3163 /* Fixup default values */ 3164 arg->end = ALIGN(arg->end, PAGE_SIZE); 3165 arg->walk_end = 0; 3166 if (!arg->max_pages) 3167 arg->max_pages = ULONG_MAX; 3168 3169 return 0; 3170 } 3171 3172 static int pagemap_scan_writeback_args(struct pm_scan_arg *arg, 3173 unsigned long uargl) 3174 { 3175 struct pm_scan_arg __user *uarg = (void __user *)uargl; 3176 3177 if (copy_to_user(&uarg->walk_end, &arg->walk_end, sizeof(arg->walk_end))) 3178 return -EFAULT; 3179 3180 return 0; 3181 } 3182 3183 static int pagemap_scan_init_bounce_buffer(struct pagemap_scan_private *p) 3184 { 3185 if (!p->arg.vec_len) 3186 return 0; 3187 3188 p->vec_buf_len = min_t(size_t, PAGEMAP_WALK_SIZE >> PAGE_SHIFT, 3189 p->arg.vec_len); 3190 p->vec_buf = kmalloc_objs(*p->vec_buf, p->vec_buf_len); 3191 if (!p->vec_buf) 3192 return -ENOMEM; 3193 3194 p->vec_buf->start = p->vec_buf->end = 0; 3195 p->vec_out = (struct page_region __user *)(long)p->arg.vec; 3196 3197 return 0; 3198 } 3199 3200 static long pagemap_scan_flush_buffer(struct pagemap_scan_private *p) 3201 { 3202 const struct page_region *buf = p->vec_buf; 3203 long n = p->vec_buf_index; 3204 3205 if (!p->vec_buf) 3206 return 0; 3207 3208 if (buf[n].end != buf[n].start) 3209 n++; 3210 3211 if (!n) 3212 return 0; 3213 3214 if (copy_to_user(p->vec_out, buf, n * sizeof(*buf))) 3215 return -EFAULT; 3216 3217 p->arg.vec_len -= n; 3218 p->vec_out += n; 3219 3220 p->vec_buf_index = 0; 3221 p->vec_buf_len = min_t(size_t, p->vec_buf_len, p->arg.vec_len); 3222 p->vec_buf->start = p->vec_buf->end = 0; 3223 3224 return n; 3225 } 3226 3227 static long do_pagemap_scan(struct mm_struct *mm, unsigned long uarg) 3228 { 3229 struct pagemap_scan_private p = {0}; 3230 unsigned long walk_start; 3231 size_t n_ranges_out = 0; 3232 int ret; 3233 3234 ret = pagemap_scan_get_args(&p.arg, uarg); 3235 if (ret) 3236 return ret; 3237 3238 p.masks_of_interest = p.arg.category_mask | p.arg.category_anyof_mask | 3239 p.arg.return_mask; 3240 ret = pagemap_scan_init_bounce_buffer(&p); 3241 if (ret) 3242 return ret; 3243 3244 for (walk_start = p.arg.start; walk_start < p.arg.end; 3245 walk_start = p.arg.walk_end) { 3246 struct mmu_notifier_range range; 3247 long n_out; 3248 3249 if (fatal_signal_pending(current)) { 3250 ret = -EINTR; 3251 break; 3252 } 3253 3254 ret = mmap_read_lock_killable(mm); 3255 if (ret) 3256 break; 3257 3258 /* Protection change for the range is going to happen. */ 3259 if (p.arg.flags & PM_SCAN_WP_MATCHING) { 3260 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_VMA, 0, 3261 mm, walk_start, p.arg.end); 3262 mmu_notifier_invalidate_range_start(&range); 3263 } 3264 3265 ret = walk_page_range(mm, walk_start, p.arg.end, 3266 &pagemap_scan_ops, &p); 3267 3268 if (p.arg.flags & PM_SCAN_WP_MATCHING) 3269 mmu_notifier_invalidate_range_end(&range); 3270 3271 mmap_read_unlock(mm); 3272 3273 n_out = pagemap_scan_flush_buffer(&p); 3274 if (n_out < 0) 3275 ret = n_out; 3276 else 3277 n_ranges_out += n_out; 3278 3279 if (ret != -ENOSPC) 3280 break; 3281 3282 if (p.arg.vec_len == 0 || p.found_pages == p.arg.max_pages) 3283 break; 3284 } 3285 3286 /* ENOSPC signifies early stop (buffer full) from the walk. */ 3287 if (!ret || ret == -ENOSPC) 3288 ret = n_ranges_out; 3289 3290 /* The walk_end isn't set when ret is zero */ 3291 if (!p.arg.walk_end) 3292 p.arg.walk_end = p.arg.end; 3293 if (pagemap_scan_writeback_args(&p.arg, uarg)) 3294 ret = -EFAULT; 3295 3296 kfree(p.vec_buf); 3297 return ret; 3298 } 3299 3300 static long do_pagemap_cmd(struct file *file, unsigned int cmd, 3301 unsigned long arg) 3302 { 3303 struct mm_struct *mm = file->private_data; 3304 3305 switch (cmd) { 3306 case PAGEMAP_SCAN: 3307 return do_pagemap_scan(mm, arg); 3308 3309 default: 3310 return -EINVAL; 3311 } 3312 } 3313 3314 const struct file_operations proc_pagemap_operations = { 3315 .llseek = mem_lseek, /* borrow this */ 3316 .read = pagemap_read, 3317 .open = pagemap_open, 3318 .release = pagemap_release, 3319 .unlocked_ioctl = do_pagemap_cmd, 3320 .compat_ioctl = do_pagemap_cmd, 3321 }; 3322 #endif /* CONFIG_PROC_PAGE_MONITOR */ 3323 3324 #ifdef CONFIG_NUMA 3325 3326 struct numa_maps { 3327 unsigned long pages; 3328 unsigned long anon; 3329 unsigned long active; 3330 unsigned long writeback; 3331 unsigned long mapcount_max; 3332 unsigned long dirty; 3333 unsigned long swapcache; 3334 unsigned long node[MAX_NUMNODES]; 3335 }; 3336 3337 struct numa_maps_private { 3338 struct proc_maps_private proc_maps; 3339 struct numa_maps md; 3340 }; 3341 3342 static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty, 3343 unsigned long nr_pages) 3344 { 3345 struct folio *folio = page_folio(page); 3346 int count; 3347 3348 if (IS_ENABLED(CONFIG_PAGE_MAPCOUNT)) 3349 count = folio_precise_page_mapcount(folio, page); 3350 else 3351 count = folio_average_page_mapcount(folio); 3352 3353 md->pages += nr_pages; 3354 if (pte_dirty || folio_test_dirty(folio)) 3355 md->dirty += nr_pages; 3356 3357 if (folio_test_swapcache(folio)) 3358 md->swapcache += nr_pages; 3359 3360 if (folio_test_active(folio) || folio_test_unevictable(folio)) 3361 md->active += nr_pages; 3362 3363 if (folio_test_writeback(folio)) 3364 md->writeback += nr_pages; 3365 3366 if (folio_test_anon(folio)) 3367 md->anon += nr_pages; 3368 3369 if (count > md->mapcount_max) 3370 md->mapcount_max = count; 3371 3372 md->node[folio_nid(folio)] += nr_pages; 3373 } 3374 3375 static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma, 3376 unsigned long addr) 3377 { 3378 struct page *page; 3379 int nid; 3380 3381 if (!pte_present(pte)) 3382 return NULL; 3383 3384 page = vm_normal_page(vma, addr, pte); 3385 if (!page || is_zone_device_page(page)) 3386 return NULL; 3387 3388 if (PageReserved(page)) 3389 return NULL; 3390 3391 nid = page_to_nid(page); 3392 if (!node_isset(nid, node_states[N_MEMORY])) 3393 return NULL; 3394 3395 return page; 3396 } 3397 3398 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3399 static struct page *can_gather_numa_stats_pmd(pmd_t pmd, 3400 struct vm_area_struct *vma, 3401 unsigned long addr) 3402 { 3403 struct page *page; 3404 int nid; 3405 3406 if (!pmd_present(pmd)) 3407 return NULL; 3408 3409 page = vm_normal_page_pmd(vma, addr, pmd); 3410 if (!page) 3411 return NULL; 3412 3413 if (PageReserved(page)) 3414 return NULL; 3415 3416 nid = page_to_nid(page); 3417 if (!node_isset(nid, node_states[N_MEMORY])) 3418 return NULL; 3419 3420 return page; 3421 } 3422 #endif 3423 3424 static int gather_pte_stats(pmd_t *pmd, unsigned long addr, 3425 unsigned long end, struct mm_walk *walk) 3426 { 3427 struct numa_maps *md = walk->private; 3428 struct vm_area_struct *vma = walk->vma; 3429 spinlock_t *ptl; 3430 pte_t *orig_pte; 3431 pte_t *pte; 3432 3433 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3434 ptl = pmd_trans_huge_lock(pmd, vma); 3435 if (ptl) { 3436 struct page *page; 3437 3438 page = can_gather_numa_stats_pmd(*pmd, vma, addr); 3439 if (page) 3440 gather_stats(page, md, pmd_dirty(*pmd), 3441 HPAGE_PMD_SIZE/PAGE_SIZE); 3442 spin_unlock(ptl); 3443 return 0; 3444 } 3445 #endif 3446 orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl); 3447 if (!pte) { 3448 walk->action = ACTION_AGAIN; 3449 return 0; 3450 } 3451 do { 3452 pte_t ptent = ptep_get(pte); 3453 struct page *page = can_gather_numa_stats(ptent, vma, addr); 3454 if (!page) 3455 continue; 3456 gather_stats(page, md, pte_dirty(ptent), 1); 3457 3458 } while (pte++, addr += PAGE_SIZE, addr != end); 3459 pte_unmap_unlock(orig_pte, ptl); 3460 cond_resched(); 3461 return 0; 3462 } 3463 #ifdef CONFIG_HUGETLB_PAGE 3464 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, 3465 unsigned long addr, unsigned long end, struct mm_walk *walk) 3466 { 3467 pte_t huge_pte; 3468 struct numa_maps *md; 3469 struct page *page; 3470 spinlock_t *ptl; 3471 3472 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte); 3473 huge_pte = huge_ptep_get(walk->mm, addr, pte); 3474 if (!pte_present(huge_pte)) 3475 goto out; 3476 3477 page = pte_page(huge_pte); 3478 3479 md = walk->private; 3480 gather_stats(page, md, pte_dirty(huge_pte), 1); 3481 out: 3482 spin_unlock(ptl); 3483 return 0; 3484 } 3485 3486 #else 3487 static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask, 3488 unsigned long addr, unsigned long end, struct mm_walk *walk) 3489 { 3490 return 0; 3491 } 3492 #endif 3493 3494 static const struct mm_walk_ops show_numa_ops = { 3495 .hugetlb_entry = gather_hugetlb_stats, 3496 .pmd_entry = gather_pte_stats, 3497 .walk_lock = PGWALK_RDLOCK, 3498 }; 3499 3500 #ifdef CONFIG_PER_VMA_LOCK 3501 static const struct mm_walk_ops show_numa_vma_lock_ops = { 3502 .hugetlb_entry = gather_hugetlb_stats, 3503 .pmd_entry = gather_pte_stats, 3504 .walk_lock = PGWALK_VMA_RDLOCK_VERIFY, 3505 }; 3506 3507 static inline const struct mm_walk_ops * 3508 get_show_numa_ops(struct proc_maps_private *priv) 3509 { 3510 if (priv->lock_ctx.mmap_locked) 3511 return &show_numa_ops; 3512 return &show_numa_vma_lock_ops; 3513 } 3514 3515 #else /* CONFIG_PER_VMA_LOCK */ 3516 3517 static inline const struct mm_walk_ops * 3518 get_show_numa_ops(struct proc_maps_private *priv) 3519 { 3520 return &show_numa_ops; 3521 } 3522 3523 #endif /* CONFIG_PER_VMA_LOCK */ 3524 3525 /* 3526 * Display pages allocated per node and memory policy via /proc. 3527 */ 3528 static int show_numa_map(struct seq_file *m, void *v) 3529 { 3530 struct numa_maps_private *numa_priv = m->private; 3531 struct proc_maps_private *proc_priv = &numa_priv->proc_maps; 3532 struct vm_area_struct *vma = v; 3533 struct numa_maps *md = &numa_priv->md; 3534 struct file *file = vma->vm_file; 3535 struct mm_struct *mm = vma->vm_mm; 3536 char buffer[64]; 3537 struct mempolicy *pol; 3538 pgoff_t ilx; 3539 int nid; 3540 3541 if (!mm) 3542 return 0; 3543 3544 /* Ensure we start with an empty set of numa_maps statistics. */ 3545 memset(md, 0, sizeof(*md)); 3546 3547 pol = __get_vma_policy(vma, vma->vm_start, &ilx); 3548 if (pol) { 3549 mpol_to_str(buffer, sizeof(buffer), pol); 3550 mpol_cond_put(pol); 3551 } else { 3552 mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy); 3553 } 3554 3555 seq_printf(m, "%08lx %s", vma->vm_start, buffer); 3556 3557 if (file) { 3558 seq_puts(m, " file="); 3559 seq_path(m, file_user_path(file), "\n\t= "); 3560 } else if (vma_is_initial_heap(vma)) { 3561 seq_puts(m, " heap"); 3562 } else if (vma_is_initial_stack(vma)) { 3563 seq_puts(m, " stack"); 3564 } 3565 3566 if (is_vm_hugetlb_page(vma)) 3567 seq_puts(m, " huge"); 3568 3569 /* Skip walking pages if gate VMA */ 3570 if (vma != get_gate_vma(proc_priv->lock_ctx.mm)) { 3571 /* Might sleep. Drop RCU read lock but keep the VMA locked. */ 3572 drop_rcu(proc_priv); 3573 walk_page_vma(vma, get_show_numa_ops(proc_priv), md); 3574 reacquire_rcu(proc_priv); 3575 } 3576 3577 if (!md->pages) 3578 goto out; 3579 3580 if (md->anon) 3581 seq_printf(m, " anon=%lu", md->anon); 3582 3583 if (md->dirty) 3584 seq_printf(m, " dirty=%lu", md->dirty); 3585 3586 if (md->pages != md->anon && md->pages != md->dirty) 3587 seq_printf(m, " mapped=%lu", md->pages); 3588 3589 if (md->mapcount_max > 1) 3590 seq_printf(m, " mapmax=%lu", md->mapcount_max); 3591 3592 if (md->swapcache) 3593 seq_printf(m, " swapcache=%lu", md->swapcache); 3594 3595 if (md->active < md->pages && !is_vm_hugetlb_page(vma)) 3596 seq_printf(m, " active=%lu", md->active); 3597 3598 if (md->writeback) 3599 seq_printf(m, " writeback=%lu", md->writeback); 3600 3601 for_each_node_state(nid, N_MEMORY) 3602 if (md->node[nid]) 3603 seq_printf(m, " N%d=%lu", nid, md->node[nid]); 3604 3605 seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10); 3606 out: 3607 seq_putc(m, '\n'); 3608 return 0; 3609 } 3610 3611 static const struct seq_operations proc_pid_numa_maps_op = { 3612 .start = m_start, 3613 .next = m_next, 3614 .stop = m_stop, 3615 .show = show_numa_map, 3616 }; 3617 3618 static int pid_numa_maps_open(struct inode *inode, struct file *file) 3619 { 3620 return proc_maps_open(inode, file, &proc_pid_numa_maps_op, 3621 sizeof(struct numa_maps_private)); 3622 } 3623 3624 const struct file_operations proc_pid_numa_maps_operations = { 3625 .open = pid_numa_maps_open, 3626 .read = seq_read, 3627 .llseek = seq_lseek, 3628 .release = proc_map_release, 3629 }; 3630 3631 #endif /* CONFIG_NUMA */ 3632